Compare commits

...

40 Commits

Author SHA1 Message Date
dotnet-maestro[bot] 6c5908f3b3 Update dependencies from https://github.com/dotnet/arcade build 20220905.1 4 years ago
dotnet-maestro[bot] 8973236abf Update dependencies from https://github.com/dotnet/arcade build 20220902.1 4 years ago
dotnet-maestro[bot] 2ba69dd78c Update dependencies from https://github.com/dotnet/arcade build 20220831.1 4 years ago
dotnet-maestro[bot] b96e3b0938 Update dependencies from https://github.com/dotnet/arcade build 20220830.3 4 years ago
dotnet-maestro[bot] 7fede15057 Update dependencies from https://github.com/dotnet/arcade build 20220826.8 4 years ago
dotnet-maestro[bot] 7d02fa8424 Update dependencies from https://github.com/dotnet/arcade build 20220826.1 4 years ago
dotnet-maestro[bot] d4950531d6 Update dependencies from https://github.com/dotnet/arcade build 20220825.1 4 years ago
dotnet-maestro[bot] 259daf1741 Update dependencies from https://github.com/dotnet/arcade build 20220824.3 4 years ago
dotnet-maestro[bot] 065d812637 Update dependencies from https://github.com/dotnet/arcade build 20220823.2 4 years ago
dotnet-maestro[bot] 6c526be7d8 Update dependencies from https://github.com/dotnet/arcade build 20220819.1 4 years ago
dotnet-maestro[bot] abe6887f21 Update dependencies from https://github.com/dotnet/arcade build 20220818.4 4 years ago
dotnet-maestro[bot] bfdf2e1410 Update dependencies from https://github.com/dotnet/arcade build 20220817.2 4 years ago
dotnet-maestro[bot] 81d2d29af4 Update dependencies from https://github.com/dotnet/arcade build 20220816.5 4 years ago
dotnet-maestro[bot] b646844f03 Update dependencies from https://github.com/dotnet/arcade build 20220811.2 4 years ago
dotnet-maestro[bot] 9cc66c72f9 Update dependencies from https://github.com/dotnet/arcade build 20220810.3 4 years ago
dotnet-maestro[bot] 5f5bafcb26 Update dependencies from https://github.com/dotnet/arcade build 20220808.3 4 years ago
dotnet-maestro[bot] 96ca105369 Update dependencies from https://github.com/dotnet/arcade build 20220805.6 4 years ago
dotnet-maestro[bot] 5e2c7b3fa9 Update dependencies from https://github.com/dotnet/arcade build 20220805.2 4 years ago
dotnet-maestro[bot] 80faf1d7df Update dependencies from https://github.com/dotnet/arcade build 20220803.1 4 years ago
dotnet-maestro[bot] 0a49491343 Update dependencies from https://github.com/dotnet/arcade build 20220802.4 4 years ago
dotnet-maestro[bot] ae2b0633c9 Update dependencies from https://github.com/dotnet/arcade build 20220801.3 4 years ago
dotnet-maestro[bot] 37deb2721f Update dependencies from https://github.com/dotnet/arcade build 20220729.10 4 years ago
dotnet-maestro[bot] 0ac322a067 Update dependencies from https://github.com/dotnet/arcade build 20220728.16 4 years ago
dotnet-maestro[bot] 1fc7f2f319 Update dependencies from https://github.com/dotnet/arcade build 20220727.15 4 years ago
dotnet-maestro[bot] 4a0b9dc258 Update dependencies from https://github.com/dotnet/arcade build 20220726.3 4 years ago
dotnet-maestro[bot] 91a4aad62b Update dependencies from https://github.com/dotnet/arcade build 20220725.4 4 years ago
dotnet-maestro[bot] 1f896ac6f4 Update dependencies from https://github.com/dotnet/arcade build 20220722.1 4 years ago
dotnet-maestro[bot] b3e6e5122d Update dependencies from https://github.com/dotnet/arcade build 20220721.8 4 years ago
dotnet-maestro[bot] 44e51dfc04 Update dependencies from https://github.com/dotnet/arcade build 20220720.1 4 years ago
dotnet-maestro[bot] b44f7b2da7 Update dependencies from https://github.com/dotnet/arcade build 20220719.9 4 years ago
dotnet-maestro[bot] 7576554346 Update dependencies from https://github.com/dotnet/arcade build 20220718.5 4 years ago
dotnet-maestro[bot] fb16dd57d1 Update dependencies from https://github.com/dotnet/arcade build 20220717.1 4 years ago
dotnet-maestro[bot] bd4858a4d0 Update dependencies from https://github.com/dotnet/arcade build 20220715.4 4 years ago
dotnet-maestro[bot] b698e51e00 Update dependencies from https://github.com/dotnet/arcade build 20220714.1 4 years ago
dotnet-maestro[bot] 94c74f053e Update dependencies from https://github.com/dotnet/arcade build 20220713.1 4 years ago
dotnet-maestro[bot] 8603ccb7e4 Update dependencies from https://github.com/dotnet/arcade build 20220712.1 4 years ago
dotnet-maestro[bot] 1719dd7049 Update dependencies from https://github.com/dotnet/arcade build 20220708.3 4 years ago
dotnet-maestro[bot] 7aed022e7c Update dependencies from https://github.com/dotnet/arcade build 20220706.3 4 years ago
dotnet-maestro[bot] 4ca31fabb3 Update dependencies from https://github.com/dotnet/arcade build 20220705.4 4 years ago
dotnet-maestro[bot] 35a717a4db Update dependencies from https://github.com/dotnet/arcade build 20220704.1 4 years ago
  1. 20
      eng/Version.Details.xml
  2. 18
      eng/common/SetupNugetSources.ps1
  3. 68
      eng/common/SetupNugetSources.sh
  4. 17
      eng/common/build.sh
  5. 330
      eng/common/cross/build-rootfs.sh
  6. 1
      eng/common/cross/riscv64/sources.list.sid
  7. 98
      eng/common/cross/toolchain.cmake
  8. 47
      eng/common/generate-locproject.ps1
  9. 2
      eng/common/generate-sbom-prep.ps1
  10. 12
      eng/common/generate-sbom-prep.sh
  11. 4
      eng/common/init-tools-native.ps1
  12. 2
      eng/common/native/init-compiler.sh
  13. 2
      eng/common/sdk-task.ps1
  14. 37
      eng/common/sdl/sdl.ps1
  15. 2
      eng/common/templates/job/execute-sdl.yml
  16. 2
      eng/common/templates/job/onelocbuild.yml
  17. 2
      eng/common/templates/job/source-build.yml
  18. 8
      eng/common/templates/job/source-index-stage1.yml
  19. 2
      eng/common/templates/jobs/jobs.yml
  20. 10
      eng/common/templates/post-build/post-build.yml
  21. 58
      eng/common/templates/steps/execute-sdl.yml
  22. 15
      eng/common/tools.ps1
  23. 4
      global.json

20
eng/Version.Details.xml

@ -3,25 +3,25 @@
<ProductDependencies> <ProductDependencies>
</ProductDependencies> </ProductDependencies>
<ToolsetDependencies> <ToolsetDependencies>
<Dependency Name="Microsoft.DotNet.Arcade.Sdk" Version="7.0.0-beta.22327.2"> <Dependency Name="Microsoft.DotNet.Arcade.Sdk" Version="8.0.0-beta.22455.1">
<Uri>https://github.com/dotnet/arcade</Uri> <Uri>https://github.com/dotnet/arcade</Uri>
<Sha>a264eb13fea14125f3ef8d4056586cd66fa55309</Sha> <Sha>fca51250390d403827cf50c0df9dbc632514b739</Sha>
</Dependency> </Dependency>
<Dependency Name="Microsoft.DotNet.Build.Tasks.Feed" Version="7.0.0-beta.22327.2"> <Dependency Name="Microsoft.DotNet.Build.Tasks.Feed" Version="8.0.0-beta.22455.1">
<Uri>https://github.com/dotnet/arcade</Uri> <Uri>https://github.com/dotnet/arcade</Uri>
<Sha>a264eb13fea14125f3ef8d4056586cd66fa55309</Sha> <Sha>fca51250390d403827cf50c0df9dbc632514b739</Sha>
</Dependency> </Dependency>
<Dependency Name="Microsoft.DotNet.SignTool" Version="7.0.0-beta.22327.2"> <Dependency Name="Microsoft.DotNet.SignTool" Version="8.0.0-beta.22455.1">
<Uri>https://github.com/dotnet/arcade</Uri> <Uri>https://github.com/dotnet/arcade</Uri>
<Sha>a264eb13fea14125f3ef8d4056586cd66fa55309</Sha> <Sha>fca51250390d403827cf50c0df9dbc632514b739</Sha>
</Dependency> </Dependency>
<Dependency Name="Microsoft.DotNet.Helix.Sdk" Version="7.0.0-beta.22327.2"> <Dependency Name="Microsoft.DotNet.Helix.Sdk" Version="8.0.0-beta.22455.1">
<Uri>https://github.com/dotnet/arcade</Uri> <Uri>https://github.com/dotnet/arcade</Uri>
<Sha>a264eb13fea14125f3ef8d4056586cd66fa55309</Sha> <Sha>fca51250390d403827cf50c0df9dbc632514b739</Sha>
</Dependency> </Dependency>
<Dependency Name="Microsoft.DotNet.SwaggerGenerator.MSBuild" Version="7.0.0-beta.22327.2"> <Dependency Name="Microsoft.DotNet.SwaggerGenerator.MSBuild" Version="8.0.0-beta.22455.1">
<Uri>https://github.com/dotnet/arcade</Uri> <Uri>https://github.com/dotnet/arcade</Uri>
<Sha>a264eb13fea14125f3ef8d4056586cd66fa55309</Sha> <Sha>fca51250390d403827cf50c0df9dbc632514b739</Sha>
</Dependency> </Dependency>
<Dependency Name="Microsoft.DotNet.Maestro.Client" Version="1.1.0-beta.19556.4"> <Dependency Name="Microsoft.DotNet.Maestro.Client" Version="1.1.0-beta.19556.4">
<Uri>https://github.com/dotnet/arcade-services</Uri> <Uri>https://github.com/dotnet/arcade-services</Uri>

18
eng/common/SetupNugetSources.ps1

@ -146,22 +146,22 @@ $userName = "dn-bot"
# Insert credential nodes for Maestro's private feeds # Insert credential nodes for Maestro's private feeds
InsertMaestroPrivateFeedCredentials -Sources $sources -Creds $creds -Username $userName -Password $Password InsertMaestroPrivateFeedCredentials -Sources $sources -Creds $creds -Username $userName -Password $Password
# 3.1 uses a different feed url format so it's handled differently here
$dotnet31Source = $sources.SelectSingleNode("add[@key='dotnet3.1']") $dotnet31Source = $sources.SelectSingleNode("add[@key='dotnet3.1']")
if ($dotnet31Source -ne $null) { if ($dotnet31Source -ne $null) {
AddPackageSource -Sources $sources -SourceName "dotnet3.1-internal" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/_packaging/dotnet3.1-internal/nuget/v2" -Creds $creds -Username $userName -Password $Password AddPackageSource -Sources $sources -SourceName "dotnet3.1-internal" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/_packaging/dotnet3.1-internal/nuget/v2" -Creds $creds -Username $userName -Password $Password
AddPackageSource -Sources $sources -SourceName "dotnet3.1-internal-transport" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/_packaging/dotnet3.1-internal-transport/nuget/v2" -Creds $creds -Username $userName -Password $Password AddPackageSource -Sources $sources -SourceName "dotnet3.1-internal-transport" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/_packaging/dotnet3.1-internal-transport/nuget/v2" -Creds $creds -Username $userName -Password $Password
} }
$dotnet5Source = $sources.SelectSingleNode("add[@key='dotnet5']") $dotnetVersions = @('5','6','7')
if ($dotnet5Source -ne $null) {
AddPackageSource -Sources $sources -SourceName "dotnet5-internal" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/internal/_packaging/dotnet5-internal/nuget/v2" -Creds $creds -Username $userName -Password $Password
AddPackageSource -Sources $sources -SourceName "dotnet5-internal-transport" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/internal/_packaging/dotnet5-internal-transport/nuget/v2" -Creds $creds -Username $userName -Password $Password
}
$dotnet6Source = $sources.SelectSingleNode("add[@key='dotnet6']") foreach ($dotnetVersion in $dotnetVersions) {
if ($dotnet6Source -ne $null) { $feedPrefix = "dotnet" + $dotnetVersion;
AddPackageSource -Sources $sources -SourceName "dotnet6-internal" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/internal/_packaging/dotnet6-internal/nuget/v2" -Creds $creds -Username $userName -Password $Password $dotnetSource = $sources.SelectSingleNode("add[@key='$feedPrefix']")
AddPackageSource -Sources $sources -SourceName "dotnet6-internal-transport" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/internal/_packaging/dotnet6-internal-transport/nuget/v2" -Creds $creds -Username $userName -Password $Password if ($dotnetSource -ne $null) {
AddPackageSource -Sources $sources -SourceName "$feedPrefix-internal" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/internal/_packaging/$feedPrefix-internal/nuget/v2" -Creds $creds -Username $userName -Password $Password
AddPackageSource -Sources $sources -SourceName "$feedPrefix-internal-transport" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/internal/_packaging/$feedPrefix-internal-transport/nuget/v2" -Creds $creds -Username $userName -Password $Password
}
} }
$doc.Save($filename) $doc.Save($filename)

68
eng/common/SetupNugetSources.sh

@ -105,53 +105,33 @@ if [ "$?" == "0" ]; then
PackageSources+=('dotnet3.1-internal-transport') PackageSources+=('dotnet3.1-internal-transport')
fi fi
# Ensure dotnet5-internal and dotnet5-internal-transport are in the packageSources if the public dotnet5 feeds are present DotNetVersions=('5' '6' '7')
grep -i "<add key=\"dotnet5\"" $ConfigFile
if [ "$?" == "0" ]; then for DotNetVersion in ${DotNetVersions[@]} ; do
grep -i "<add key=\"dotnet5-internal\"" $ConfigFile FeedPrefix="dotnet${DotNetVersion}";
if [ "$?" != "0" ]; then grep -i "<add key=\"$FeedPrefix\"" $ConfigFile
echo "Adding dotnet5-internal to the packageSources." if [ "$?" == "0" ]; then
PackageSourcesNodeFooter="</packageSources>" grep -i "<add key=\"$FeedPrefix-internal\"" $ConfigFile
PackageSourceTemplate="${TB}<add key=\"dotnet5-internal\" value=\"https://pkgs.dev.azure.com/dnceng/internal/_packaging/dotnet5-internal/nuget/v2\" />" if [ "$?" != "0" ]; then
echo "Adding $FeedPrefix-internal to the packageSources."
sed -i.bak "s|$PackageSourcesNodeFooter|$PackageSourceTemplate${NL}$PackageSourcesNodeFooter|" $ConfigFile PackageSourcesNodeFooter="</packageSources>"
fi PackageSourceTemplate="${TB}<add key=\"$FeedPrefix-internal\" value=\"https://pkgs.dev.azure.com/dnceng/internal/_packaging/$FeedPrefix-internal/nuget/v2\" />"
PackageSources+=('dotnet5-internal')
sed -i.bak "s|$PackageSourcesNodeFooter|$PackageSourceTemplate${NL}$PackageSourcesNodeFooter|" $ConfigFile
grep -i "<add key=\"dotnet5-internal-transport\">" $ConfigFile fi
if [ "$?" != "0" ]; then PackageSources+=("$FeedPrefix-internal")
echo "Adding dotnet5-internal-transport to the packageSources."
PackageSourcesNodeFooter="</packageSources>"
PackageSourceTemplate="${TB}<add key=\"dotnet5-internal-transport\" value=\"https://pkgs.dev.azure.com/dnceng/internal/_packaging/dotnet5-internal-transport/nuget/v2\" />"
sed -i.bak "s|$PackageSourcesNodeFooter|$PackageSourceTemplate${NL}$PackageSourcesNodeFooter|" $ConfigFile
fi
PackageSources+=('dotnet5-internal-transport')
fi
# Ensure dotnet6-internal and dotnet6-internal-transport are in the packageSources if the public dotnet6 feeds are present
grep -i "<add key=\"dotnet6\"" $ConfigFile
if [ "$?" == "0" ]; then
grep -i "<add key=\"dotnet6-internal\"" $ConfigFile
if [ "$?" != "0" ]; then
echo "Adding dotnet6-internal to the packageSources."
PackageSourcesNodeFooter="</packageSources>"
PackageSourceTemplate="${TB}<add key=\"dotnet6-internal\" value=\"https://pkgs.dev.azure.com/dnceng/internal/_packaging/dotnet6-internal/nuget/v2\" />"
sed -i.bak "s|$PackageSourcesNodeFooter|$PackageSourceTemplate${NL}$PackageSourcesNodeFooter|" $ConfigFile grep -i "<add key=\"$FeedPrefix-internal-transport\">" $ConfigFile
fi if [ "$?" != "0" ]; then
PackageSources+=('dotnet6-internal') echo "Adding $FeedPrefix-internal-transport to the packageSources."
PackageSourcesNodeFooter="</packageSources>"
PackageSourceTemplate="${TB}<add key=\"$FeedPrefix-internal-transport\" value=\"https://pkgs.dev.azure.com/dnceng/internal/_packaging/$FeedPrefix-internal-transport/nuget/v2\" />"
grep -i "<add key=\"dotnet6-internal-transport\">" $ConfigFile sed -i.bak "s|$PackageSourcesNodeFooter|$PackageSourceTemplate${NL}$PackageSourcesNodeFooter|" $ConfigFile
if [ "$?" != "0" ]; then fi
echo "Adding dotnet6-internal-transport to the packageSources." PackageSources+=("$FeedPrefix-internal-transport")
PackageSourcesNodeFooter="</packageSources>"
PackageSourceTemplate="${TB}<add key=\"dotnet6-internal-transport\" value=\"https://pkgs.dev.azure.com/dnceng/internal/_packaging/dotnet6-internal-transport/nuget/v2\" />"
sed -i.bak "s|$PackageSourcesNodeFooter|$PackageSourceTemplate${NL}$PackageSourcesNodeFooter|" $ConfigFile
fi fi
PackageSources+=('dotnet6-internal-transport') done
fi
# I want things split line by line # I want things split line by line
PrevIFS=$IFS PrevIFS=$IFS

17
eng/common/build.sh

@ -19,6 +19,9 @@ usage()
echo "Actions:" echo "Actions:"
echo " --restore Restore dependencies (short: -r)" echo " --restore Restore dependencies (short: -r)"
echo " --build Build solution (short: -b)" echo " --build Build solution (short: -b)"
echo " --sourceBuild Source-build the solution (short: -sb)"
echo " Will additionally trigger the following actions: --restore, --build, --pack"
echo " If --configuration is not set explicitly, will also set it to 'Release'"
echo " --rebuild Rebuild solution" echo " --rebuild Rebuild solution"
echo " --test Run all unit tests in the solution (short: -t)" echo " --test Run all unit tests in the solution (short: -t)"
echo " --integrationTest Run all integration tests in the solution" echo " --integrationTest Run all integration tests in the solution"
@ -55,6 +58,7 @@ scriptroot="$( cd -P "$( dirname "$source" )" && pwd )"
restore=false restore=false
build=false build=false
source_build=false
rebuild=false rebuild=false
test=false test=false
integration_test=false integration_test=false
@ -73,7 +77,7 @@ exclude_ci_binary_log=false
pipelines_log=false pipelines_log=false
projects='' projects=''
configuration='Debug' configuration=''
prepare_machine=false prepare_machine=false
verbosity='minimal' verbosity='minimal'
runtime_source_feed='' runtime_source_feed=''
@ -119,6 +123,12 @@ while [[ $# > 0 ]]; do
-pack) -pack)
pack=true pack=true
;; ;;
-sourcebuild|-sb)
build=true
source_build=true
restore=true
pack=true
;;
-test|-t) -test|-t)
test=true test=true
;; ;;
@ -168,6 +178,10 @@ while [[ $# > 0 ]]; do
shift shift
done done
if [[ -z "$configuration" ]]; then
if [[ "$source_build" = true ]]; then configuration="Release"; else configuration="Debug"; fi
fi
if [[ "$ci" == true ]]; then if [[ "$ci" == true ]]; then
pipelines_log=true pipelines_log=true
node_reuse=false node_reuse=false
@ -205,6 +219,7 @@ function Build {
/p:RepoRoot="$repo_root" \ /p:RepoRoot="$repo_root" \
/p:Restore=$restore \ /p:Restore=$restore \
/p:Build=$build \ /p:Build=$build \
/p:ArcadeBuildFromSource=$source_build \
/p:Rebuild=$rebuild \ /p:Rebuild=$rebuild \
/p:Test=$test \ /p:Test=$test \
/p:Pack=$pack \ /p:Pack=$pack \

330
eng/common/cross/build-rootfs.sh

@ -5,22 +5,26 @@ set -e
usage() usage()
{ {
echo "Usage: $0 [BuildArch] [CodeName] [lldbx.y] [llvmx[.y]] [--skipunmount] --rootfsdir <directory>]" echo "Usage: $0 [BuildArch] [CodeName] [lldbx.y] [llvmx[.y]] [--skipunmount] --rootfsdir <directory>]"
echo "BuildArch can be: arm(default), armel, arm64, x86" echo "BuildArch can be: arm(default), arm64, armel, armv6, ppc64le, riscv64, s390x, x64, x86"
echo "CodeName - optional, Code name for Linux, can be: xenial(default), zesty, bionic, alpine, alpine3.13 or alpine3.14. If BuildArch is armel, LinuxCodeName is jessie(default) or tizen." echo "CodeName - optional, Code name for Linux, can be: xenial(default), zesty, bionic, alpine, alpine3.13 or alpine3.14. If BuildArch is armel, LinuxCodeName is jessie(default) or tizen."
echo " for FreeBSD can be: freebsd12, freebsd13" echo " for FreeBSD can be: freebsd12, freebsd13"
echo " for illumos can be: illumos." echo " for illumos can be: illumos"
echo " for Haiku can be: haiku."
echo "lldbx.y - optional, LLDB version, can be: lldb3.9(default), lldb4.0, lldb5.0, lldb6.0 no-lldb. Ignored for alpine and FreeBSD" echo "lldbx.y - optional, LLDB version, can be: lldb3.9(default), lldb4.0, lldb5.0, lldb6.0 no-lldb. Ignored for alpine and FreeBSD"
echo "llvmx[.y] - optional, LLVM version for LLVM related packages." echo "llvmx[.y] - optional, LLVM version for LLVM related packages."
echo "--skipunmount - optional, will skip the unmount of rootfs folder." echo "--skipunmount - optional, will skip the unmount of rootfs folder."
echo "--use-mirror - optional, use mirror URL to fetch resources, when available." echo "--use-mirror - optional, use mirror URL to fetch resources, when available."
echo "--jobs N - optional, restrict to N jobs."
exit 1 exit 1
} }
__CodeName=xenial __CodeName=xenial
__CrossDir=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd ) __CrossDir=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )
__InitialDir=$PWD
__BuildArch=arm __BuildArch=arm
__AlpineArch=armv7 __AlpineArch=armv7
__FreeBSDArch=arm
__FreeBSDMachineArch=armv7
__IllumosArch=arm7
__QEMUArch=arm __QEMUArch=arm
__UbuntuArch=armhf __UbuntuArch=armhf
__UbuntuRepo="http://ports.ubuntu.com/" __UbuntuRepo="http://ports.ubuntu.com/"
@ -40,7 +44,7 @@ __AlpinePackages+=" libedit"
# symlinks fixer # symlinks fixer
__UbuntuPackages+=" symlinks" __UbuntuPackages+=" symlinks"
# CoreCLR and CoreFX dependencies # runtime dependencies
__UbuntuPackages+=" libicu-dev" __UbuntuPackages+=" libicu-dev"
__UbuntuPackages+=" liblttng-ust-dev" __UbuntuPackages+=" liblttng-ust-dev"
__UbuntuPackages+=" libunwind8-dev" __UbuntuPackages+=" libunwind8-dev"
@ -51,7 +55,7 @@ __AlpinePackages+=" libunwind-dev"
__AlpinePackages+=" lttng-ust-dev" __AlpinePackages+=" lttng-ust-dev"
__AlpinePackages+=" compiler-rt-static" __AlpinePackages+=" compiler-rt-static"
# CoreFX dependencies # runtime libraries' dependencies
__UbuntuPackages+=" libcurl4-openssl-dev" __UbuntuPackages+=" libcurl4-openssl-dev"
__UbuntuPackages+=" libkrb5-dev" __UbuntuPackages+=" libkrb5-dev"
__UbuntuPackages+=" libssl-dev" __UbuntuPackages+=" libssl-dev"
@ -72,26 +76,38 @@ __FreeBSDPackages+=" openssl"
__FreeBSDPackages+=" krb5" __FreeBSDPackages+=" krb5"
__FreeBSDPackages+=" terminfo-db" __FreeBSDPackages+=" terminfo-db"
__IllumosPackages="icu-64.2nb2" __IllumosPackages="icu"
__IllumosPackages+=" mit-krb5-1.16.2nb4" __IllumosPackages+=" mit-krb5"
__IllumosPackages+=" openssl-1.1.1e" __IllumosPackages+=" openssl"
__IllumosPackages+=" zlib-1.2.11" __IllumosPackages+=" zlib"
__HaikuPackages="gmp"
__HaikuPackages+=" gmp_devel"
__HaikuPackages+=" krb5"
__HaikuPackages+=" krb5_devel"
__HaikuPackages+=" libiconv"
__HaikuPackages+=" libiconv_devel"
__HaikuPackages+=" llvm12_libunwind"
__HaikuPackages+=" llvm12_libunwind_devel"
__HaikuPackages+=" mpfr"
__HaikuPackages+=" mpfr_devel"
# ML.NET dependencies # ML.NET dependencies
__UbuntuPackages+=" libomp5" __UbuntuPackages+=" libomp5"
__UbuntuPackages+=" libomp-dev" __UbuntuPackages+=" libomp-dev"
__Keyring=
__UseMirror=0 __UseMirror=0
__UnprocessedBuildArgs= __UnprocessedBuildArgs=
while :; do while :; do
if [ $# -le 0 ]; then if [[ "$#" -le 0 ]]; then
break break
fi fi
lowerI="$(echo $1 | tr "[:upper:]" "[:lower:]")" lowerI="$(echo "$1" | tr "[:upper:]" "[:lower:]")"
case $lowerI in case $lowerI in
-?|-h|--help) -\?|-h|--help)
usage usage
exit 1 exit 1
;; ;;
@ -101,20 +117,13 @@ while :; do
__AlpineArch=armv7 __AlpineArch=armv7
__QEMUArch=arm __QEMUArch=arm
;; ;;
armv6)
__BuildArch=armv6
__UbuntuArch=armhf
__QEMUArch=arm
__UbuntuRepo="http://raspbian.raspberrypi.org/raspbian/"
__CodeName=buster
__LLDB_Package="liblldb-6.0-dev"
__Keyring="/usr/share/keyrings/raspbian-archive-keyring.gpg"
;;
arm64) arm64)
__BuildArch=arm64 __BuildArch=arm64
__UbuntuArch=arm64 __UbuntuArch=arm64
__AlpineArch=aarch64 __AlpineArch=aarch64
__QEMUArch=aarch64 __QEMUArch=aarch64
__FreeBSDArch=arm64
__FreeBSDMachineArch=aarch64
;; ;;
armel) armel)
__BuildArch=armel __BuildArch=armel
@ -122,6 +131,18 @@ while :; do
__UbuntuRepo="http://ftp.debian.org/debian/" __UbuntuRepo="http://ftp.debian.org/debian/"
__CodeName=jessie __CodeName=jessie
;; ;;
armv6)
__BuildArch=armv6
__UbuntuArch=armhf
__QEMUArch=arm
__UbuntuRepo="http://raspbian.raspberrypi.org/raspbian/"
__CodeName=buster
__LLDB_Package="liblldb-6.0-dev"
if [[ -e "/usr/share/keyrings/raspbian-archive-keyring.gpg" ]]; then
__Keyring="--keyring /usr/share/keyrings/raspbian-archive-keyring.gpg"
fi
;;
ppc64le) ppc64le)
__BuildArch=ppc64le __BuildArch=ppc64le
__UbuntuArch=ppc64el __UbuntuArch=ppc64el
@ -131,6 +152,18 @@ while :; do
__UbuntuPackages=$(echo ${__UbuntuPackages} | sed 's/ libomp5//') __UbuntuPackages=$(echo ${__UbuntuPackages} | sed 's/ libomp5//')
unset __LLDB_Package unset __LLDB_Package
;; ;;
riscv64)
__BuildArch=riscv64
__UbuntuArch=riscv64
__UbuntuRepo="http://deb.debian.org/debian-ports"
__CodeName=sid
__UbuntuPackages=$(echo ${__UbuntuPackages} | sed 's/ libunwind8-dev//')
unset __LLDB_Package
if [[ -e "/usr/share/keyrings/debian-ports-archive-keyring.gpg" ]]; then
__Keyring="--keyring /usr/share/keyrings/debian-ports-archive-keyring.gpg --include=debian-ports-archive-keyring"
fi
;;
s390x) s390x)
__BuildArch=s390x __BuildArch=s390x
__UbuntuArch=s390x __UbuntuArch=s390x
@ -140,56 +173,69 @@ while :; do
__UbuntuPackages=$(echo ${__UbuntuPackages} | sed 's/ libomp5//') __UbuntuPackages=$(echo ${__UbuntuPackages} | sed 's/ libomp5//')
unset __LLDB_Package unset __LLDB_Package
;; ;;
x64)
__BuildArch=x64
__UbuntuArch=amd64
__FreeBSDArch=amd64
__FreeBSDMachineArch=amd64
__illumosArch=x86_64
__UbuntuRepo=
;;
x86) x86)
__BuildArch=x86 __BuildArch=x86
__UbuntuArch=i386 __UbuntuArch=i386
__UbuntuRepo="http://archive.ubuntu.com/ubuntu/" __UbuntuRepo="http://archive.ubuntu.com/ubuntu/"
;; ;;
lldb3.6) lldb*)
__LLDB_Package="lldb-3.6-dev" version="${lowerI/lldb/}"
;; parts=(${version//./ })
lldb3.8)
__LLDB_Package="lldb-3.8-dev" # for versions > 6.0, lldb has dropped the minor version
;; if [[ "${parts[0]}" -gt 6 ]]; then
lldb3.9) version="${parts[0]}"
__LLDB_Package="liblldb-3.9-dev" fi
;;
lldb4.0) __LLDB_Package="liblldb-${version}-dev"
__LLDB_Package="liblldb-4.0-dev"
;;
lldb5.0)
__LLDB_Package="liblldb-5.0-dev"
;;
lldb6.0)
__LLDB_Package="liblldb-6.0-dev"
;; ;;
no-lldb) no-lldb)
unset __LLDB_Package unset __LLDB_Package
;; ;;
llvm*) llvm*)
version="$(echo "$lowerI" | tr -d '[:alpha:]-=')" version="${lowerI/llvm/}"
parts=(${version//./ }) parts=(${version//./ })
__LLVM_MajorVersion="${parts[0]}" __LLVM_MajorVersion="${parts[0]}"
__LLVM_MinorVersion="${parts[1]}" __LLVM_MinorVersion="${parts[1]}"
# for versions > 6.0, llvm has dropped the minor version
if [[ -z "$__LLVM_MinorVersion" && "$__LLVM_MajorVersion" -le 6 ]]; then if [[ -z "$__LLVM_MinorVersion" && "$__LLVM_MajorVersion" -le 6 ]]; then
__LLVM_MinorVersion=0; __LLVM_MinorVersion=0;
fi fi
;; ;;
xenial) # Ubuntu 16.04 xenial) # Ubuntu 16.04
if [ "$__CodeName" != "jessie" ]; then if [[ "$__CodeName" != "jessie" ]]; then
__CodeName=xenial __CodeName=xenial
fi fi
;; ;;
zesty) # Ubuntu 17.04 zesty) # Ubuntu 17.04
if [ "$__CodeName" != "jessie" ]; then if [[ "$__CodeName" != "jessie" ]]; then
__CodeName=zesty __CodeName=zesty
fi fi
;; ;;
bionic) # Ubuntu 18.04 bionic) # Ubuntu 18.04
if [ "$__CodeName" != "jessie" ]; then if [[ "$__CodeName" != "jessie" ]]; then
__CodeName=bionic __CodeName=bionic
fi fi
;; ;;
focal) # Ubuntu 20.04
if [[ "$__CodeName" != "jessie" ]]; then
__CodeName=focal
fi
;;
jammy) # Ubuntu 22.04
if [[ "$__CodeName" != "jessie" ]]; then
__CodeName=jammy
fi
;;
jessie) # Debian 8 jessie) # Debian 8
__CodeName=jessie __CodeName=jessie
__UbuntuRepo="http://ftp.debian.org/debian/" __UbuntuRepo="http://ftp.debian.org/debian/"
@ -205,11 +251,6 @@ while :; do
__LLDB_Package="liblldb-6.0-dev" __LLDB_Package="liblldb-6.0-dev"
;; ;;
tizen) tizen)
if [ "$__BuildArch" != "arm" ] && [ "$__BuildArch" != "armel" ] && [ "$__BuildArch" != "arm64" ] && [ "$__BuildArch" != "x86" ] ; then
echo "Tizen is available only for arm, armel, arm64 and x86."
usage;
exit 1;
fi
__CodeName= __CodeName=
__UbuntuRepo= __UbuntuRepo=
__Tizen=tizen __Tizen=tizen
@ -228,18 +269,20 @@ while :; do
;; ;;
freebsd12) freebsd12)
__CodeName=freebsd __CodeName=freebsd
__BuildArch=x64
__SkipUnmount=1 __SkipUnmount=1
;; ;;
freebsd13) freebsd13)
__CodeName=freebsd __CodeName=freebsd
__FreeBSDBase="13.0-RELEASE" __FreeBSDBase="13.0-RELEASE"
__FreeBSDABI="13" __FreeBSDABI="13"
__BuildArch=x64
__SkipUnmount=1 __SkipUnmount=1
;; ;;
illumos) illumos)
__CodeName=illumos __CodeName=illumos
__SkipUnmount=1
;;
haiku)
__CodeName=haiku
__BuildArch=x64 __BuildArch=x64
__SkipUnmount=1 __SkipUnmount=1
;; ;;
@ -248,11 +291,15 @@ while :; do
;; ;;
--rootfsdir|-rootfsdir) --rootfsdir|-rootfsdir)
shift shift
__RootfsDir=$1 __RootfsDir="$1"
;; ;;
--use-mirror) --use-mirror)
__UseMirror=1 __UseMirror=1
;; ;;
--use-jobs)
shift
MAXJOBS=$1
;;
*) *)
__UnprocessedBuildArgs="$__UnprocessedBuildArgs $1" __UnprocessedBuildArgs="$__UnprocessedBuildArgs $1"
;; ;;
@ -261,81 +308,76 @@ while :; do
shift shift
done done
if [ -e "$__Keyring" ]; then if [[ "$__BuildArch" == "armel" ]]; then
__Keyring="--keyring=$__Keyring"
else
__Keyring=""
fi
if [ "$__BuildArch" == "armel" ]; then
__LLDB_Package="lldb-3.5-dev" __LLDB_Package="lldb-3.5-dev"
fi fi
__UbuntuPackages+=" ${__LLDB_Package:-}" __UbuntuPackages+=" ${__LLDB_Package:-}"
if [ ! -z "$__LLVM_MajorVersion" ]; then if [[ -n "$__LLVM_MajorVersion" ]]; then
__UbuntuPackages+=" libclang-common-${__LLVM_MajorVersion}${__LLVM_MinorVersion:+.$__LLVM_MinorVersion}-dev" __UbuntuPackages+=" libclang-common-${__LLVM_MajorVersion}${__LLVM_MinorVersion:+.$__LLVM_MinorVersion}-dev"
fi fi
if [ -z "$__RootfsDir" ] && [ ! -z "$ROOTFS_DIR" ]; then if [[ -z "$__RootfsDir" && -n "$ROOTFS_DIR" ]]; then
__RootfsDir=$ROOTFS_DIR __RootfsDir="$ROOTFS_DIR"
fi fi
if [ -z "$__RootfsDir" ]; then if [[ -z "$__RootfsDir" ]]; then
__RootfsDir="$__CrossDir/../../../.tools/rootfs/$__BuildArch" __RootfsDir="$__CrossDir/../../../.tools/rootfs/$__BuildArch"
fi fi
if [ -d "$__RootfsDir" ]; then if [[ -d "$__RootfsDir" ]]; then
if [ $__SkipUnmount == 0 ]; then if [[ "$__SkipUnmount" == "0" ]]; then
umount $__RootfsDir/* || true umount "$__RootfsDir"/* || true
fi fi
rm -rf $__RootfsDir rm -rf "$__RootfsDir"
fi fi
mkdir -p $__RootfsDir mkdir -p "$__RootfsDir"
__RootfsDir="$( cd "$__RootfsDir" && pwd )" __RootfsDir="$( cd "$__RootfsDir" && pwd )"
if [[ "$__CodeName" == "alpine" ]]; then if [[ "$__CodeName" == "alpine" ]]; then
__ApkToolsVersion=2.9.1 __ApkToolsVersion=2.9.1
__ApkToolsDir=$(mktemp -d) __ApkToolsDir="$(mktemp -d)"
wget https://github.com/alpinelinux/apk-tools/releases/download/v$__ApkToolsVersion/apk-tools-$__ApkToolsVersion-x86_64-linux.tar.gz -P $__ApkToolsDir wget "https://github.com/alpinelinux/apk-tools/releases/download/v$__ApkToolsVersion/apk-tools-$__ApkToolsVersion-x86_64-linux.tar.gz" -P "$__ApkToolsDir"
tar -xf $__ApkToolsDir/apk-tools-$__ApkToolsVersion-x86_64-linux.tar.gz -C $__ApkToolsDir tar -xf "$__ApkToolsDir/apk-tools-$__ApkToolsVersion-x86_64-linux.tar.gz" -C "$__ApkToolsDir"
mkdir -p $__RootfsDir/usr/bin mkdir -p "$__RootfsDir"/usr/bin
cp -v /usr/bin/qemu-$__QEMUArch-static $__RootfsDir/usr/bin cp -v "/usr/bin/qemu-$__QEMUArch-static" "$__RootfsDir/usr/bin"
$__ApkToolsDir/apk-tools-$__ApkToolsVersion/apk \ "$__ApkToolsDir/apk-tools-$__ApkToolsVersion/apk" \
-X http://dl-cdn.alpinelinux.org/alpine/v$__AlpineVersion/main \ -X "http://dl-cdn.alpinelinux.org/alpine/v$__AlpineVersion/main" \
-X http://dl-cdn.alpinelinux.org/alpine/v$__AlpineVersion/community \ -X "http://dl-cdn.alpinelinux.org/alpine/v$__AlpineVersion/community" \
-U --allow-untrusted --root $__RootfsDir --arch $__AlpineArch --initdb \ -U --allow-untrusted --root "$__RootfsDir" --arch "$__AlpineArch" --initdb \
add $__AlpinePackages add $__AlpinePackages
rm -r $__ApkToolsDir rm -r "$__ApkToolsDir"
elif [[ "$__CodeName" == "freebsd" ]]; then elif [[ "$__CodeName" == "freebsd" ]]; then
mkdir -p $__RootfsDir/usr/local/etc mkdir -p "$__RootfsDir"/usr/local/etc
JOBS="$(getconf _NPROCESSORS_ONLN)" JOBS=${MAXJOBS:="$(getconf _NPROCESSORS_ONLN)"}
wget -O - https://download.freebsd.org/ftp/releases/amd64/${__FreeBSDBase}/base.txz | tar -C $__RootfsDir -Jxf - ./lib ./usr/lib ./usr/libdata ./usr/include ./usr/share/keys ./etc ./bin/freebsd-version wget -O - "https://download.freebsd.org/ftp/releases/${__FreeBSDArch}/${__FreeBSDMachineArch}/${__FreeBSDBase}/base.txz" | tar -C "$__RootfsDir" -Jxf - ./lib ./usr/lib ./usr/libdata ./usr/include ./usr/share/keys ./etc ./bin/freebsd-version
echo "ABI = \"FreeBSD:${__FreeBSDABI}:amd64\"; FINGERPRINTS = \"${__RootfsDir}/usr/share/keys\"; REPOS_DIR = [\"${__RootfsDir}/etc/pkg\"]; REPO_AUTOUPDATE = NO; RUN_SCRIPTS = NO;" > ${__RootfsDir}/usr/local/etc/pkg.conf echo "ABI = \"FreeBSD:${__FreeBSDABI}:${__FreeBSDMachineArch}\"; FINGERPRINTS = \"${__RootfsDir}/usr/share/keys\"; REPOS_DIR = [\"${__RootfsDir}/etc/pkg\"]; REPO_AUTOUPDATE = NO; RUN_SCRIPTS = NO;" > "${__RootfsDir}"/usr/local/etc/pkg.conf
echo "FreeBSD: { url: "pkg+http://pkg.FreeBSD.org/\${ABI}/quarterly", mirror_type: \"srv\", signature_type: \"fingerprints\", fingerprints: \"${__RootfsDir}/usr/share/keys/pkg\", enabled: yes }" > ${__RootfsDir}/etc/pkg/FreeBSD.conf echo "FreeBSD: { url: \"pkg+http://pkg.FreeBSD.org/\${ABI}/quarterly\", mirror_type: \"srv\", signature_type: \"fingerprints\", fingerprints: \"${__RootfsDir}/usr/share/keys/pkg\", enabled: yes }" > "${__RootfsDir}"/etc/pkg/FreeBSD.conf
mkdir -p $__RootfsDir/tmp mkdir -p "$__RootfsDir"/tmp
# get and build package manager # get and build package manager
wget -O - https://github.com/freebsd/pkg/archive/${__FreeBSDPkg}.tar.gz | tar -C $__RootfsDir/tmp -zxf - wget -O - "https://github.com/freebsd/pkg/archive/${__FreeBSDPkg}.tar.gz" | tar -C "$__RootfsDir"/tmp -zxf -
cd $__RootfsDir/tmp/pkg-${__FreeBSDPkg} cd "$__RootfsDir/tmp/pkg-${__FreeBSDPkg}"
# needed for install to succeed # needed for install to succeed
mkdir -p $__RootfsDir/host/etc mkdir -p "$__RootfsDir"/host/etc
./autogen.sh && ./configure --prefix=$__RootfsDir/host && make -j "$JOBS" && make install ./autogen.sh && ./configure --prefix="$__RootfsDir"/host && make -j "$JOBS" && make install
rm -rf $__RootfsDir/tmp/pkg-${__FreeBSDPkg} rm -rf "$__RootfsDir/tmp/pkg-${__FreeBSDPkg}"
# install packages we need. # install packages we need.
INSTALL_AS_USER=$(whoami) $__RootfsDir/host/sbin/pkg -r $__RootfsDir -C $__RootfsDir/usr/local/etc/pkg.conf update INSTALL_AS_USER=$(whoami) "$__RootfsDir"/host/sbin/pkg -r "$__RootfsDir" -C "$__RootfsDir"/usr/local/etc/pkg.conf update
INSTALL_AS_USER=$(whoami) $__RootfsDir/host/sbin/pkg -r $__RootfsDir -C $__RootfsDir/usr/local/etc/pkg.conf install --yes $__FreeBSDPackages INSTALL_AS_USER=$(whoami) "$__RootfsDir"/host/sbin/pkg -r "$__RootfsDir" -C "$__RootfsDir"/usr/local/etc/pkg.conf install --yes $__FreeBSDPackages
elif [[ "$__CodeName" == "illumos" ]]; then elif [[ "$__CodeName" == "illumos" ]]; then
mkdir "$__RootfsDir/tmp" mkdir "$__RootfsDir/tmp"
pushd "$__RootfsDir/tmp" pushd "$__RootfsDir/tmp"
JOBS="$(getconf _NPROCESSORS_ONLN)" JOBS=${MAXJOBS:="$(getconf _NPROCESSORS_ONLN)"}
echo "Downloading sysroot." echo "Downloading sysroot."
wget -O - https://github.com/illumos/sysroot/releases/download/20181213-de6af22ae73b-v1/illumos-sysroot-i386-20181213-de6af22ae73b-v1.tar.gz | tar -C "$__RootfsDir" -xzf - wget -O - https://github.com/illumos/sysroot/releases/download/20181213-de6af22ae73b-v1/illumos-sysroot-i386-20181213-de6af22ae73b-v1.tar.gz | tar -C "$__RootfsDir" -xzf -
echo "Building binutils. Please wait.." echo "Building binutils. Please wait.."
wget -O - https://ftp.gnu.org/gnu/binutils/binutils-2.33.1.tar.bz2 | tar -xjf - wget -O - https://ftp.gnu.org/gnu/binutils/binutils-2.33.1.tar.bz2 | tar -xjf -
mkdir build-binutils && cd build-binutils mkdir build-binutils && cd build-binutils
../binutils-2.33.1/configure --prefix="$__RootfsDir" --target="x86_64-sun-solaris2.10" --program-prefix="x86_64-illumos-" --with-sysroot="$__RootfsDir" ../binutils-2.33.1/configure --prefix="$__RootfsDir" --target="${__illumosArch}-sun-solaris2.10" --program-prefix="${__illumosArch}-illumos-" --with-sysroot="$__RootfsDir"
make -j "$JOBS" && make install && cd .. make -j "$JOBS" && make install && cd ..
echo "Building gcc. Please wait.." echo "Building gcc. Please wait.."
wget -O - https://ftp.gnu.org/gnu/gcc/gcc-8.4.0/gcc-8.4.0.tar.xz | tar -xJf - wget -O - https://ftp.gnu.org/gnu/gcc/gcc-8.4.0/gcc-8.4.0.tar.xz | tar -xJf -
@ -345,7 +387,7 @@ elif [[ "$__CodeName" == "illumos" ]]; then
CFLAGS_FOR_TARGET="-fPIC" CFLAGS_FOR_TARGET="-fPIC"
export CFLAGS CXXFLAGS CXXFLAGS_FOR_TARGET CFLAGS_FOR_TARGET export CFLAGS CXXFLAGS CXXFLAGS_FOR_TARGET CFLAGS_FOR_TARGET
mkdir build-gcc && cd build-gcc mkdir build-gcc && cd build-gcc
../gcc-8.4.0/configure --prefix="$__RootfsDir" --target="x86_64-sun-solaris2.10" --program-prefix="x86_64-illumos-" --with-sysroot="$__RootfsDir" --with-gnu-as \ ../gcc-8.4.0/configure --prefix="$__RootfsDir" --target="${__illumosArch}-sun-solaris2.10" --program-prefix="${__illumosArch}-illumos-" --with-sysroot="$__RootfsDir" --with-gnu-as \
--with-gnu-ld --disable-nls --disable-libgomp --disable-libquadmath --disable-libssp --disable-libvtv --disable-libcilkrts --disable-libada --disable-libsanitizer \ --with-gnu-ld --disable-nls --disable-libgomp --disable-libquadmath --disable-libssp --disable-libvtv --disable-libcilkrts --disable-libada --disable-libsanitizer \
--disable-libquadmath-support --disable-shared --enable-tls --disable-libquadmath-support --disable-shared --enable-tls
make -j "$JOBS" && make install && cd .. make -j "$JOBS" && make install && cd ..
@ -353,14 +395,18 @@ elif [[ "$__CodeName" == "illumos" ]]; then
if [[ "$__UseMirror" == 1 ]]; then if [[ "$__UseMirror" == 1 ]]; then
BaseUrl=http://pkgsrc.smartos.skylime.net BaseUrl=http://pkgsrc.smartos.skylime.net
fi fi
BaseUrl="$BaseUrl"/packages/SmartOS/2020Q1/x86_64/All BaseUrl="$BaseUrl/packages/SmartOS/trunk/${__illumosArch}/All"
echo "Downloading manifest"
wget "$BaseUrl"
echo "Downloading dependencies." echo "Downloading dependencies."
read -ra array <<<"$__IllumosPackages" read -ra array <<<"$__IllumosPackages"
for package in "${array[@]}"; do for package in "${array[@]}"; do
echo "Installing $package..." echo "Installing '$package'"
package="$(grep ">$package-[0-9]" All | sed -En 's/.*href="(.*)\.tgz".*/\1/p')"
echo "Resolved name '$package'"
wget "$BaseUrl"/"$package".tgz wget "$BaseUrl"/"$package".tgz
ar -x "$package".tgz ar -x "$package".tgz
tar --skip-old-files -xzf "$package".tmp.tgz -C "$__RootfsDir" 2>/dev/null tar --skip-old-files -xzf "$package".tmp.tg* -C "$__RootfsDir" 2>/dev/null
done done
echo "Cleaning up temporary files." echo "Cleaning up temporary files."
popd popd
@ -371,26 +417,90 @@ elif [[ "$__CodeName" == "illumos" ]]; then
wget -P "$__RootfsDir"/usr/include/net https://raw.githubusercontent.com/illumos/illumos-gate/master/usr/src/uts/common/io/bpf/net/dlt.h wget -P "$__RootfsDir"/usr/include/net https://raw.githubusercontent.com/illumos/illumos-gate/master/usr/src/uts/common/io/bpf/net/dlt.h
wget -P "$__RootfsDir"/usr/include/netpacket https://raw.githubusercontent.com/illumos/illumos-gate/master/usr/src/uts/common/inet/sockmods/netpacket/packet.h wget -P "$__RootfsDir"/usr/include/netpacket https://raw.githubusercontent.com/illumos/illumos-gate/master/usr/src/uts/common/inet/sockmods/netpacket/packet.h
wget -P "$__RootfsDir"/usr/include/sys https://raw.githubusercontent.com/illumos/illumos-gate/master/usr/src/uts/common/sys/sdt.h wget -P "$__RootfsDir"/usr/include/sys https://raw.githubusercontent.com/illumos/illumos-gate/master/usr/src/uts/common/sys/sdt.h
elif [[ -n $__CodeName ]]; then elif [[ "$__CodeName" == "haiku" ]]; then
qemu-debootstrap $__Keyring --arch $__UbuntuArch $__CodeName $__RootfsDir $__UbuntuRepo JOBS=${MAXJOBS:="$(getconf _NPROCESSORS_ONLN)"}
cp $__CrossDir/$__BuildArch/sources.list.$__CodeName $__RootfsDir/etc/apt/sources.list
chroot $__RootfsDir apt-get update echo "Building Haiku sysroot for x86_64"
chroot $__RootfsDir apt-get -f -y install mkdir -p "$__RootfsDir/tmp"
chroot $__RootfsDir apt-get -y install $__UbuntuPackages cd "$__RootfsDir/tmp"
chroot $__RootfsDir symlinks -cr /usr git clone -b hrev56235 https://review.haiku-os.org/haiku
chroot $__RootfsDir apt-get clean git clone -b btrev43195 https://review.haiku-os.org/buildtools
cd "$__RootfsDir/tmp/buildtools" && git checkout 7487388f5110021d400b9f3b88e1a7f310dc066d
if [ $__SkipUnmount == 0 ]; then
umount $__RootfsDir/* || true # Fetch some unmerged patches
cd "$__RootfsDir/tmp/haiku"
## Add development build profile (slimmer than nightly)
git fetch origin refs/changes/64/4164/1 && git -c commit.gpgsign=false cherry-pick FETCH_HEAD
# Build jam
cd "$__RootfsDir/tmp/buildtools/jam"
make
# Configure cross tools
echo "Building cross-compiler"
mkdir -p "$__RootfsDir/generated"
cd "$__RootfsDir/generated"
"$__RootfsDir/tmp/haiku/configure" -j"$JOBS" --sysroot "$__RootfsDir" --cross-tools-source "$__RootfsDir/tmp/buildtools" --build-cross-tools x86_64
# Build Haiku packages
echo "Building Haiku"
echo 'HAIKU_BUILD_PROFILE = "development-raw" ;' > UserProfileConfig
"$__RootfsDir/tmp/buildtools/jam/jam0" -j"$JOBS" -q '<build>package' '<repository>Haiku'
BaseUrl="https://depot.haiku-os.org/__api/v2/pkg/get-pkg"
# Download additional packages
echo "Downloading additional required packages"
read -ra array <<<"$__HaikuPackages"
for package in "${array[@]}"; do
echo "Downloading $package..."
# API documented here: https://github.com/haiku/haikudepotserver/blob/master/haikudepotserver-api2/src/main/resources/api2/pkg.yaml#L60
# The schema here: https://github.com/haiku/haikudepotserver/blob/master/haikudepotserver-api2/src/main/resources/api2/pkg.yaml#L598
hpkgDownloadUrl="$(wget -qO- --post-data='{"name":"'"$package"'","repositorySourceCode":"haikuports_x86_64","versionType":"LATEST","naturalLanguageCode":"en"}' \
--header='Content-Type:application/json' "$BaseUrl" | jq -r '.result.versions[].hpkgDownloadURL')"
wget -P "$__RootfsDir/generated/download" "$hpkgDownloadUrl"
done
# Setup the sysroot
echo "Setting up sysroot and extracting needed packages"
mkdir -p "$__RootfsDir/boot/system"
for file in "$__RootfsDir/generated/objects/haiku/x86_64/packaging/packages/"*.hpkg; do
"$__RootfsDir/generated/objects/linux/x86_64/release/tools/package/package" extract -C "$__RootfsDir/boot/system" "$file"
done
for file in "$__RootfsDir/generated/download/"*.hpkg; do
"$__RootfsDir/generated/objects/linux/x86_64/release/tools/package/package" extract -C "$__RootfsDir/boot/system" "$file"
done
# Cleaning up temporary files
echo "Cleaning up temporary files"
rm -rf "$__RootfsDir/tmp"
for name in "$__RootfsDir/generated/"*; do
if [[ "$name" =~ "cross-tools-" ]]; then
: # Keep the cross-compiler
else
rm -rf "$name"
fi
done
elif [[ -n "$__CodeName" ]]; then
qemu-debootstrap $__Keyring --arch "$__UbuntuArch" "$__CodeName" "$__RootfsDir" "$__UbuntuRepo"
cp "$__CrossDir/$__BuildArch/sources.list.$__CodeName" "$__RootfsDir/etc/apt/sources.list"
chroot "$__RootfsDir" apt-get update
chroot "$__RootfsDir" apt-get -f -y install
chroot "$__RootfsDir" apt-get -y install $__UbuntuPackages
chroot "$__RootfsDir" symlinks -cr /usr
chroot "$__RootfsDir" apt-get clean
if [[ "$__SkipUnmount" == "0" ]]; then
umount "$__RootfsDir"/* || true
fi fi
if [[ "$__BuildArch" == "armel" && "$__CodeName" == "jessie" ]]; then if [[ "$__BuildArch" == "armel" && "$__CodeName" == "jessie" ]]; then
pushd $__RootfsDir pushd "$__RootfsDir"
patch -p1 < $__CrossDir/$__BuildArch/armel.jessie.patch patch -p1 < "$__CrossDir/$__BuildArch/armel.jessie.patch"
popd popd
fi fi
elif [[ "$__Tizen" == "tizen" ]]; then elif [[ "$__Tizen" == "tizen" ]]; then
ROOTFS_DIR=$__RootfsDir $__CrossDir/$__BuildArch/tizen-build-rootfs.sh ROOTFS_DIR="$__RootfsDir" "$__CrossDir/$__BuildArch/tizen-build-rootfs.sh"
else else
echo "Unsupported target platform." echo "Unsupported target platform."
usage; usage;

1
eng/common/cross/riscv64/sources.list.sid

@ -0,0 +1 @@
deb http://deb.debian.org/debian-ports sid main

98
eng/common/cross/toolchain.cmake

@ -7,6 +7,8 @@ if(EXISTS ${CROSS_ROOTFS}/bin/freebsd-version)
elseif(EXISTS ${CROSS_ROOTFS}/usr/platform/i86pc) elseif(EXISTS ${CROSS_ROOTFS}/usr/platform/i86pc)
set(CMAKE_SYSTEM_NAME SunOS) set(CMAKE_SYSTEM_NAME SunOS)
set(ILLUMOS 1) set(ILLUMOS 1)
elseif(EXISTS ${CROSS_ROOTFS}/boot/system/develop/headers/config/HaikuConfig.h)
set(CMAKE_SYSTEM_NAME Haiku)
else() else()
set(CMAKE_SYSTEM_NAME Linux) set(CMAKE_SYSTEM_NAME Linux)
set(LINUX 1) set(LINUX 1)
@ -19,13 +21,7 @@ elseif(EXISTS ${CROSS_ROOTFS}/android_platform)
set(ANDROID 1) set(ANDROID 1)
endif() endif()
if(TARGET_ARCH_NAME STREQUAL "armel") if(TARGET_ARCH_NAME STREQUAL "arm")
set(CMAKE_SYSTEM_PROCESSOR armv7l)
set(TOOLCHAIN "arm-linux-gnueabi")
if(TIZEN)
set(TIZEN_TOOLCHAIN "armv7l-tizen-linux-gnueabi/9.2.0")
endif()
elseif(TARGET_ARCH_NAME STREQUAL "arm")
set(CMAKE_SYSTEM_PROCESSOR armv7l) set(CMAKE_SYSTEM_PROCESSOR armv7l)
if(EXISTS ${CROSS_ROOTFS}/usr/lib/gcc/armv7-alpine-linux-musleabihf) if(EXISTS ${CROSS_ROOTFS}/usr/lib/gcc/armv7-alpine-linux-musleabihf)
set(TOOLCHAIN "armv7-alpine-linux-musleabihf") set(TOOLCHAIN "armv7-alpine-linux-musleabihf")
@ -37,43 +33,62 @@ elseif(TARGET_ARCH_NAME STREQUAL "arm")
if(TIZEN) if(TIZEN)
set(TIZEN_TOOLCHAIN "armv7hl-tizen-linux-gnueabihf/9.2.0") set(TIZEN_TOOLCHAIN "armv7hl-tizen-linux-gnueabihf/9.2.0")
endif() endif()
elseif(TARGET_ARCH_NAME STREQUAL "armv6")
set(CMAKE_SYSTEM_PROCESSOR armv6l)
if(EXISTS ${CROSS_ROOTFS}/usr/lib/gcc/armv6-alpine-linux-musleabihf)
set(TOOLCHAIN "armv6-alpine-linux-musleabihf")
else()
set(TOOLCHAIN "arm-linux-gnueabihf")
endif()
elseif(TARGET_ARCH_NAME STREQUAL "arm64") elseif(TARGET_ARCH_NAME STREQUAL "arm64")
set(CMAKE_SYSTEM_PROCESSOR aarch64) set(CMAKE_SYSTEM_PROCESSOR aarch64)
if(EXISTS ${CROSS_ROOTFS}/usr/lib/gcc/aarch64-alpine-linux-musl) if(EXISTS ${CROSS_ROOTFS}/usr/lib/gcc/aarch64-alpine-linux-musl)
set(TOOLCHAIN "aarch64-alpine-linux-musl") set(TOOLCHAIN "aarch64-alpine-linux-musl")
else() elseif(LINUX)
set(TOOLCHAIN "aarch64-linux-gnu") set(TOOLCHAIN "aarch64-linux-gnu")
if(TIZEN)
set(TIZEN_TOOLCHAIN "aarch64-tizen-linux-gnu/9.2.0")
endif()
elseif(FREEBSD)
set(triple "aarch64-unknown-freebsd12")
endif() endif()
elseif(TARGET_ARCH_NAME STREQUAL "armel")
set(CMAKE_SYSTEM_PROCESSOR armv7l)
set(TOOLCHAIN "arm-linux-gnueabi")
if(TIZEN) if(TIZEN)
set(TIZEN_TOOLCHAIN "aarch64-tizen-linux-gnu/9.2.0") set(TIZEN_TOOLCHAIN "armv7l-tizen-linux-gnueabi/9.2.0")
endif()
elseif(TARGET_ARCH_NAME STREQUAL "armv6")
set(CMAKE_SYSTEM_PROCESSOR armv6l)
if(EXISTS ${CROSS_ROOTFS}/usr/lib/gcc/armv6-alpine-linux-musleabihf)
set(TOOLCHAIN "armv6-alpine-linux-musleabihf")
else()
set(TOOLCHAIN "arm-linux-gnueabihf")
endif() endif()
elseif(TARGET_ARCH_NAME STREQUAL "ppc64le") elseif(TARGET_ARCH_NAME STREQUAL "ppc64le")
set(CMAKE_SYSTEM_PROCESSOR ppc64le) set(CMAKE_SYSTEM_PROCESSOR ppc64le)
set(TOOLCHAIN "powerpc64le-linux-gnu") set(TOOLCHAIN "powerpc64le-linux-gnu")
elseif(TARGET_ARCH_NAME STREQUAL "riscv64")
set(CMAKE_SYSTEM_PROCESSOR riscv64)
set(TOOLCHAIN "riscv64-linux-gnu")
elseif(TARGET_ARCH_NAME STREQUAL "s390x") elseif(TARGET_ARCH_NAME STREQUAL "s390x")
set(CMAKE_SYSTEM_PROCESSOR s390x) set(CMAKE_SYSTEM_PROCESSOR s390x)
set(TOOLCHAIN "s390x-linux-gnu") set(TOOLCHAIN "s390x-linux-gnu")
elseif(TARGET_ARCH_NAME STREQUAL "x64")
set(CMAKE_SYSTEM_PROCESSOR x86_64)
if(LINUX)
set(TOOLCHAIN "x86_64-linux-gnu")
if(TIZEN)
set(TIZEN_TOOLCHAIN "x86_64-tizen-linux-gnu/9.2.0")
endif()
elseif(FREEBSD)
set(triple "x86_64-unknown-freebsd12")
elseif(ILLUMOS)
set(TOOLCHAIN "x86_64-illumos")
elseif(HAIKU)
set(TOOLCHAIN "x64_64-unknown-haiku")
endif()
elseif(TARGET_ARCH_NAME STREQUAL "x86") elseif(TARGET_ARCH_NAME STREQUAL "x86")
set(CMAKE_SYSTEM_PROCESSOR i686) set(CMAKE_SYSTEM_PROCESSOR i686)
set(TOOLCHAIN "i686-linux-gnu") set(TOOLCHAIN "i686-linux-gnu")
if(TIZEN) if(TIZEN)
set(TIZEN_TOOLCHAIN "i586-tizen-linux-gnu/9.2.0") set(TIZEN_TOOLCHAIN "i586-tizen-linux-gnu/9.2.0")
endif() endif()
elseif (FREEBSD)
set(CMAKE_SYSTEM_PROCESSOR "x86_64")
set(triple "x86_64-unknown-freebsd12")
elseif (ILLUMOS)
set(CMAKE_SYSTEM_PROCESSOR "x86_64")
set(TOOLCHAIN "x86_64-illumos")
else() else()
message(FATAL_ERROR "Arch is ${TARGET_ARCH_NAME}. Only armel, arm, armv6, arm64, ppc64le, s390x and x86 are supported!") message(FATAL_ERROR "Arch is ${TARGET_ARCH_NAME}. Only arm, arm64, armel, armv6, ppc64le, riscv64, s390x, x64 and x86 are supported!")
endif() endif()
if(DEFINED ENV{TOOLCHAIN}) if(DEFINED ENV{TOOLCHAIN})
@ -159,6 +174,41 @@ elseif(ILLUMOS)
set(CMAKE_C_STANDARD_LIBRARIES "${CMAKE_C_STANDARD_LIBRARIES} -lssp") set(CMAKE_C_STANDARD_LIBRARIES "${CMAKE_C_STANDARD_LIBRARIES} -lssp")
set(CMAKE_CXX_STANDARD_LIBRARIES "${CMAKE_CXX_STANDARD_LIBRARIES} -lssp") set(CMAKE_CXX_STANDARD_LIBRARIES "${CMAKE_CXX_STANDARD_LIBRARIES} -lssp")
elseif(HAIKU)
set(CMAKE_SYSROOT "${CROSS_ROOTFS}")
set(TOOLSET_PREFIX ${TOOLCHAIN}-)
function(locate_toolchain_exec exec var)
string(TOUPPER ${exec} EXEC_UPPERCASE)
if(NOT "$ENV{CLR_${EXEC_UPPERCASE}}" STREQUAL "")
set(${var} "$ENV{CLR_${EXEC_UPPERCASE}}" PARENT_SCOPE)
return()
endif()
set(SEARCH_PATH "${CROSS_ROOTFS}/generated/cross-tools-x86_64/bin")
find_program(EXEC_LOCATION_${exec}
PATHS ${SEARCH_PATH}
NAMES
"${TOOLSET_PREFIX}${exec}${CLR_CMAKE_COMPILER_FILE_NAME_VERSION}"
"${TOOLSET_PREFIX}${exec}")
if (EXEC_LOCATION_${exec} STREQUAL "EXEC_LOCATION_${exec}-NOTFOUND")
message(FATAL_ERROR "Unable to find toolchain executable. Name: ${exec}, Prefix: ${TOOLSET_PREFIX}.")
endif()
set(${var} ${EXEC_LOCATION_${exec}} PARENT_SCOPE)
endfunction()
set(CMAKE_SYSTEM_PREFIX_PATH "${CROSS_ROOTFS}")
locate_toolchain_exec(gcc CMAKE_C_COMPILER)
locate_toolchain_exec(g++ CMAKE_CXX_COMPILER)
set(CMAKE_C_STANDARD_LIBRARIES "${CMAKE_C_STANDARD_LIBRARIES} -lssp")
set(CMAKE_CXX_STANDARD_LIBRARIES "${CMAKE_CXX_STANDARD_LIBRARIES} -lssp")
# let CMake set up the correct search paths
include(Platform/Haiku)
else() else()
set(CMAKE_SYSROOT "${CROSS_ROOTFS}") set(CMAKE_SYSROOT "${CROSS_ROOTFS}")
@ -218,7 +268,7 @@ endif()
# Specify compile options # Specify compile options
if((TARGET_ARCH_NAME MATCHES "^(arm|armv6|armel|arm64|ppc64le|s390x)$" AND NOT ANDROID) OR ILLUMOS) if((TARGET_ARCH_NAME MATCHES "^(arm|arm64|armel|armv6|ppc64le|riscv64|s390x)$" AND NOT ANDROID AND NOT FREEBSD) OR ILLUMOS OR HAIKU)
set(CMAKE_C_COMPILER_TARGET ${TOOLCHAIN}) set(CMAKE_C_COMPILER_TARGET ${TOOLCHAIN})
set(CMAKE_CXX_COMPILER_TARGET ${TOOLCHAIN}) set(CMAKE_CXX_COMPILER_TARGET ${TOOLCHAIN})
set(CMAKE_ASM_COMPILER_TARGET ${TOOLCHAIN}) set(CMAKE_ASM_COMPILER_TARGET ${TOOLCHAIN})

47
eng/common/generate-locproject.ps1

@ -10,9 +10,7 @@ Param(
Set-StrictMode -Version 2.0 Set-StrictMode -Version 2.0
$ErrorActionPreference = "Stop" $ErrorActionPreference = "Stop"
. $PSScriptRoot\tools.ps1 . $PSScriptRoot\pipeline-logging-functions.ps1
Import-Module -Name (Join-Path $PSScriptRoot 'native\CommonLibrary.psm1')
$exclusionsFilePath = "$SourcesDirectory\eng\Localize\LocExclusions.json" $exclusionsFilePath = "$SourcesDirectory\eng\Localize\LocExclusions.json"
$exclusions = @{ Exclusions = @() } $exclusions = @{ Exclusions = @() }
@ -28,13 +26,15 @@ $jsonFiles = @()
$jsonTemplateFiles = Get-ChildItem -Recurse -Path "$SourcesDirectory" | Where-Object { $_.FullName -Match "\.template\.config\\localize\\.+\.en\.json" } # .NET templating pattern $jsonTemplateFiles = Get-ChildItem -Recurse -Path "$SourcesDirectory" | Where-Object { $_.FullName -Match "\.template\.config\\localize\\.+\.en\.json" } # .NET templating pattern
$jsonTemplateFiles | ForEach-Object { $jsonTemplateFiles | ForEach-Object {
$null = $_.Name -Match "(.+)\.[\w-]+\.json" # matches '[filename].[langcode].json $null = $_.Name -Match "(.+)\.[\w-]+\.json" # matches '[filename].[langcode].json
$destinationFile = "$($_.Directory.FullName)\$($Matches.1).json" $destinationFile = "$($_.Directory.FullName)\$($Matches.1).json"
$jsonFiles += Copy-Item "$($_.FullName)" -Destination $destinationFile -PassThru $jsonFiles += Copy-Item "$($_.FullName)" -Destination $destinationFile -PassThru
} }
$jsonWinformsTemplateFiles = Get-ChildItem -Recurse -Path "$SourcesDirectory" | Where-Object { $_.FullName -Match "en\\strings\.json" } # current winforms pattern $jsonWinformsTemplateFiles = Get-ChildItem -Recurse -Path "$SourcesDirectory" | Where-Object { $_.FullName -Match "en\\strings\.json" } # current winforms pattern
$wxlFiles = Get-ChildItem -Recurse -Path "$SourcesDirectory" | Where-Object { $_.FullName -Match "\\.+\.wxl" -And -Not( $_.Directory.Name -Match "\d{4}" ) } # localized files live in four digit lang ID directories; this excludes them
$xlfFiles = @() $xlfFiles = @()
$allXlfFiles = Get-ChildItem -Recurse -Path "$SourcesDirectory\*\*.xlf" $allXlfFiles = Get-ChildItem -Recurse -Path "$SourcesDirectory\*\*.xlf"
@ -46,7 +46,7 @@ if ($allXlfFiles) {
} }
$langXlfFiles | ForEach-Object { $langXlfFiles | ForEach-Object {
$null = $_.Name -Match "(.+)\.[\w-]+\.xlf" # matches '[filename].[langcode].xlf $null = $_.Name -Match "(.+)\.[\w-]+\.xlf" # matches '[filename].[langcode].xlf
$destinationFile = "$($_.Directory.FullName)\$($Matches.1).xlf" $destinationFile = "$($_.Directory.FullName)\$($Matches.1).xlf"
$xlfFiles += Copy-Item "$($_.FullName)" -Destination $destinationFile -PassThru $xlfFiles += Copy-Item "$($_.FullName)" -Destination $destinationFile -PassThru
} }
@ -59,10 +59,10 @@ $locJson = @{
LanguageSet = $LanguageSet LanguageSet = $LanguageSet
LocItems = @( LocItems = @(
$locFiles | ForEach-Object { $locFiles | ForEach-Object {
$outputPath = "$(($_.DirectoryName | Resolve-Path -Relative) + "\")" $outputPath = "$(($_.DirectoryName | Resolve-Path -Relative) + "\")"
$continue = $true $continue = $true
foreach ($exclusion in $exclusions.Exclusions) { foreach ($exclusion in $exclusions.Exclusions) {
if ($outputPath.Contains($exclusion)) if ($_.FullName.Contains($exclusion))
{ {
$continue = $false $continue = $false
} }
@ -79,8 +79,7 @@ $locJson = @{
CopyOption = "LangIDOnPath" CopyOption = "LangIDOnPath"
OutputPath = "$($_.Directory.Parent.FullName | Resolve-Path -Relative)\" OutputPath = "$($_.Directory.Parent.FullName | Resolve-Path -Relative)\"
} }
} } else {
else {
return @{ return @{
SourceFile = $sourceFile SourceFile = $sourceFile
CopyOption = "LangIDOnName" CopyOption = "LangIDOnName"
@ -90,6 +89,32 @@ $locJson = @{
} }
} }
) )
},
@{
CloneLanguageSet = "WiX_CloneLanguages"
LssFiles = @( "wxl_loc.lss" )
LocItems = @(
$wxlFiles | ForEach-Object {
$outputPath = "$($_.Directory.FullName | Resolve-Path -Relative)\"
$continue = $true
foreach ($exclusion in $exclusions.Exclusions) {
if ($_.FullName.Contains($exclusion))
{
$continue = $false
}
}
$sourceFile = ($_.FullName | Resolve-Path -Relative)
if ($continue)
{
return @{
SourceFile = $sourceFile
CopyOption = "LangIDOnPath"
OutputPath = $outputPath
Languages = "cs-CZ;de-DE;es-ES;fr-FR;it-IT;ja-JP;ko-KR;pl-PL;pt-BR;ru-RU;tr-TR;zh-CN;zh-TW"
}
}
}
)
} }
) )
} }
@ -108,10 +133,10 @@ else {
if ((Get-FileHash "$SourcesDirectory\eng\Localize\LocProject-generated.json").Hash -ne (Get-FileHash "$SourcesDirectory\eng\Localize\LocProject.json").Hash) { if ((Get-FileHash "$SourcesDirectory\eng\Localize\LocProject-generated.json").Hash -ne (Get-FileHash "$SourcesDirectory\eng\Localize\LocProject.json").Hash) {
Write-PipelineTelemetryError -Category "OneLocBuild" -Message "Existing LocProject.json differs from generated LocProject.json. Download LocProject-generated.json and compare them." Write-PipelineTelemetryError -Category "OneLocBuild" -Message "Existing LocProject.json differs from generated LocProject.json. Download LocProject-generated.json and compare them."
exit 1 exit 1
} }
else { else {
Write-Host "Generated LocProject.json and current LocProject.json are identical." Write-Host "Generated LocProject.json and current LocProject.json are identical."
} }
} }

2
eng/common/generate-sbom-prep.ps1

@ -2,6 +2,8 @@ Param(
[Parameter(Mandatory=$true)][string] $ManifestDirPath # Manifest directory where sbom will be placed [Parameter(Mandatory=$true)][string] $ManifestDirPath # Manifest directory where sbom will be placed
) )
. $PSScriptRoot\pipeline-logging-functions.ps1
Write-Host "Creating dir $ManifestDirPath" Write-Host "Creating dir $ManifestDirPath"
# create directory for sbom manifest to be placed # create directory for sbom manifest to be placed
if (!(Test-Path -path $ManifestDirPath)) if (!(Test-Path -path $ManifestDirPath))

12
eng/common/generate-sbom-prep.sh

@ -2,6 +2,18 @@
source="${BASH_SOURCE[0]}" source="${BASH_SOURCE[0]}"
# resolve $SOURCE until the file is no longer a symlink
while [[ -h $source ]]; do
scriptroot="$( cd -P "$( dirname "$source" )" && pwd )"
source="$(readlink "$source")"
# if $source was a relative symlink, we need to resolve it relative to the path where the
# symlink file was located
[[ $source != /* ]] && source="$scriptroot/$source"
done
scriptroot="$( cd -P "$( dirname "$source" )" && pwd )"
. $scriptroot/pipeline-logging-functions.sh
manifest_dir=$1 manifest_dir=$1
if [ ! -d "$manifest_dir" ] ; then if [ ! -d "$manifest_dir" ] ; then

4
eng/common/init-tools-native.ps1

@ -87,6 +87,7 @@ try {
$NativeTools.PSObject.Properties | ForEach-Object { $NativeTools.PSObject.Properties | ForEach-Object {
$ToolName = $_.Name $ToolName = $_.Name
$ToolVersion = $_.Value $ToolVersion = $_.Value
$InstalledTools = @{}
if ((Get-Command "$ToolName" -ErrorAction SilentlyContinue) -eq $null) { if ((Get-Command "$ToolName" -ErrorAction SilentlyContinue) -eq $null) {
if ($ToolVersion -eq "latest") { if ($ToolVersion -eq "latest") {
@ -111,9 +112,10 @@ try {
$ToolPath = Convert-Path -Path $BinPath $ToolPath = Convert-Path -Path $BinPath
Write-Host "Adding $ToolName to the path ($ToolPath)..." Write-Host "Adding $ToolName to the path ($ToolPath)..."
Write-Host "##vso[task.prependpath]$ToolPath" Write-Host "##vso[task.prependpath]$ToolPath"
$InstalledTools += @{ $ToolName = $ToolDirectory.FullName }
} }
} }
exit 0 return $InstalledTools
} else { } else {
$NativeTools.PSObject.Properties | ForEach-Object { $NativeTools.PSObject.Properties | ForEach-Object {
$ToolName = $_.Name $ToolName = $_.Name

2
eng/common/native/init-compiler.sh

@ -71,7 +71,7 @@ if [[ -z "$CLR_CC" ]]; then
# Set default versions # Set default versions
if [[ -z "$majorVersion" ]]; then if [[ -z "$majorVersion" ]]; then
# note: gcc (all versions) and clang versions higher than 6 do not have minor version in file name, if it is zero. # note: gcc (all versions) and clang versions higher than 6 do not have minor version in file name, if it is zero.
if [[ "$compiler" == "clang" ]]; then versions=( 13 12 11 10 9 8 7 6.0 5.0 4.0 3.9 3.8 3.7 3.6 3.5 ) if [[ "$compiler" == "clang" ]]; then versions=( 15 14 13 12 11 10 9 8 7 6.0 5.0 4.0 3.9 3.8 3.7 3.6 3.5 )
elif [[ "$compiler" == "gcc" ]]; then versions=( 12 11 10 9 8 7 6 5 4.9 ); fi elif [[ "$compiler" == "gcc" ]]; then versions=( 12 11 10 9 8 7 6 5 4.9 ); fi
for version in "${versions[@]}"; do for version in "${versions[@]}"; do

2
eng/common/sdk-task.ps1

@ -64,7 +64,7 @@ try {
$GlobalJson.tools | Add-Member -Name "vs" -Value (ConvertFrom-Json "{ `"version`": `"16.5`" }") -MemberType NoteProperty $GlobalJson.tools | Add-Member -Name "vs" -Value (ConvertFrom-Json "{ `"version`": `"16.5`" }") -MemberType NoteProperty
} }
if( -not ($GlobalJson.tools.PSObject.Properties.Name -match "xcopy-msbuild" )) { if( -not ($GlobalJson.tools.PSObject.Properties.Name -match "xcopy-msbuild" )) {
$GlobalJson.tools | Add-Member -Name "xcopy-msbuild" -Value "17.1.0" -MemberType NoteProperty $GlobalJson.tools | Add-Member -Name "xcopy-msbuild" -Value "17.2.1" -MemberType NoteProperty
} }
if ($GlobalJson.tools."xcopy-msbuild".Trim() -ine "none") { if ($GlobalJson.tools."xcopy-msbuild".Trim() -ine "none") {
$xcopyMSBuildToolsFolder = InitializeXCopyMSBuild $GlobalJson.tools."xcopy-msbuild" -install $true $xcopyMSBuildToolsFolder = InitializeXCopyMSBuild $GlobalJson.tools."xcopy-msbuild" -install $true

37
eng/common/sdl/sdl.ps1

@ -0,0 +1,37 @@
function Install-Gdn {
param(
[string]$Path,
# If omitted, install the latest version of Guardian, otherwise install that specific version.
[string]$Version
)
$ErrorActionPreference = 'Stop'
Set-StrictMode -Version 2.0
$disableConfigureToolsetImport = $true
$global:LASTEXITCODE = 0
# `tools.ps1` checks $ci to perform some actions. Since the SDL
# scripts don't necessarily execute in the same agent that run the
# build.ps1/sh script this variable isn't automatically set.
$ci = $true
. $PSScriptRoot\..\tools.ps1
$argumentList = @("install", "Microsoft.Guardian.Cli", "-Source https://securitytools.pkgs.visualstudio.com/_packaging/Guardian/nuget/v3/index.json", "-OutputDirectory $Path", "-NonInteractive", "-NoCache")
if ($Version) {
$argumentList += "-Version $Version"
}
Start-Process nuget -Verbose -ArgumentList $argumentList -NoNewWindow -Wait
$gdnCliPath = Get-ChildItem -Filter guardian.cmd -Recurse -Path $Path
if (!$gdnCliPath)
{
Write-PipelineTelemetryError -Category 'Sdl' -Message 'Failure installing Guardian'
}
return $gdnCliPath.FullName
}

2
eng/common/templates/job/execute-sdl.yml

@ -54,7 +54,7 @@ jobs:
# If it's not devdiv, it's dnceng # If it's not devdiv, it's dnceng
${{ if ne(variables['System.TeamProject'], 'DevDiv') }}: ${{ if ne(variables['System.TeamProject'], 'DevDiv') }}:
name: NetCore1ESPool-Internal name: NetCore1ESPool-Internal
demands: ImageOverride -equals Build.Server.Amd64.VS2019 demands: ImageOverride -equals windows.vs2019.amd64
steps: steps:
- checkout: self - checkout: self
clean: true clean: true

2
eng/common/templates/job/onelocbuild.yml

@ -41,7 +41,7 @@ jobs:
# If it's not devdiv, it's dnceng # If it's not devdiv, it's dnceng
${{ if ne(variables['System.TeamProject'], 'DevDiv') }}: ${{ if ne(variables['System.TeamProject'], 'DevDiv') }}:
name: NetCore1ESPool-Internal name: NetCore1ESPool-Internal
demands: ImageOverride -equals Build.Server.Amd64.VS2019 demands: ImageOverride -equals windows.vs2019.amd64
variables: variables:
- group: OneLocBuildVariables # Contains the CeapexPat and GithubPat - group: OneLocBuildVariables # Contains the CeapexPat and GithubPat

2
eng/common/templates/job/source-build.yml

@ -46,7 +46,7 @@ jobs:
# source-build builds run in Docker, including the default managed platform. # source-build builds run in Docker, including the default managed platform.
pool: pool:
${{ if eq(variables['System.TeamProject'], 'public') }}: ${{ if eq(variables['System.TeamProject'], 'public') }}:
name: NetCore1ESPool-Public name: NetCore-Public
demands: ImageOverride -equals Build.Ubuntu.1804.Amd64.Open demands: ImageOverride -equals Build.Ubuntu.1804.Amd64.Open
${{ if eq(variables['System.TeamProject'], 'internal') }}: ${{ if eq(variables['System.TeamProject'], 'internal') }}:
name: NetCore1ESPool-Internal name: NetCore1ESPool-Internal

8
eng/common/templates/job/source-index-stage1.yml

@ -1,6 +1,6 @@
parameters: parameters:
runAsPublic: false runAsPublic: false
sourceIndexPackageVersion: 1.0.1-20210614.1 sourceIndexPackageVersion: 1.0.1-20220804.1
sourceIndexPackageSource: https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools/nuget/v3/index.json sourceIndexPackageSource: https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools/nuget/v3/index.json
sourceIndexBuildCommand: powershell -NoLogo -NoProfile -ExecutionPolicy Bypass -Command "eng/common/build.ps1 -restore -build -binarylog -ci" sourceIndexBuildCommand: powershell -NoLogo -NoProfile -ExecutionPolicy Bypass -Command "eng/common/build.ps1 -restore -build -binarylog -ci"
preSteps: [] preSteps: []
@ -28,11 +28,11 @@ jobs:
${{ if eq(parameters.pool, '') }}: ${{ if eq(parameters.pool, '') }}:
pool: pool:
${{ if eq(variables['System.TeamProject'], 'public') }}: ${{ if eq(variables['System.TeamProject'], 'public') }}:
name: NetCore1ESPool-Public name: NetCore-Public
demands: ImageOverride -equals Build.Server.Amd64.VS2019.Open demands: ImageOverride -equals windows.vs2019.amd64.open
${{ if eq(variables['System.TeamProject'], 'internal') }}: ${{ if eq(variables['System.TeamProject'], 'internal') }}:
name: NetCore1ESPool-Internal name: NetCore1ESPool-Internal
demands: ImageOverride -equals Build.Server.Amd64.VS2019 demands: ImageOverride -equals windows.vs2019.amd64
steps: steps:
- ${{ each preStep in parameters.preSteps }}: - ${{ each preStep in parameters.preSteps }}:

2
eng/common/templates/jobs/jobs.yml

@ -96,7 +96,7 @@ jobs:
# If it's not devdiv, it's dnceng # If it's not devdiv, it's dnceng
${{ if ne(variables['System.TeamProject'], 'DevDiv') }}: ${{ if ne(variables['System.TeamProject'], 'DevDiv') }}:
name: NetCore1ESPool-Internal name: NetCore1ESPool-Internal
demands: ImageOverride -equals Build.Server.Amd64.VS2019 demands: ImageOverride -equals windows.vs2019.amd64
runAsPublic: ${{ parameters.runAsPublic }} runAsPublic: ${{ parameters.runAsPublic }}
publishUsingPipelines: ${{ parameters.enablePublishUsingPipelines }} publishUsingPipelines: ${{ parameters.enablePublishUsingPipelines }}

10
eng/common/templates/post-build/post-build.yml

@ -49,6 +49,7 @@ parameters:
type: object type: object
default: default:
enable: false enable: false
publishGdn: false
continueOnError: false continueOnError: false
params: '' params: ''
artifactNames: '' artifactNames: ''
@ -106,7 +107,7 @@ stages:
# If it's not devdiv, it's dnceng # If it's not devdiv, it's dnceng
${{ else }}: ${{ else }}:
name: NetCore1ESPool-Internal name: NetCore1ESPool-Internal
demands: ImageOverride -equals Build.Server.Amd64.VS2019 demands: ImageOverride -equals windows.vs2019.amd64
steps: steps:
- template: setup-maestro-vars.yml - template: setup-maestro-vars.yml
@ -143,7 +144,7 @@ stages:
# If it's not devdiv, it's dnceng # If it's not devdiv, it's dnceng
${{ else }}: ${{ else }}:
name: NetCore1ESPool-Internal name: NetCore1ESPool-Internal
demands: ImageOverride -equals Build.Server.Amd64.VS2019 demands: ImageOverride -equals windows.vs2019.amd64
steps: steps:
- template: setup-maestro-vars.yml - template: setup-maestro-vars.yml
parameters: parameters:
@ -203,7 +204,7 @@ stages:
# If it's not devdiv, it's dnceng # If it's not devdiv, it's dnceng
${{ else }}: ${{ else }}:
name: NetCore1ESPool-Internal name: NetCore1ESPool-Internal
demands: ImageOverride -equals Build.Server.Amd64.VS2019 demands: ImageOverride -equals windows.vs2019.amd64
steps: steps:
- template: setup-maestro-vars.yml - template: setup-maestro-vars.yml
parameters: parameters:
@ -235,6 +236,7 @@ stages:
- template: /eng/common/templates/job/execute-sdl.yml - template: /eng/common/templates/job/execute-sdl.yml
parameters: parameters:
enable: ${{ parameters.SDLValidationParameters.enable }} enable: ${{ parameters.SDLValidationParameters.enable }}
publishGuardianDirectoryToPipeline: ${{ parameters.SDLValidationParameters.publishGdn }}
additionalParameters: ${{ parameters.SDLValidationParameters.params }} additionalParameters: ${{ parameters.SDLValidationParameters.params }}
continueOnError: ${{ parameters.SDLValidationParameters.continueOnError }} continueOnError: ${{ parameters.SDLValidationParameters.continueOnError }}
artifactNames: ${{ parameters.SDLValidationParameters.artifactNames }} artifactNames: ${{ parameters.SDLValidationParameters.artifactNames }}
@ -261,7 +263,7 @@ stages:
# If it's not devdiv, it's dnceng # If it's not devdiv, it's dnceng
${{ else }}: ${{ else }}:
name: NetCore1ESPool-Internal name: NetCore1ESPool-Internal
demands: ImageOverride -equals Build.Server.Amd64.VS2019 demands: ImageOverride -equals windows.vs2019.amd64
steps: steps:
- template: setup-maestro-vars.yml - template: setup-maestro-vars.yml
parameters: parameters:

58
eng/common/templates/steps/execute-sdl.yml

@ -8,29 +8,26 @@ parameters:
condition: '' condition: ''
steps: steps:
- ${{ if ne(parameters.overrideGuardianVersion, '') }}: - task: NuGetAuthenticate@1
- powershell: | inputs:
$content = Get-Content $(GuardianPackagesConfigFile) nuGetServiceConnections: GuardianConnect
Write-Host "packages.config content was:`n$content"
$content = $content.Replace('$(DefaultGuardianVersion)', '$(GuardianVersion)')
$content | Set-Content $(GuardianPackagesConfigFile)
Write-Host "packages.config content updated to:`n$content"
displayName: Use overridden Guardian version ${{ parameters.overrideGuardianVersion }}
- task: NuGetToolInstaller@1 - task: NuGetToolInstaller@1
displayName: 'Install NuGet.exe' displayName: 'Install NuGet.exe'
- task: NuGetCommand@2 - ${{ if ne(parameters.overrideGuardianVersion, '') }}:
displayName: 'Install Guardian' - pwsh: |
inputs: . $(Build.SourcesDirectory)\eng\common\sdl\sdl.ps1
restoreSolution: $(Build.SourcesDirectory)\eng\common\sdl\packages.config $guardianCliLocation = Install-Gdn -Path $(Build.SourcesDirectory)\.artifacts -Version ${{ parameters.overrideGuardianVersion }}
feedsToUse: config Write-Host "##vso[task.setvariable variable=GuardianCliLocation]$guardianCliLocation"
nugetConfigPath: $(Build.SourcesDirectory)\eng\common\sdl\NuGet.config displayName: Install Guardian (Overridden)
externalFeedCredentials: GuardianConnect
restoreDirectory: $(Build.SourcesDirectory)\.packages - ${{ if eq(parameters.overrideGuardianVersion, '') }}:
- pwsh: |
. $(Build.SourcesDirectory)\eng\common\sdl\sdl.ps1
$guardianCliLocation = Install-Gdn -Path $(Build.SourcesDirectory)\.artifacts
Write-Host "##vso[task.setvariable variable=GuardianCliLocation]$guardianCliLocation"
displayName: Install Guardian
- ${{ if ne(parameters.overrideParameters, '') }}: - ${{ if ne(parameters.overrideParameters, '') }}:
- powershell: ${{ parameters.executeAllSdlToolsScript }} ${{ parameters.overrideParameters }} - powershell: ${{ parameters.executeAllSdlToolsScript }} ${{ parameters.overrideParameters }}
@ -40,7 +37,7 @@ steps:
- ${{ if eq(parameters.overrideParameters, '') }}: - ${{ if eq(parameters.overrideParameters, '') }}:
- powershell: ${{ parameters.executeAllSdlToolsScript }} - powershell: ${{ parameters.executeAllSdlToolsScript }}
-GuardianPackageName Microsoft.Guardian.Cli.$(GuardianVersion) -GuardianCliLocation $(GuardianCliLocation)
-NugetPackageDirectory $(Build.SourcesDirectory)\.packages -NugetPackageDirectory $(Build.SourcesDirectory)\.packages
-AzureDevOpsAccessToken $(dn-bot-dotnet-build-rw-code-rw) -AzureDevOpsAccessToken $(dn-bot-dotnet-build-rw-code-rw)
${{ parameters.additionalParameters }} ${{ parameters.additionalParameters }}
@ -62,7 +59,28 @@ steps:
c c
i i
condition: succeededOrFailed() condition: succeededOrFailed()
- publish: $(Agent.BuildDirectory)/.gdn - publish: $(Agent.BuildDirectory)/.gdn
artifact: GuardianConfiguration artifact: GuardianConfiguration
displayName: Publish GuardianConfiguration displayName: Publish GuardianConfiguration
condition: succeededOrFailed()
# Publish the SARIF files in a container named CodeAnalysisLogs to enable integration
# with the "SARIF SAST Scans Tab" Azure DevOps extension
- task: CopyFiles@2
displayName: Copy SARIF files
inputs:
flattenFolders: true
sourceFolder: $(Agent.BuildDirectory)/.gdn/rc/
contents: '**/*.sarif'
targetFolder: $(Build.SourcesDirectory)/CodeAnalysisLogs
condition: succeededOrFailed()
# Use PublishBuildArtifacts because the SARIF extension only checks this case
# see microsoft/sarif-azuredevops-extension#4
- task: PublishBuildArtifacts@1
displayName: Publish SARIF files to CodeAnalysisLogs container
inputs:
pathToPublish: $(Build.SourcesDirectory)/CodeAnalysisLogs
artifactName: CodeAnalysisLogs
condition: succeededOrFailed() condition: succeededOrFailed()

15
eng/common/tools.ps1

@ -365,10 +365,17 @@ function InitializeVisualStudioMSBuild([bool]$install, [object]$vsRequirements =
# If the version of msbuild is going to be xcopied, # If the version of msbuild is going to be xcopied,
# use this version. Version matches a package here: # use this version. Version matches a package here:
# https://dev.azure.com/dnceng/public/_packaging?_a=package&feed=dotnet-eng&package=RoslynTools.MSBuild&protocolType=NuGet&version=17.1.0&view=overview # https://dev.azure.com/dnceng/public/_packaging?_a=package&feed=dotnet-eng&package=RoslynTools.MSBuild&protocolType=NuGet&version=17.2.1&view=overview
$defaultXCopyMSBuildVersion = '17.1.0' $defaultXCopyMSBuildVersion = '17.2.1'
if (!$vsRequirements) { $vsRequirements = $GlobalJson.tools.vs } if (!$vsRequirements) {
if (Get-Member -InputObject $GlobalJson.tools -Name 'vs') {
$vsRequirements = $GlobalJson.tools.vs
}
else {
$vsRequirements = New-Object PSObject -Property @{ version = $vsMinVersionReqdStr }
}
}
$vsMinVersionStr = if ($vsRequirements.version) { $vsRequirements.version } else { $vsMinVersionReqdStr } $vsMinVersionStr = if ($vsRequirements.version) { $vsRequirements.version } else { $vsMinVersionReqdStr }
$vsMinVersion = [Version]::new($vsMinVersionStr) $vsMinVersion = [Version]::new($vsMinVersionStr)
@ -635,7 +642,7 @@ function InitializeNativeTools() {
InstallDirectory = "$ToolsDir" InstallDirectory = "$ToolsDir"
} }
} }
if (Test-Path variable:NativeToolsOnMachine) { if ($env:NativeToolsOnMachine) {
Write-Host "Variable NativeToolsOnMachine detected, enabling native tool path promotion..." Write-Host "Variable NativeToolsOnMachine detected, enabling native tool path promotion..."
$nativeArgs += @{ PathPromotion = $true } $nativeArgs += @{ PathPromotion = $true }
} }

4
global.json

@ -3,7 +3,7 @@
"allowPrerelease": true "allowPrerelease": true
}, },
"tools": { "tools": {
"dotnet": "7.0.100-preview.2.22153.17", "dotnet": "7.0.100-preview.7.22377.5",
"runtimes": { "runtimes": {
"dotnet": [ "dotnet": [
"6.0.0" "6.0.0"
@ -14,6 +14,6 @@
} }
}, },
"msbuild-sdks": { "msbuild-sdks": {
"Microsoft.DotNet.Arcade.Sdk": "7.0.0-beta.22327.2" "Microsoft.DotNet.Arcade.Sdk": "8.0.0-beta.22455.1"
} }
} }

Loading…
Cancel
Save