Browse Source

[main] Update dependencies from dotnet/arcade (#1457)

[main] Update dependencies from dotnet/arcade


 - Update Microsoft.Tye.UnitTests.csproj

 - Work around issue on CI
main
dotnet-maestro[bot] 4 years ago
committed by GitHub
parent
commit
5ef078104e
No known key found for this signature in database GPG Key ID: 4AEE18F83AFDEB23
  1. 20
      eng/Version.Details.xml
  2. 4
      eng/common/BuildConfiguration/build-configuration.json
  3. 18
      eng/common/SetupNugetSources.ps1
  4. 68
      eng/common/SetupNugetSources.sh
  5. 5
      eng/common/build.ps1
  6. 17
      eng/common/build.sh
  7. 170
      eng/common/cross/armel/tizen-fetch.sh
  8. 50
      eng/common/cross/armel/tizen/tizen-dotnet.ks
  9. 4
      eng/common/cross/build-android-rootfs.sh
  10. 335
      eng/common/cross/build-rootfs.sh
  11. 1
      eng/common/cross/riscv64/sources.list.sid
  12. 34
      eng/common/cross/tizen-build-rootfs.sh
  13. 172
      eng/common/cross/tizen-fetch.sh
  14. 111
      eng/common/cross/toolchain.cmake
  15. 47
      eng/common/generate-locproject.ps1
  16. 2
      eng/common/generate-sbom-prep.ps1
  17. 12
      eng/common/generate-sbom-prep.sh
  18. 10
      eng/common/init-tools-native.ps1
  19. 83
      eng/common/native/init-compiler.sh
  20. 2
      eng/common/sdk-task.ps1
  21. 5
      eng/common/sdl/NuGet.config
  22. 38
      eng/common/sdl/sdl.ps1
  23. 11
      eng/common/templates/job/execute-sdl.yml
  24. 47
      eng/common/templates/job/job.yml
  25. 45
      eng/common/templates/job/onelocbuild.yml
  26. 20
      eng/common/templates/job/publish-build-assets.yml
  27. 15
      eng/common/templates/job/source-build.yml
  28. 11
      eng/common/templates/job/source-index-stage1.yml
  29. 9
      eng/common/templates/jobs/jobs.yml
  30. 2
      eng/common/templates/jobs/source-build.yml
  31. 4
      eng/common/templates/post-build/common-variables.yml
  32. 30
      eng/common/templates/post-build/post-build.yml
  33. 60
      eng/common/templates/steps/execute-sdl.yml
  34. 4
      eng/common/templates/steps/generate-sbom.yml
  35. 14
      eng/common/templates/steps/source-build.yml
  36. 48
      eng/common/templates/variables/pool-providers.yml
  37. 19
      eng/common/tools.ps1
  38. 24
      eng/common/tools.sh
  39. 4
      global.json
  40. 1
      test/E2ETest/TyeRunTests.cs
  41. 1
      test/UnitTests/Microsoft.Tye.UnitTests.csproj

20
eng/Version.Details.xml

@ -3,25 +3,25 @@
<ProductDependencies> <ProductDependencies>
</ProductDependencies> </ProductDependencies>
<ToolsetDependencies> <ToolsetDependencies>
<Dependency Name="Microsoft.DotNet.Arcade.Sdk" Version="7.0.0-beta.22327.2"> <Dependency Name="Microsoft.DotNet.Arcade.Sdk" Version="8.0.0-beta.22621.1">
<Uri>https://github.com/dotnet/arcade</Uri> <Uri>https://github.com/dotnet/arcade</Uri>
<Sha>a264eb13fea14125f3ef8d4056586cd66fa55309</Sha> <Sha>3003926e4126f827bca50d5b3ee179afc86d8a7b</Sha>
</Dependency> </Dependency>
<Dependency Name="Microsoft.DotNet.Build.Tasks.Feed" Version="7.0.0-beta.22327.2"> <Dependency Name="Microsoft.DotNet.Build.Tasks.Feed" Version="8.0.0-beta.22621.1">
<Uri>https://github.com/dotnet/arcade</Uri> <Uri>https://github.com/dotnet/arcade</Uri>
<Sha>a264eb13fea14125f3ef8d4056586cd66fa55309</Sha> <Sha>3003926e4126f827bca50d5b3ee179afc86d8a7b</Sha>
</Dependency> </Dependency>
<Dependency Name="Microsoft.DotNet.SignTool" Version="7.0.0-beta.22327.2"> <Dependency Name="Microsoft.DotNet.SignTool" Version="8.0.0-beta.22621.1">
<Uri>https://github.com/dotnet/arcade</Uri> <Uri>https://github.com/dotnet/arcade</Uri>
<Sha>a264eb13fea14125f3ef8d4056586cd66fa55309</Sha> <Sha>3003926e4126f827bca50d5b3ee179afc86d8a7b</Sha>
</Dependency> </Dependency>
<Dependency Name="Microsoft.DotNet.Helix.Sdk" Version="7.0.0-beta.22327.2"> <Dependency Name="Microsoft.DotNet.Helix.Sdk" Version="8.0.0-beta.22621.1">
<Uri>https://github.com/dotnet/arcade</Uri> <Uri>https://github.com/dotnet/arcade</Uri>
<Sha>a264eb13fea14125f3ef8d4056586cd66fa55309</Sha> <Sha>3003926e4126f827bca50d5b3ee179afc86d8a7b</Sha>
</Dependency> </Dependency>
<Dependency Name="Microsoft.DotNet.SwaggerGenerator.MSBuild" Version="7.0.0-beta.22327.2"> <Dependency Name="Microsoft.DotNet.SwaggerGenerator.MSBuild" Version="8.0.0-beta.22621.1">
<Uri>https://github.com/dotnet/arcade</Uri> <Uri>https://github.com/dotnet/arcade</Uri>
<Sha>a264eb13fea14125f3ef8d4056586cd66fa55309</Sha> <Sha>3003926e4126f827bca50d5b3ee179afc86d8a7b</Sha>
</Dependency> </Dependency>
<Dependency Name="Microsoft.DotNet.Maestro.Client" Version="1.1.0-beta.19556.4"> <Dependency Name="Microsoft.DotNet.Maestro.Client" Version="1.1.0-beta.19556.4">
<Uri>https://github.com/dotnet/arcade-services</Uri> <Uri>https://github.com/dotnet/arcade-services</Uri>

4
eng/common/BuildConfiguration/build-configuration.json

@ -0,0 +1,4 @@
{
"RetryCountLimit": 1,
"RetryByAnyError": false
}

18
eng/common/SetupNugetSources.ps1

@ -146,22 +146,22 @@ $userName = "dn-bot"
# Insert credential nodes for Maestro's private feeds # Insert credential nodes for Maestro's private feeds
InsertMaestroPrivateFeedCredentials -Sources $sources -Creds $creds -Username $userName -Password $Password InsertMaestroPrivateFeedCredentials -Sources $sources -Creds $creds -Username $userName -Password $Password
# 3.1 uses a different feed url format so it's handled differently here
$dotnet31Source = $sources.SelectSingleNode("add[@key='dotnet3.1']") $dotnet31Source = $sources.SelectSingleNode("add[@key='dotnet3.1']")
if ($dotnet31Source -ne $null) { if ($dotnet31Source -ne $null) {
AddPackageSource -Sources $sources -SourceName "dotnet3.1-internal" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/_packaging/dotnet3.1-internal/nuget/v2" -Creds $creds -Username $userName -Password $Password AddPackageSource -Sources $sources -SourceName "dotnet3.1-internal" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/_packaging/dotnet3.1-internal/nuget/v2" -Creds $creds -Username $userName -Password $Password
AddPackageSource -Sources $sources -SourceName "dotnet3.1-internal-transport" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/_packaging/dotnet3.1-internal-transport/nuget/v2" -Creds $creds -Username $userName -Password $Password AddPackageSource -Sources $sources -SourceName "dotnet3.1-internal-transport" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/_packaging/dotnet3.1-internal-transport/nuget/v2" -Creds $creds -Username $userName -Password $Password
} }
$dotnet5Source = $sources.SelectSingleNode("add[@key='dotnet5']") $dotnetVersions = @('5','6','7')
if ($dotnet5Source -ne $null) {
AddPackageSource -Sources $sources -SourceName "dotnet5-internal" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/internal/_packaging/dotnet5-internal/nuget/v2" -Creds $creds -Username $userName -Password $Password
AddPackageSource -Sources $sources -SourceName "dotnet5-internal-transport" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/internal/_packaging/dotnet5-internal-transport/nuget/v2" -Creds $creds -Username $userName -Password $Password
}
$dotnet6Source = $sources.SelectSingleNode("add[@key='dotnet6']") foreach ($dotnetVersion in $dotnetVersions) {
if ($dotnet6Source -ne $null) { $feedPrefix = "dotnet" + $dotnetVersion;
AddPackageSource -Sources $sources -SourceName "dotnet6-internal" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/internal/_packaging/dotnet6-internal/nuget/v2" -Creds $creds -Username $userName -Password $Password $dotnetSource = $sources.SelectSingleNode("add[@key='$feedPrefix']")
AddPackageSource -Sources $sources -SourceName "dotnet6-internal-transport" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/internal/_packaging/dotnet6-internal-transport/nuget/v2" -Creds $creds -Username $userName -Password $Password if ($dotnetSource -ne $null) {
AddPackageSource -Sources $sources -SourceName "$feedPrefix-internal" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/internal/_packaging/$feedPrefix-internal/nuget/v2" -Creds $creds -Username $userName -Password $Password
AddPackageSource -Sources $sources -SourceName "$feedPrefix-internal-transport" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/internal/_packaging/$feedPrefix-internal-transport/nuget/v2" -Creds $creds -Username $userName -Password $Password
}
} }
$doc.Save($filename) $doc.Save($filename)

68
eng/common/SetupNugetSources.sh

@ -105,53 +105,33 @@ if [ "$?" == "0" ]; then
PackageSources+=('dotnet3.1-internal-transport') PackageSources+=('dotnet3.1-internal-transport')
fi fi
# Ensure dotnet5-internal and dotnet5-internal-transport are in the packageSources if the public dotnet5 feeds are present DotNetVersions=('5' '6' '7')
grep -i "<add key=\"dotnet5\"" $ConfigFile
if [ "$?" == "0" ]; then for DotNetVersion in ${DotNetVersions[@]} ; do
grep -i "<add key=\"dotnet5-internal\"" $ConfigFile FeedPrefix="dotnet${DotNetVersion}";
if [ "$?" != "0" ]; then grep -i "<add key=\"$FeedPrefix\"" $ConfigFile
echo "Adding dotnet5-internal to the packageSources." if [ "$?" == "0" ]; then
PackageSourcesNodeFooter="</packageSources>" grep -i "<add key=\"$FeedPrefix-internal\"" $ConfigFile
PackageSourceTemplate="${TB}<add key=\"dotnet5-internal\" value=\"https://pkgs.dev.azure.com/dnceng/internal/_packaging/dotnet5-internal/nuget/v2\" />" if [ "$?" != "0" ]; then
echo "Adding $FeedPrefix-internal to the packageSources."
sed -i.bak "s|$PackageSourcesNodeFooter|$PackageSourceTemplate${NL}$PackageSourcesNodeFooter|" $ConfigFile PackageSourcesNodeFooter="</packageSources>"
fi PackageSourceTemplate="${TB}<add key=\"$FeedPrefix-internal\" value=\"https://pkgs.dev.azure.com/dnceng/internal/_packaging/$FeedPrefix-internal/nuget/v2\" />"
PackageSources+=('dotnet5-internal')
sed -i.bak "s|$PackageSourcesNodeFooter|$PackageSourceTemplate${NL}$PackageSourcesNodeFooter|" $ConfigFile
grep -i "<add key=\"dotnet5-internal-transport\">" $ConfigFile fi
if [ "$?" != "0" ]; then PackageSources+=("$FeedPrefix-internal")
echo "Adding dotnet5-internal-transport to the packageSources."
PackageSourcesNodeFooter="</packageSources>"
PackageSourceTemplate="${TB}<add key=\"dotnet5-internal-transport\" value=\"https://pkgs.dev.azure.com/dnceng/internal/_packaging/dotnet5-internal-transport/nuget/v2\" />"
sed -i.bak "s|$PackageSourcesNodeFooter|$PackageSourceTemplate${NL}$PackageSourcesNodeFooter|" $ConfigFile
fi
PackageSources+=('dotnet5-internal-transport')
fi
# Ensure dotnet6-internal and dotnet6-internal-transport are in the packageSources if the public dotnet6 feeds are present
grep -i "<add key=\"dotnet6\"" $ConfigFile
if [ "$?" == "0" ]; then
grep -i "<add key=\"dotnet6-internal\"" $ConfigFile
if [ "$?" != "0" ]; then
echo "Adding dotnet6-internal to the packageSources."
PackageSourcesNodeFooter="</packageSources>"
PackageSourceTemplate="${TB}<add key=\"dotnet6-internal\" value=\"https://pkgs.dev.azure.com/dnceng/internal/_packaging/dotnet6-internal/nuget/v2\" />"
sed -i.bak "s|$PackageSourcesNodeFooter|$PackageSourceTemplate${NL}$PackageSourcesNodeFooter|" $ConfigFile grep -i "<add key=\"$FeedPrefix-internal-transport\">" $ConfigFile
fi if [ "$?" != "0" ]; then
PackageSources+=('dotnet6-internal') echo "Adding $FeedPrefix-internal-transport to the packageSources."
PackageSourcesNodeFooter="</packageSources>"
PackageSourceTemplate="${TB}<add key=\"$FeedPrefix-internal-transport\" value=\"https://pkgs.dev.azure.com/dnceng/internal/_packaging/$FeedPrefix-internal-transport/nuget/v2\" />"
grep -i "<add key=\"dotnet6-internal-transport\">" $ConfigFile sed -i.bak "s|$PackageSourcesNodeFooter|$PackageSourceTemplate${NL}$PackageSourcesNodeFooter|" $ConfigFile
if [ "$?" != "0" ]; then fi
echo "Adding dotnet6-internal-transport to the packageSources." PackageSources+=("$FeedPrefix-internal-transport")
PackageSourcesNodeFooter="</packageSources>"
PackageSourceTemplate="${TB}<add key=\"dotnet6-internal-transport\" value=\"https://pkgs.dev.azure.com/dnceng/internal/_packaging/dotnet6-internal-transport/nuget/v2\" />"
sed -i.bak "s|$PackageSourcesNodeFooter|$PackageSourceTemplate${NL}$PackageSourcesNodeFooter|" $ConfigFile
fi fi
PackageSources+=('dotnet6-internal-transport') done
fi
# I want things split line by line # I want things split line by line
PrevIFS=$IFS PrevIFS=$IFS

5
eng/common/build.ps1

@ -26,6 +26,7 @@ Param(
[string] $runtimeSourceFeed = '', [string] $runtimeSourceFeed = '',
[string] $runtimeSourceFeedKey = '', [string] $runtimeSourceFeedKey = '',
[switch] $excludePrereleaseVS, [switch] $excludePrereleaseVS,
[switch] $nativeToolsOnMachine,
[switch] $help, [switch] $help,
[Parameter(ValueFromRemainingArguments=$true)][String[]]$properties [Parameter(ValueFromRemainingArguments=$true)][String[]]$properties
) )
@ -67,6 +68,7 @@ function Print-Usage() {
Write-Host " -warnAsError <value> Sets warnaserror msbuild parameter ('true' or 'false')" Write-Host " -warnAsError <value> Sets warnaserror msbuild parameter ('true' or 'false')"
Write-Host " -msbuildEngine <value> Msbuild engine to use to run build ('dotnet', 'vs', or unspecified)." Write-Host " -msbuildEngine <value> Msbuild engine to use to run build ('dotnet', 'vs', or unspecified)."
Write-Host " -excludePrereleaseVS Set to exclude build engines in prerelease versions of Visual Studio" Write-Host " -excludePrereleaseVS Set to exclude build engines in prerelease versions of Visual Studio"
Write-Host " -nativeToolsOnMachine Sets the native tools on machine environment variable (indicating that the script should use native tools on machine)"
Write-Host "" Write-Host ""
Write-Host "Command line arguments not listed above are passed thru to msbuild." Write-Host "Command line arguments not listed above are passed thru to msbuild."
@ -146,6 +148,9 @@ try {
$nodeReuse = $false $nodeReuse = $false
} }
if ($nativeToolsOnMachine) {
$env:NativeToolsOnMachine = $true
}
if ($restore) { if ($restore) {
InitializeNativeTools InitializeNativeTools
} }

17
eng/common/build.sh

@ -19,6 +19,9 @@ usage()
echo "Actions:" echo "Actions:"
echo " --restore Restore dependencies (short: -r)" echo " --restore Restore dependencies (short: -r)"
echo " --build Build solution (short: -b)" echo " --build Build solution (short: -b)"
echo " --sourceBuild Source-build the solution (short: -sb)"
echo " Will additionally trigger the following actions: --restore, --build, --pack"
echo " If --configuration is not set explicitly, will also set it to 'Release'"
echo " --rebuild Rebuild solution" echo " --rebuild Rebuild solution"
echo " --test Run all unit tests in the solution (short: -t)" echo " --test Run all unit tests in the solution (short: -t)"
echo " --integrationTest Run all integration tests in the solution" echo " --integrationTest Run all integration tests in the solution"
@ -55,6 +58,7 @@ scriptroot="$( cd -P "$( dirname "$source" )" && pwd )"
restore=false restore=false
build=false build=false
source_build=false
rebuild=false rebuild=false
test=false test=false
integration_test=false integration_test=false
@ -73,7 +77,7 @@ exclude_ci_binary_log=false
pipelines_log=false pipelines_log=false
projects='' projects=''
configuration='Debug' configuration=''
prepare_machine=false prepare_machine=false
verbosity='minimal' verbosity='minimal'
runtime_source_feed='' runtime_source_feed=''
@ -119,6 +123,12 @@ while [[ $# > 0 ]]; do
-pack) -pack)
pack=true pack=true
;; ;;
-sourcebuild|-sb)
build=true
source_build=true
restore=true
pack=true
;;
-test|-t) -test|-t)
test=true test=true
;; ;;
@ -168,6 +178,10 @@ while [[ $# > 0 ]]; do
shift shift
done done
if [[ -z "$configuration" ]]; then
if [[ "$source_build" = true ]]; then configuration="Release"; else configuration="Debug"; fi
fi
if [[ "$ci" == true ]]; then if [[ "$ci" == true ]]; then
pipelines_log=true pipelines_log=true
node_reuse=false node_reuse=false
@ -205,6 +219,7 @@ function Build {
/p:RepoRoot="$repo_root" \ /p:RepoRoot="$repo_root" \
/p:Restore=$restore \ /p:Restore=$restore \
/p:Build=$build \ /p:Build=$build \
/p:ArcadeBuildFromSource=$source_build \
/p:Rebuild=$rebuild \ /p:Rebuild=$rebuild \
/p:Test=$test \ /p:Test=$test \
/p:Pack=$pack \ /p:Pack=$pack \

170
eng/common/cross/armel/tizen-fetch.sh

@ -1,170 +0,0 @@
#!/usr/bin/env bash
set -e
if [[ -z "${VERBOSE// }" ]] || [ "$VERBOSE" -ne "$VERBOSE" ] 2>/dev/null; then
VERBOSE=0
fi
Log()
{
if [ $VERBOSE -ge $1 ]; then
echo ${@:2}
fi
}
Inform()
{
Log 1 -e "\x1B[0;34m$@\x1B[m"
}
Debug()
{
Log 2 -e "\x1B[0;32m$@\x1B[m"
}
Error()
{
>&2 Log 0 -e "\x1B[0;31m$@\x1B[m"
}
Fetch()
{
URL=$1
FILE=$2
PROGRESS=$3
if [ $VERBOSE -ge 1 ] && [ $PROGRESS ]; then
CURL_OPT="--progress-bar"
else
CURL_OPT="--silent"
fi
curl $CURL_OPT $URL > $FILE
}
hash curl 2> /dev/null || { Error "Require 'curl' Aborting."; exit 1; }
hash xmllint 2> /dev/null || { Error "Require 'xmllint' Aborting."; exit 1; }
hash sha256sum 2> /dev/null || { Error "Require 'sha256sum' Aborting."; exit 1; }
TMPDIR=$1
if [ ! -d $TMPDIR ]; then
TMPDIR=./tizen_tmp
Debug "Create temporary directory : $TMPDIR"
mkdir -p $TMPDIR
fi
TIZEN_URL=http://download.tizen.org/snapshots/tizen
BUILD_XML=build.xml
REPOMD_XML=repomd.xml
PRIMARY_XML=primary.xml
TARGET_URL="http://__not_initialized"
Xpath_get()
{
XPATH_RESULT=''
XPATH=$1
XML_FILE=$2
RESULT=$(xmllint --xpath $XPATH $XML_FILE)
if [[ -z ${RESULT// } ]]; then
Error "Can not find target from $XML_FILE"
Debug "Xpath = $XPATH"
exit 1
fi
XPATH_RESULT=$RESULT
}
fetch_tizen_pkgs_init()
{
TARGET=$1
PROFILE=$2
Debug "Initialize TARGET=$TARGET, PROFILE=$PROFILE"
TMP_PKG_DIR=$TMPDIR/tizen_${PROFILE}_pkgs
if [ -d $TMP_PKG_DIR ]; then rm -rf $TMP_PKG_DIR; fi
mkdir -p $TMP_PKG_DIR
PKG_URL=$TIZEN_URL/$PROFILE/latest
BUILD_XML_URL=$PKG_URL/$BUILD_XML
TMP_BUILD=$TMP_PKG_DIR/$BUILD_XML
TMP_REPOMD=$TMP_PKG_DIR/$REPOMD_XML
TMP_PRIMARY=$TMP_PKG_DIR/$PRIMARY_XML
TMP_PRIMARYGZ=${TMP_PRIMARY}.gz
Fetch $BUILD_XML_URL $TMP_BUILD
Debug "fetch $BUILD_XML_URL to $TMP_BUILD"
TARGET_XPATH="//build/buildtargets/buildtarget[@name=\"$TARGET\"]/repo[@type=\"binary\"]/text()"
Xpath_get $TARGET_XPATH $TMP_BUILD
TARGET_PATH=$XPATH_RESULT
TARGET_URL=$PKG_URL/$TARGET_PATH
REPOMD_URL=$TARGET_URL/repodata/repomd.xml
PRIMARY_XPATH='string(//*[local-name()="data"][@type="primary"]/*[local-name()="location"]/@href)'
Fetch $REPOMD_URL $TMP_REPOMD
Debug "fetch $REPOMD_URL to $TMP_REPOMD"
Xpath_get $PRIMARY_XPATH $TMP_REPOMD
PRIMARY_XML_PATH=$XPATH_RESULT
PRIMARY_URL=$TARGET_URL/$PRIMARY_XML_PATH
Fetch $PRIMARY_URL $TMP_PRIMARYGZ
Debug "fetch $PRIMARY_URL to $TMP_PRIMARYGZ"
gunzip $TMP_PRIMARYGZ
Debug "unzip $TMP_PRIMARYGZ to $TMP_PRIMARY"
}
fetch_tizen_pkgs()
{
ARCH=$1
PACKAGE_XPATH_TPL='string(//*[local-name()="metadata"]/*[local-name()="package"][*[local-name()="name"][text()="_PKG_"]][*[local-name()="arch"][text()="_ARCH_"]]/*[local-name()="location"]/@href)'
PACKAGE_CHECKSUM_XPATH_TPL='string(//*[local-name()="metadata"]/*[local-name()="package"][*[local-name()="name"][text()="_PKG_"]][*[local-name()="arch"][text()="_ARCH_"]]/*[local-name()="checksum"]/text())'
for pkg in ${@:2}
do
Inform "Fetching... $pkg"
XPATH=${PACKAGE_XPATH_TPL/_PKG_/$pkg}
XPATH=${XPATH/_ARCH_/$ARCH}
Xpath_get $XPATH $TMP_PRIMARY
PKG_PATH=$XPATH_RESULT
XPATH=${PACKAGE_CHECKSUM_XPATH_TPL/_PKG_/$pkg}
XPATH=${XPATH/_ARCH_/$ARCH}
Xpath_get $XPATH $TMP_PRIMARY
CHECKSUM=$XPATH_RESULT
PKG_URL=$TARGET_URL/$PKG_PATH
PKG_FILE=$(basename $PKG_PATH)
PKG_PATH=$TMPDIR/$PKG_FILE
Debug "Download $PKG_URL to $PKG_PATH"
Fetch $PKG_URL $PKG_PATH true
echo "$CHECKSUM $PKG_PATH" | sha256sum -c - > /dev/null
if [ $? -ne 0 ]; then
Error "Fail to fetch $PKG_URL to $PKG_PATH"
Debug "Checksum = $CHECKSUM"
exit 1
fi
done
}
Inform "Initialize arm base"
fetch_tizen_pkgs_init standard base
Inform "fetch common packages"
fetch_tizen_pkgs armv7l gcc gcc-devel-static glibc glibc-devel libicu libicu-devel libatomic linux-glibc-devel keyutils keyutils-devel libkeyutils
Inform "fetch coreclr packages"
fetch_tizen_pkgs armv7l lldb lldb-devel libgcc libstdc++ libstdc++-devel libunwind libunwind-devel lttng-ust-devel lttng-ust userspace-rcu-devel userspace-rcu
Inform "fetch corefx packages"
fetch_tizen_pkgs armv7l libcom_err libcom_err-devel zlib zlib-devel libopenssl11 libopenssl1.1-devel krb5 krb5-devel
Inform "Initialize standard unified"
fetch_tizen_pkgs_init standard unified
Inform "fetch corefx packages"
fetch_tizen_pkgs armv7l gssdp gssdp-devel tizen-release

50
eng/common/cross/armel/tizen/tizen-dotnet.ks

@ -1,50 +0,0 @@
lang en_US.UTF-8
keyboard us
timezone --utc Asia/Seoul
part / --fstype="ext4" --size=3500 --ondisk=mmcblk0 --label rootfs --fsoptions=defaults,noatime
rootpw tizen
desktop --autologinuser=root
user --name root --groups audio,video --password 'tizen'
repo --name=standard --baseurl=http://download.tizen.org/releases/milestone/tizen/unified/latest/repos/standard/packages/ --ssl_verify=no
repo --name=base --baseurl=http://download.tizen.org/releases/milestone/tizen/base/latest/repos/standard/packages/ --ssl_verify=no
%packages
tar
gzip
sed
grep
gawk
perl
binutils
findutils
util-linux
lttng-ust
userspace-rcu
procps-ng
tzdata
ca-certificates
### Core FX
libicu
libunwind
iputils
zlib
krb5
libcurl
libopenssl
%end
%post
### Update /tmp privilege
chmod 777 /tmp
####################################
%end

4
eng/common/cross/build-android-rootfs.sh

@ -107,12 +107,12 @@ __AndroidPackages+=" liblzma"
__AndroidPackages+=" krb5" __AndroidPackages+=" krb5"
__AndroidPackages+=" openssl" __AndroidPackages+=" openssl"
for path in $(wget -qO- http://termux.net/dists/stable/main/binary-$__AndroidArch/Packages |\ for path in $(wget -qO- https://packages.termux.dev/termux-main-21/dists/stable/main/binary-$__AndroidArch/Packages |\
grep -A15 "Package: \(${__AndroidPackages// /\\|}\)" | grep -v "static\|tool" | grep Filename); do grep -A15 "Package: \(${__AndroidPackages// /\\|}\)" | grep -v "static\|tool" | grep Filename); do
if [[ "$path" != "Filename:" ]]; then if [[ "$path" != "Filename:" ]]; then
echo "Working on: $path" echo "Working on: $path"
wget -qO- http://termux.net/$path | dpkg -x - "$__TmpDir" wget -qO- https://packages.termux.dev/termux-main-21/$path | dpkg -x - "$__TmpDir"
fi fi
done done

335
eng/common/cross/build-rootfs.sh

@ -5,22 +5,26 @@ set -e
usage() usage()
{ {
echo "Usage: $0 [BuildArch] [CodeName] [lldbx.y] [llvmx[.y]] [--skipunmount] --rootfsdir <directory>]" echo "Usage: $0 [BuildArch] [CodeName] [lldbx.y] [llvmx[.y]] [--skipunmount] --rootfsdir <directory>]"
echo "BuildArch can be: arm(default), armel, arm64, x86" echo "BuildArch can be: arm(default), arm64, armel, armv6, ppc64le, riscv64, s390x, x64, x86"
echo "CodeName - optional, Code name for Linux, can be: xenial(default), zesty, bionic, alpine, alpine3.13 or alpine3.14. If BuildArch is armel, LinuxCodeName is jessie(default) or tizen." echo "CodeName - optional, Code name for Linux, can be: xenial(default), zesty, bionic, alpine, alpine3.13 or alpine3.14. If BuildArch is armel, LinuxCodeName is jessie(default) or tizen."
echo " for FreeBSD can be: freebsd12, freebsd13" echo " for FreeBSD can be: freebsd12, freebsd13"
echo " for illumos can be: illumos." echo " for illumos can be: illumos"
echo " for Haiku can be: haiku."
echo "lldbx.y - optional, LLDB version, can be: lldb3.9(default), lldb4.0, lldb5.0, lldb6.0 no-lldb. Ignored for alpine and FreeBSD" echo "lldbx.y - optional, LLDB version, can be: lldb3.9(default), lldb4.0, lldb5.0, lldb6.0 no-lldb. Ignored for alpine and FreeBSD"
echo "llvmx[.y] - optional, LLVM version for LLVM related packages." echo "llvmx[.y] - optional, LLVM version for LLVM related packages."
echo "--skipunmount - optional, will skip the unmount of rootfs folder." echo "--skipunmount - optional, will skip the unmount of rootfs folder."
echo "--use-mirror - optional, use mirror URL to fetch resources, when available." echo "--use-mirror - optional, use mirror URL to fetch resources, when available."
echo "--jobs N - optional, restrict to N jobs."
exit 1 exit 1
} }
__CodeName=xenial __CodeName=xenial
__CrossDir=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd ) __CrossDir=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )
__InitialDir=$PWD
__BuildArch=arm __BuildArch=arm
__AlpineArch=armv7 __AlpineArch=armv7
__FreeBSDArch=arm
__FreeBSDMachineArch=armv7
__IllumosArch=arm7
__QEMUArch=arm __QEMUArch=arm
__UbuntuArch=armhf __UbuntuArch=armhf
__UbuntuRepo="http://ports.ubuntu.com/" __UbuntuRepo="http://ports.ubuntu.com/"
@ -40,7 +44,7 @@ __AlpinePackages+=" libedit"
# symlinks fixer # symlinks fixer
__UbuntuPackages+=" symlinks" __UbuntuPackages+=" symlinks"
# CoreCLR and CoreFX dependencies # runtime dependencies
__UbuntuPackages+=" libicu-dev" __UbuntuPackages+=" libicu-dev"
__UbuntuPackages+=" liblttng-ust-dev" __UbuntuPackages+=" liblttng-ust-dev"
__UbuntuPackages+=" libunwind8-dev" __UbuntuPackages+=" libunwind8-dev"
@ -51,7 +55,7 @@ __AlpinePackages+=" libunwind-dev"
__AlpinePackages+=" lttng-ust-dev" __AlpinePackages+=" lttng-ust-dev"
__AlpinePackages+=" compiler-rt-static" __AlpinePackages+=" compiler-rt-static"
# CoreFX dependencies # runtime libraries' dependencies
__UbuntuPackages+=" libcurl4-openssl-dev" __UbuntuPackages+=" libcurl4-openssl-dev"
__UbuntuPackages+=" libkrb5-dev" __UbuntuPackages+=" libkrb5-dev"
__UbuntuPackages+=" libssl-dev" __UbuntuPackages+=" libssl-dev"
@ -72,26 +76,38 @@ __FreeBSDPackages+=" openssl"
__FreeBSDPackages+=" krb5" __FreeBSDPackages+=" krb5"
__FreeBSDPackages+=" terminfo-db" __FreeBSDPackages+=" terminfo-db"
__IllumosPackages="icu-64.2nb2" __IllumosPackages="icu"
__IllumosPackages+=" mit-krb5-1.16.2nb4" __IllumosPackages+=" mit-krb5"
__IllumosPackages+=" openssl-1.1.1e" __IllumosPackages+=" openssl"
__IllumosPackages+=" zlib-1.2.11" __IllumosPackages+=" zlib"
__HaikuPackages="gmp"
__HaikuPackages+=" gmp_devel"
__HaikuPackages+=" krb5"
__HaikuPackages+=" krb5_devel"
__HaikuPackages+=" libiconv"
__HaikuPackages+=" libiconv_devel"
__HaikuPackages+=" llvm12_libunwind"
__HaikuPackages+=" llvm12_libunwind_devel"
__HaikuPackages+=" mpfr"
__HaikuPackages+=" mpfr_devel"
# ML.NET dependencies # ML.NET dependencies
__UbuntuPackages+=" libomp5" __UbuntuPackages+=" libomp5"
__UbuntuPackages+=" libomp-dev" __UbuntuPackages+=" libomp-dev"
__Keyring=
__UseMirror=0 __UseMirror=0
__UnprocessedBuildArgs= __UnprocessedBuildArgs=
while :; do while :; do
if [ $# -le 0 ]; then if [[ "$#" -le 0 ]]; then
break break
fi fi
lowerI="$(echo $1 | tr "[:upper:]" "[:lower:]")" lowerI="$(echo "$1" | tr "[:upper:]" "[:lower:]")"
case $lowerI in case $lowerI in
-?|-h|--help) -\?|-h|--help)
usage usage
exit 1 exit 1
;; ;;
@ -101,20 +117,13 @@ while :; do
__AlpineArch=armv7 __AlpineArch=armv7
__QEMUArch=arm __QEMUArch=arm
;; ;;
armv6)
__BuildArch=armv6
__UbuntuArch=armhf
__QEMUArch=arm
__UbuntuRepo="http://raspbian.raspberrypi.org/raspbian/"
__CodeName=buster
__LLDB_Package="liblldb-6.0-dev"
__Keyring="/usr/share/keyrings/raspbian-archive-keyring.gpg"
;;
arm64) arm64)
__BuildArch=arm64 __BuildArch=arm64
__UbuntuArch=arm64 __UbuntuArch=arm64
__AlpineArch=aarch64 __AlpineArch=aarch64
__QEMUArch=aarch64 __QEMUArch=aarch64
__FreeBSDArch=arm64
__FreeBSDMachineArch=aarch64
;; ;;
armel) armel)
__BuildArch=armel __BuildArch=armel
@ -122,6 +131,18 @@ while :; do
__UbuntuRepo="http://ftp.debian.org/debian/" __UbuntuRepo="http://ftp.debian.org/debian/"
__CodeName=jessie __CodeName=jessie
;; ;;
armv6)
__BuildArch=armv6
__UbuntuArch=armhf
__QEMUArch=arm
__UbuntuRepo="http://raspbian.raspberrypi.org/raspbian/"
__CodeName=buster
__LLDB_Package="liblldb-6.0-dev"
if [[ -e "/usr/share/keyrings/raspbian-archive-keyring.gpg" ]]; then
__Keyring="--keyring /usr/share/keyrings/raspbian-archive-keyring.gpg"
fi
;;
ppc64le) ppc64le)
__BuildArch=ppc64le __BuildArch=ppc64le
__UbuntuArch=ppc64el __UbuntuArch=ppc64el
@ -131,6 +152,18 @@ while :; do
__UbuntuPackages=$(echo ${__UbuntuPackages} | sed 's/ libomp5//') __UbuntuPackages=$(echo ${__UbuntuPackages} | sed 's/ libomp5//')
unset __LLDB_Package unset __LLDB_Package
;; ;;
riscv64)
__BuildArch=riscv64
__UbuntuArch=riscv64
__UbuntuRepo="http://deb.debian.org/debian-ports"
__CodeName=sid
__UbuntuPackages=$(echo ${__UbuntuPackages} | sed 's/ libunwind8-dev//')
unset __LLDB_Package
if [[ -e "/usr/share/keyrings/debian-ports-archive-keyring.gpg" ]]; then
__Keyring="--keyring /usr/share/keyrings/debian-ports-archive-keyring.gpg --include=debian-ports-archive-keyring"
fi
;;
s390x) s390x)
__BuildArch=s390x __BuildArch=s390x
__UbuntuArch=s390x __UbuntuArch=s390x
@ -140,56 +173,69 @@ while :; do
__UbuntuPackages=$(echo ${__UbuntuPackages} | sed 's/ libomp5//') __UbuntuPackages=$(echo ${__UbuntuPackages} | sed 's/ libomp5//')
unset __LLDB_Package unset __LLDB_Package
;; ;;
x64)
__BuildArch=x64
__UbuntuArch=amd64
__FreeBSDArch=amd64
__FreeBSDMachineArch=amd64
__illumosArch=x86_64
__UbuntuRepo=
;;
x86) x86)
__BuildArch=x86 __BuildArch=x86
__UbuntuArch=i386 __UbuntuArch=i386
__UbuntuRepo="http://archive.ubuntu.com/ubuntu/" __UbuntuRepo="http://archive.ubuntu.com/ubuntu/"
;; ;;
lldb3.6) lldb*)
__LLDB_Package="lldb-3.6-dev" version="${lowerI/lldb/}"
;; parts=(${version//./ })
lldb3.8)
__LLDB_Package="lldb-3.8-dev" # for versions > 6.0, lldb has dropped the minor version
;; if [[ "${parts[0]}" -gt 6 ]]; then
lldb3.9) version="${parts[0]}"
__LLDB_Package="liblldb-3.9-dev" fi
;;
lldb4.0) __LLDB_Package="liblldb-${version}-dev"
__LLDB_Package="liblldb-4.0-dev"
;;
lldb5.0)
__LLDB_Package="liblldb-5.0-dev"
;;
lldb6.0)
__LLDB_Package="liblldb-6.0-dev"
;; ;;
no-lldb) no-lldb)
unset __LLDB_Package unset __LLDB_Package
;; ;;
llvm*) llvm*)
version="$(echo "$lowerI" | tr -d '[:alpha:]-=')" version="${lowerI/llvm/}"
parts=(${version//./ }) parts=(${version//./ })
__LLVM_MajorVersion="${parts[0]}" __LLVM_MajorVersion="${parts[0]}"
__LLVM_MinorVersion="${parts[1]}" __LLVM_MinorVersion="${parts[1]}"
# for versions > 6.0, llvm has dropped the minor version
if [[ -z "$__LLVM_MinorVersion" && "$__LLVM_MajorVersion" -le 6 ]]; then if [[ -z "$__LLVM_MinorVersion" && "$__LLVM_MajorVersion" -le 6 ]]; then
__LLVM_MinorVersion=0; __LLVM_MinorVersion=0;
fi fi
;; ;;
xenial) # Ubuntu 16.04 xenial) # Ubuntu 16.04
if [ "$__CodeName" != "jessie" ]; then if [[ "$__CodeName" != "jessie" ]]; then
__CodeName=xenial __CodeName=xenial
fi fi
;; ;;
zesty) # Ubuntu 17.04 zesty) # Ubuntu 17.04
if [ "$__CodeName" != "jessie" ]; then if [[ "$__CodeName" != "jessie" ]]; then
__CodeName=zesty __CodeName=zesty
fi fi
;; ;;
bionic) # Ubuntu 18.04 bionic) # Ubuntu 18.04
if [ "$__CodeName" != "jessie" ]; then if [[ "$__CodeName" != "jessie" ]]; then
__CodeName=bionic __CodeName=bionic
fi fi
;; ;;
focal) # Ubuntu 20.04
if [[ "$__CodeName" != "jessie" ]]; then
__CodeName=focal
fi
;;
jammy) # Ubuntu 22.04
if [[ "$__CodeName" != "jessie" ]]; then
__CodeName=jammy
fi
;;
jessie) # Debian 8 jessie) # Debian 8
__CodeName=jessie __CodeName=jessie
__UbuntuRepo="http://ftp.debian.org/debian/" __UbuntuRepo="http://ftp.debian.org/debian/"
@ -205,11 +251,6 @@ while :; do
__LLDB_Package="liblldb-6.0-dev" __LLDB_Package="liblldb-6.0-dev"
;; ;;
tizen) tizen)
if [ "$__BuildArch" != "arm" ] && [ "$__BuildArch" != "armel" ] && [ "$__BuildArch" != "arm64" ] && [ "$__BuildArch" != "x86" ] ; then
echo "Tizen is available only for arm, armel, arm64 and x86."
usage;
exit 1;
fi
__CodeName= __CodeName=
__UbuntuRepo= __UbuntuRepo=
__Tizen=tizen __Tizen=tizen
@ -228,18 +269,20 @@ while :; do
;; ;;
freebsd12) freebsd12)
__CodeName=freebsd __CodeName=freebsd
__BuildArch=x64
__SkipUnmount=1 __SkipUnmount=1
;; ;;
freebsd13) freebsd13)
__CodeName=freebsd __CodeName=freebsd
__FreeBSDBase="13.0-RELEASE" __FreeBSDBase="13.0-RELEASE"
__FreeBSDABI="13" __FreeBSDABI="13"
__BuildArch=x64
__SkipUnmount=1 __SkipUnmount=1
;; ;;
illumos) illumos)
__CodeName=illumos __CodeName=illumos
__SkipUnmount=1
;;
haiku)
__CodeName=haiku
__BuildArch=x64 __BuildArch=x64
__SkipUnmount=1 __SkipUnmount=1
;; ;;
@ -248,11 +291,15 @@ while :; do
;; ;;
--rootfsdir|-rootfsdir) --rootfsdir|-rootfsdir)
shift shift
__RootfsDir=$1 __RootfsDir="$1"
;; ;;
--use-mirror) --use-mirror)
__UseMirror=1 __UseMirror=1
;; ;;
--use-jobs)
shift
MAXJOBS=$1
;;
*) *)
__UnprocessedBuildArgs="$__UnprocessedBuildArgs $1" __UnprocessedBuildArgs="$__UnprocessedBuildArgs $1"
;; ;;
@ -261,81 +308,76 @@ while :; do
shift shift
done done
if [ -e "$__Keyring" ]; then if [[ "$__BuildArch" == "armel" ]]; then
__Keyring="--keyring=$__Keyring"
else
__Keyring=""
fi
if [ "$__BuildArch" == "armel" ]; then
__LLDB_Package="lldb-3.5-dev" __LLDB_Package="lldb-3.5-dev"
fi fi
__UbuntuPackages+=" ${__LLDB_Package:-}" __UbuntuPackages+=" ${__LLDB_Package:-}"
if [ ! -z "$__LLVM_MajorVersion" ]; then if [[ -n "$__LLVM_MajorVersion" ]]; then
__UbuntuPackages+=" libclang-common-${__LLVM_MajorVersion}${__LLVM_MinorVersion:+.$__LLVM_MinorVersion}-dev" __UbuntuPackages+=" libclang-common-${__LLVM_MajorVersion}${__LLVM_MinorVersion:+.$__LLVM_MinorVersion}-dev"
fi fi
if [ -z "$__RootfsDir" ] && [ ! -z "$ROOTFS_DIR" ]; then if [[ -z "$__RootfsDir" && -n "$ROOTFS_DIR" ]]; then
__RootfsDir=$ROOTFS_DIR __RootfsDir="$ROOTFS_DIR"
fi fi
if [ -z "$__RootfsDir" ]; then if [[ -z "$__RootfsDir" ]]; then
__RootfsDir="$__CrossDir/../../../.tools/rootfs/$__BuildArch" __RootfsDir="$__CrossDir/../../../.tools/rootfs/$__BuildArch"
fi fi
if [ -d "$__RootfsDir" ]; then if [[ -d "$__RootfsDir" ]]; then
if [ $__SkipUnmount == 0 ]; then if [[ "$__SkipUnmount" == "0" ]]; then
umount $__RootfsDir/* || true umount "$__RootfsDir"/* || true
fi fi
rm -rf $__RootfsDir rm -rf "$__RootfsDir"
fi fi
mkdir -p $__RootfsDir mkdir -p "$__RootfsDir"
__RootfsDir="$( cd "$__RootfsDir" && pwd )" __RootfsDir="$( cd "$__RootfsDir" && pwd )"
if [[ "$__CodeName" == "alpine" ]]; then if [[ "$__CodeName" == "alpine" ]]; then
__ApkToolsVersion=2.9.1 __ApkToolsVersion=2.9.1
__ApkToolsDir=$(mktemp -d) __ApkToolsDir="$(mktemp -d)"
wget https://github.com/alpinelinux/apk-tools/releases/download/v$__ApkToolsVersion/apk-tools-$__ApkToolsVersion-x86_64-linux.tar.gz -P $__ApkToolsDir wget "https://github.com/alpinelinux/apk-tools/releases/download/v$__ApkToolsVersion/apk-tools-$__ApkToolsVersion-x86_64-linux.tar.gz" -P "$__ApkToolsDir"
tar -xf $__ApkToolsDir/apk-tools-$__ApkToolsVersion-x86_64-linux.tar.gz -C $__ApkToolsDir tar -xf "$__ApkToolsDir/apk-tools-$__ApkToolsVersion-x86_64-linux.tar.gz" -C "$__ApkToolsDir"
mkdir -p $__RootfsDir/usr/bin mkdir -p "$__RootfsDir"/usr/bin
cp -v /usr/bin/qemu-$__QEMUArch-static $__RootfsDir/usr/bin cp -v "/usr/bin/qemu-$__QEMUArch-static" "$__RootfsDir/usr/bin"
$__ApkToolsDir/apk-tools-$__ApkToolsVersion/apk \ "$__ApkToolsDir/apk-tools-$__ApkToolsVersion/apk" \
-X http://dl-cdn.alpinelinux.org/alpine/v$__AlpineVersion/main \ -X "http://dl-cdn.alpinelinux.org/alpine/v$__AlpineVersion/main" \
-X http://dl-cdn.alpinelinux.org/alpine/v$__AlpineVersion/community \ -X "http://dl-cdn.alpinelinux.org/alpine/v$__AlpineVersion/community" \
-U --allow-untrusted --root $__RootfsDir --arch $__AlpineArch --initdb \ -U --allow-untrusted --root "$__RootfsDir" --arch "$__AlpineArch" --initdb \
add $__AlpinePackages add $__AlpinePackages
rm -r $__ApkToolsDir rm -r "$__ApkToolsDir"
elif [[ "$__CodeName" == "freebsd" ]]; then elif [[ "$__CodeName" == "freebsd" ]]; then
mkdir -p $__RootfsDir/usr/local/etc mkdir -p "$__RootfsDir"/usr/local/etc
JOBS="$(getconf _NPROCESSORS_ONLN)" JOBS=${MAXJOBS:="$(getconf _NPROCESSORS_ONLN)"}
wget -O - https://download.freebsd.org/ftp/releases/amd64/${__FreeBSDBase}/base.txz | tar -C $__RootfsDir -Jxf - ./lib ./usr/lib ./usr/libdata ./usr/include ./usr/share/keys ./etc ./bin/freebsd-version wget -O - "https://download.freebsd.org/ftp/releases/${__FreeBSDArch}/${__FreeBSDMachineArch}/${__FreeBSDBase}/base.txz" | tar -C "$__RootfsDir" -Jxf - ./lib ./usr/lib ./usr/libdata ./usr/include ./usr/share/keys ./etc ./bin/freebsd-version
echo "ABI = \"FreeBSD:${__FreeBSDABI}:amd64\"; FINGERPRINTS = \"${__RootfsDir}/usr/share/keys\"; REPOS_DIR = [\"${__RootfsDir}/etc/pkg\"]; REPO_AUTOUPDATE = NO; RUN_SCRIPTS = NO;" > ${__RootfsDir}/usr/local/etc/pkg.conf echo "ABI = \"FreeBSD:${__FreeBSDABI}:${__FreeBSDMachineArch}\"; FINGERPRINTS = \"${__RootfsDir}/usr/share/keys\"; REPOS_DIR = [\"${__RootfsDir}/etc/pkg\"]; REPO_AUTOUPDATE = NO; RUN_SCRIPTS = NO;" > "${__RootfsDir}"/usr/local/etc/pkg.conf
echo "FreeBSD: { url: "pkg+http://pkg.FreeBSD.org/\${ABI}/quarterly", mirror_type: \"srv\", signature_type: \"fingerprints\", fingerprints: \"${__RootfsDir}/usr/share/keys/pkg\", enabled: yes }" > ${__RootfsDir}/etc/pkg/FreeBSD.conf echo "FreeBSD: { url: \"pkg+http://pkg.FreeBSD.org/\${ABI}/quarterly\", mirror_type: \"srv\", signature_type: \"fingerprints\", fingerprints: \"${__RootfsDir}/usr/share/keys/pkg\", enabled: yes }" > "${__RootfsDir}"/etc/pkg/FreeBSD.conf
mkdir -p $__RootfsDir/tmp mkdir -p "$__RootfsDir"/tmp
# get and build package manager # get and build package manager
wget -O - https://github.com/freebsd/pkg/archive/${__FreeBSDPkg}.tar.gz | tar -C $__RootfsDir/tmp -zxf - wget -O - "https://github.com/freebsd/pkg/archive/${__FreeBSDPkg}.tar.gz" | tar -C "$__RootfsDir"/tmp -zxf -
cd $__RootfsDir/tmp/pkg-${__FreeBSDPkg} cd "$__RootfsDir/tmp/pkg-${__FreeBSDPkg}"
# needed for install to succeed # needed for install to succeed
mkdir -p $__RootfsDir/host/etc mkdir -p "$__RootfsDir"/host/etc
./autogen.sh && ./configure --prefix=$__RootfsDir/host && make -j "$JOBS" && make install ./autogen.sh && ./configure --prefix="$__RootfsDir"/host && make -j "$JOBS" && make install
rm -rf $__RootfsDir/tmp/pkg-${__FreeBSDPkg} rm -rf "$__RootfsDir/tmp/pkg-${__FreeBSDPkg}"
# install packages we need. # install packages we need.
INSTALL_AS_USER=$(whoami) $__RootfsDir/host/sbin/pkg -r $__RootfsDir -C $__RootfsDir/usr/local/etc/pkg.conf update INSTALL_AS_USER=$(whoami) "$__RootfsDir"/host/sbin/pkg -r "$__RootfsDir" -C "$__RootfsDir"/usr/local/etc/pkg.conf update
INSTALL_AS_USER=$(whoami) $__RootfsDir/host/sbin/pkg -r $__RootfsDir -C $__RootfsDir/usr/local/etc/pkg.conf install --yes $__FreeBSDPackages INSTALL_AS_USER=$(whoami) "$__RootfsDir"/host/sbin/pkg -r "$__RootfsDir" -C "$__RootfsDir"/usr/local/etc/pkg.conf install --yes $__FreeBSDPackages
elif [[ "$__CodeName" == "illumos" ]]; then elif [[ "$__CodeName" == "illumos" ]]; then
mkdir "$__RootfsDir/tmp" mkdir "$__RootfsDir/tmp"
pushd "$__RootfsDir/tmp" pushd "$__RootfsDir/tmp"
JOBS="$(getconf _NPROCESSORS_ONLN)" JOBS=${MAXJOBS:="$(getconf _NPROCESSORS_ONLN)"}
echo "Downloading sysroot." echo "Downloading sysroot."
wget -O - https://github.com/illumos/sysroot/releases/download/20181213-de6af22ae73b-v1/illumos-sysroot-i386-20181213-de6af22ae73b-v1.tar.gz | tar -C "$__RootfsDir" -xzf - wget -O - https://github.com/illumos/sysroot/releases/download/20181213-de6af22ae73b-v1/illumos-sysroot-i386-20181213-de6af22ae73b-v1.tar.gz | tar -C "$__RootfsDir" -xzf -
echo "Building binutils. Please wait.." echo "Building binutils. Please wait.."
wget -O - https://ftp.gnu.org/gnu/binutils/binutils-2.33.1.tar.bz2 | tar -xjf - wget -O - https://ftp.gnu.org/gnu/binutils/binutils-2.33.1.tar.bz2 | tar -xjf -
mkdir build-binutils && cd build-binutils mkdir build-binutils && cd build-binutils
../binutils-2.33.1/configure --prefix="$__RootfsDir" --target="x86_64-sun-solaris2.10" --program-prefix="x86_64-illumos-" --with-sysroot="$__RootfsDir" ../binutils-2.33.1/configure --prefix="$__RootfsDir" --target="${__illumosArch}-sun-solaris2.10" --program-prefix="${__illumosArch}-illumos-" --with-sysroot="$__RootfsDir"
make -j "$JOBS" && make install && cd .. make -j "$JOBS" && make install && cd ..
echo "Building gcc. Please wait.." echo "Building gcc. Please wait.."
wget -O - https://ftp.gnu.org/gnu/gcc/gcc-8.4.0/gcc-8.4.0.tar.xz | tar -xJf - wget -O - https://ftp.gnu.org/gnu/gcc/gcc-8.4.0/gcc-8.4.0.tar.xz | tar -xJf -
@ -345,22 +387,27 @@ elif [[ "$__CodeName" == "illumos" ]]; then
CFLAGS_FOR_TARGET="-fPIC" CFLAGS_FOR_TARGET="-fPIC"
export CFLAGS CXXFLAGS CXXFLAGS_FOR_TARGET CFLAGS_FOR_TARGET export CFLAGS CXXFLAGS CXXFLAGS_FOR_TARGET CFLAGS_FOR_TARGET
mkdir build-gcc && cd build-gcc mkdir build-gcc && cd build-gcc
../gcc-8.4.0/configure --prefix="$__RootfsDir" --target="x86_64-sun-solaris2.10" --program-prefix="x86_64-illumos-" --with-sysroot="$__RootfsDir" --with-gnu-as \ ../gcc-8.4.0/configure --prefix="$__RootfsDir" --target="${__illumosArch}-sun-solaris2.10" --program-prefix="${__illumosArch}-illumos-" --with-sysroot="$__RootfsDir" --with-gnu-as \
--with-gnu-ld --disable-nls --disable-libgomp --disable-libquadmath --disable-libssp --disable-libvtv --disable-libcilkrts --disable-libada --disable-libsanitizer \ --with-gnu-ld --disable-nls --disable-libgomp --disable-libquadmath --disable-libssp --disable-libvtv --disable-libcilkrts --disable-libada --disable-libsanitizer \
--disable-libquadmath-support --disable-shared --enable-tls --disable-libquadmath-support --disable-shared --enable-tls
make -j "$JOBS" && make install && cd .. make -j "$JOBS" && make install && cd ..
BaseUrl=https://pkgsrc.joyent.com BaseUrl=https://pkgsrc.smartos.org
if [[ "$__UseMirror" == 1 ]]; then if [[ "$__UseMirror" == 1 ]]; then
BaseUrl=http://pkgsrc.smartos.skylime.net BaseUrl=https://pkgsrc.smartos.skylime.net
fi fi
BaseUrl="$BaseUrl"/packages/SmartOS/2020Q1/x86_64/All BaseUrl="$BaseUrl/packages/SmartOS/trunk/${__illumosArch}/All"
echo "Downloading manifest"
wget "$BaseUrl"
echo "Downloading dependencies." echo "Downloading dependencies."
read -ra array <<<"$__IllumosPackages" read -ra array <<<"$__IllumosPackages"
for package in "${array[@]}"; do for package in "${array[@]}"; do
echo "Installing $package..." echo "Installing '$package'"
# find last occurrence of package in listing and extract its name
package="$(sed -En '/.*href="('"$package"'-[0-9].*).tgz".*/h;$!d;g;s//\1/p' All)"
echo "Resolved name '$package'"
wget "$BaseUrl"/"$package".tgz wget "$BaseUrl"/"$package".tgz
ar -x "$package".tgz ar -x "$package".tgz
tar --skip-old-files -xzf "$package".tmp.tgz -C "$__RootfsDir" 2>/dev/null tar --skip-old-files -xzf "$package".tmp.tg* -C "$__RootfsDir" 2>/dev/null
done done
echo "Cleaning up temporary files." echo "Cleaning up temporary files."
popd popd
@ -371,26 +418,90 @@ elif [[ "$__CodeName" == "illumos" ]]; then
wget -P "$__RootfsDir"/usr/include/net https://raw.githubusercontent.com/illumos/illumos-gate/master/usr/src/uts/common/io/bpf/net/dlt.h wget -P "$__RootfsDir"/usr/include/net https://raw.githubusercontent.com/illumos/illumos-gate/master/usr/src/uts/common/io/bpf/net/dlt.h
wget -P "$__RootfsDir"/usr/include/netpacket https://raw.githubusercontent.com/illumos/illumos-gate/master/usr/src/uts/common/inet/sockmods/netpacket/packet.h wget -P "$__RootfsDir"/usr/include/netpacket https://raw.githubusercontent.com/illumos/illumos-gate/master/usr/src/uts/common/inet/sockmods/netpacket/packet.h
wget -P "$__RootfsDir"/usr/include/sys https://raw.githubusercontent.com/illumos/illumos-gate/master/usr/src/uts/common/sys/sdt.h wget -P "$__RootfsDir"/usr/include/sys https://raw.githubusercontent.com/illumos/illumos-gate/master/usr/src/uts/common/sys/sdt.h
elif [[ -n $__CodeName ]]; then elif [[ "$__CodeName" == "haiku" ]]; then
qemu-debootstrap $__Keyring --arch $__UbuntuArch $__CodeName $__RootfsDir $__UbuntuRepo JOBS=${MAXJOBS:="$(getconf _NPROCESSORS_ONLN)"}
cp $__CrossDir/$__BuildArch/sources.list.$__CodeName $__RootfsDir/etc/apt/sources.list
chroot $__RootfsDir apt-get update echo "Building Haiku sysroot for x86_64"
chroot $__RootfsDir apt-get -f -y install mkdir -p "$__RootfsDir/tmp"
chroot $__RootfsDir apt-get -y install $__UbuntuPackages cd "$__RootfsDir/tmp"
chroot $__RootfsDir symlinks -cr /usr git clone -b hrev56235 https://review.haiku-os.org/haiku
chroot $__RootfsDir apt-get clean git clone -b btrev43195 https://review.haiku-os.org/buildtools
cd "$__RootfsDir/tmp/buildtools" && git checkout 7487388f5110021d400b9f3b88e1a7f310dc066d
if [ $__SkipUnmount == 0 ]; then
umount $__RootfsDir/* || true # Fetch some unmerged patches
cd "$__RootfsDir/tmp/haiku"
## Add development build profile (slimmer than nightly)
git fetch origin refs/changes/64/4164/1 && git -c commit.gpgsign=false cherry-pick FETCH_HEAD
# Build jam
cd "$__RootfsDir/tmp/buildtools/jam"
make
# Configure cross tools
echo "Building cross-compiler"
mkdir -p "$__RootfsDir/generated"
cd "$__RootfsDir/generated"
"$__RootfsDir/tmp/haiku/configure" -j"$JOBS" --sysroot "$__RootfsDir" --cross-tools-source "$__RootfsDir/tmp/buildtools" --build-cross-tools x86_64
# Build Haiku packages
echo "Building Haiku"
echo 'HAIKU_BUILD_PROFILE = "development-raw" ;' > UserProfileConfig
"$__RootfsDir/tmp/buildtools/jam/jam0" -j"$JOBS" -q '<build>package' '<repository>Haiku'
BaseUrl="https://depot.haiku-os.org/__api/v2/pkg/get-pkg"
# Download additional packages
echo "Downloading additional required packages"
read -ra array <<<"$__HaikuPackages"
for package in "${array[@]}"; do
echo "Downloading $package..."
# API documented here: https://github.com/haiku/haikudepotserver/blob/master/haikudepotserver-api2/src/main/resources/api2/pkg.yaml#L60
# The schema here: https://github.com/haiku/haikudepotserver/blob/master/haikudepotserver-api2/src/main/resources/api2/pkg.yaml#L598
hpkgDownloadUrl="$(wget -qO- --post-data='{"name":"'"$package"'","repositorySourceCode":"haikuports_x86_64","versionType":"LATEST","naturalLanguageCode":"en"}' \
--header='Content-Type:application/json' "$BaseUrl" | jq -r '.result.versions[].hpkgDownloadURL')"
wget -P "$__RootfsDir/generated/download" "$hpkgDownloadUrl"
done
# Setup the sysroot
echo "Setting up sysroot and extracting needed packages"
mkdir -p "$__RootfsDir/boot/system"
for file in "$__RootfsDir/generated/objects/haiku/x86_64/packaging/packages/"*.hpkg; do
"$__RootfsDir/generated/objects/linux/x86_64/release/tools/package/package" extract -C "$__RootfsDir/boot/system" "$file"
done
for file in "$__RootfsDir/generated/download/"*.hpkg; do
"$__RootfsDir/generated/objects/linux/x86_64/release/tools/package/package" extract -C "$__RootfsDir/boot/system" "$file"
done
# Cleaning up temporary files
echo "Cleaning up temporary files"
rm -rf "$__RootfsDir/tmp"
for name in "$__RootfsDir/generated/"*; do
if [[ "$name" =~ "cross-tools-" ]]; then
: # Keep the cross-compiler
else
rm -rf "$name"
fi
done
elif [[ -n "$__CodeName" ]]; then
qemu-debootstrap $__Keyring --arch "$__UbuntuArch" "$__CodeName" "$__RootfsDir" "$__UbuntuRepo"
cp "$__CrossDir/$__BuildArch/sources.list.$__CodeName" "$__RootfsDir/etc/apt/sources.list"
chroot "$__RootfsDir" apt-get update
chroot "$__RootfsDir" apt-get -f -y install
chroot "$__RootfsDir" apt-get -y install $__UbuntuPackages
chroot "$__RootfsDir" symlinks -cr /usr
chroot "$__RootfsDir" apt-get clean
if [[ "$__SkipUnmount" == "0" ]]; then
umount "$__RootfsDir"/* || true
fi fi
if [[ "$__BuildArch" == "armel" && "$__CodeName" == "jessie" ]]; then if [[ "$__BuildArch" == "armel" && "$__CodeName" == "jessie" ]]; then
pushd $__RootfsDir pushd "$__RootfsDir"
patch -p1 < $__CrossDir/$__BuildArch/armel.jessie.patch patch -p1 < "$__CrossDir/$__BuildArch/armel.jessie.patch"
popd popd
fi fi
elif [[ "$__Tizen" == "tizen" ]]; then elif [[ "$__Tizen" == "tizen" ]]; then
ROOTFS_DIR=$__RootfsDir $__CrossDir/$__BuildArch/tizen-build-rootfs.sh ROOTFS_DIR="$__RootfsDir" "$__CrossDir/tizen-build-rootfs.sh" "$__BuildArch"
else else
echo "Unsupported target platform." echo "Unsupported target platform."
usage; usage;

1
eng/common/cross/riscv64/sources.list.sid

@ -0,0 +1 @@
deb http://deb.debian.org/debian-ports sid main

34
eng/common/cross/armel/tizen-build-rootfs.sh → eng/common/cross/tizen-build-rootfs.sh

@ -1,8 +1,34 @@
#!/usr/bin/env bash #!/usr/bin/env bash
set -e set -e
__ARM_SOFTFP_CrossDir=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd ) ARCH=$1
__TIZEN_CROSSDIR="$__ARM_SOFTFP_CrossDir/tizen" LINK_ARCH=$ARCH
case "$ARCH" in
arm)
TIZEN_ARCH="armv7hl"
;;
armel)
TIZEN_ARCH="armv7l"
LINK_ARCH="arm"
;;
arm64)
TIZEN_ARCH="aarch64"
;;
x86)
TIZEN_ARCH="i686"
;;
x64)
TIZEN_ARCH="x86_64"
LINK_ARCH="x86"
;;
*)
echo "Unsupported architecture for tizen: $ARCH"
exit 1
esac
__CrossDir=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )
__TIZEN_CROSSDIR="$__CrossDir/${ARCH}/tizen"
if [[ -z "$ROOTFS_DIR" ]]; then if [[ -z "$ROOTFS_DIR" ]]; then
echo "ROOTFS_DIR is not defined." echo "ROOTFS_DIR is not defined."
@ -14,7 +40,7 @@ mkdir -p $TIZEN_TMP_DIR
# Download files # Download files
echo ">>Start downloading files" echo ">>Start downloading files"
VERBOSE=1 $__ARM_SOFTFP_CrossDir/tizen-fetch.sh $TIZEN_TMP_DIR VERBOSE=1 $__CrossDir/tizen-fetch.sh $TIZEN_TMP_DIR $TIZEN_ARCH
echo "<<Finish downloading files" echo "<<Finish downloading files"
echo ">>Start constructing Tizen rootfs" echo ">>Start constructing Tizen rootfs"
@ -30,6 +56,6 @@ rm -rf $TIZEN_TMP_DIR
# Configure Tizen rootfs # Configure Tizen rootfs
echo ">>Start configuring Tizen rootfs" echo ">>Start configuring Tizen rootfs"
ln -sfn asm-arm ./usr/include/asm ln -sfn asm-${LINK_ARCH} ./usr/include/asm
patch -p1 < $__TIZEN_CROSSDIR/tizen.patch patch -p1 < $__TIZEN_CROSSDIR/tizen.patch
echo "<<Finish configuring Tizen rootfs" echo "<<Finish configuring Tizen rootfs"

172
eng/common/cross/tizen-fetch.sh

@ -0,0 +1,172 @@
#!/usr/bin/env bash
set -e
if [[ -z "${VERBOSE// }" ]] || [ "$VERBOSE" -ne "$VERBOSE" ] 2>/dev/null; then
VERBOSE=0
fi
Log()
{
if [ $VERBOSE -ge $1 ]; then
echo ${@:2}
fi
}
Inform()
{
Log 1 -e "\x1B[0;34m$@\x1B[m"
}
Debug()
{
Log 2 -e "\x1B[0;32m$@\x1B[m"
}
Error()
{
>&2 Log 0 -e "\x1B[0;31m$@\x1B[m"
}
Fetch()
{
URL=$1
FILE=$2
PROGRESS=$3
if [ $VERBOSE -ge 1 ] && [ $PROGRESS ]; then
CURL_OPT="--progress-bar"
else
CURL_OPT="--silent"
fi
curl $CURL_OPT $URL > $FILE
}
hash curl 2> /dev/null || { Error "Require 'curl' Aborting."; exit 1; }
hash xmllint 2> /dev/null || { Error "Require 'xmllint' Aborting."; exit 1; }
hash sha256sum 2> /dev/null || { Error "Require 'sha256sum' Aborting."; exit 1; }
TMPDIR=$1
if [ ! -d $TMPDIR ]; then
TMPDIR=./tizen_tmp
Debug "Create temporary directory : $TMPDIR"
mkdir -p $TMPDIR
fi
TIZEN_ARCH=$2
TIZEN_URL=http://download.tizen.org/snapshots/TIZEN/Tizen
BUILD_XML=build.xml
REPOMD_XML=repomd.xml
PRIMARY_XML=primary.xml
TARGET_URL="http://__not_initialized"
Xpath_get()
{
XPATH_RESULT=''
XPATH=$1
XML_FILE=$2
RESULT=$(xmllint --xpath $XPATH $XML_FILE)
if [[ -z ${RESULT// } ]]; then
Error "Can not find target from $XML_FILE"
Debug "Xpath = $XPATH"
exit 1
fi
XPATH_RESULT=$RESULT
}
fetch_tizen_pkgs_init()
{
TARGET=$1
PROFILE=$2
Debug "Initialize TARGET=$TARGET, PROFILE=$PROFILE"
TMP_PKG_DIR=$TMPDIR/tizen_${PROFILE}_pkgs
if [ -d $TMP_PKG_DIR ]; then rm -rf $TMP_PKG_DIR; fi
mkdir -p $TMP_PKG_DIR
PKG_URL=$TIZEN_URL/$PROFILE/latest
BUILD_XML_URL=$PKG_URL/$BUILD_XML
TMP_BUILD=$TMP_PKG_DIR/$BUILD_XML
TMP_REPOMD=$TMP_PKG_DIR/$REPOMD_XML
TMP_PRIMARY=$TMP_PKG_DIR/$PRIMARY_XML
TMP_PRIMARYGZ=${TMP_PRIMARY}.gz
Fetch $BUILD_XML_URL $TMP_BUILD
Debug "fetch $BUILD_XML_URL to $TMP_BUILD"
TARGET_XPATH="//build/buildtargets/buildtarget[@name=\"$TARGET\"]/repo[@type=\"binary\"]/text()"
Xpath_get $TARGET_XPATH $TMP_BUILD
TARGET_PATH=$XPATH_RESULT
TARGET_URL=$PKG_URL/$TARGET_PATH
REPOMD_URL=$TARGET_URL/repodata/repomd.xml
PRIMARY_XPATH='string(//*[local-name()="data"][@type="primary"]/*[local-name()="location"]/@href)'
Fetch $REPOMD_URL $TMP_REPOMD
Debug "fetch $REPOMD_URL to $TMP_REPOMD"
Xpath_get $PRIMARY_XPATH $TMP_REPOMD
PRIMARY_XML_PATH=$XPATH_RESULT
PRIMARY_URL=$TARGET_URL/$PRIMARY_XML_PATH
Fetch $PRIMARY_URL $TMP_PRIMARYGZ
Debug "fetch $PRIMARY_URL to $TMP_PRIMARYGZ"
gunzip $TMP_PRIMARYGZ
Debug "unzip $TMP_PRIMARYGZ to $TMP_PRIMARY"
}
fetch_tizen_pkgs()
{
ARCH=$1
PACKAGE_XPATH_TPL='string(//*[local-name()="metadata"]/*[local-name()="package"][*[local-name()="name"][text()="_PKG_"]][*[local-name()="arch"][text()="_ARCH_"]]/*[local-name()="location"]/@href)'
PACKAGE_CHECKSUM_XPATH_TPL='string(//*[local-name()="metadata"]/*[local-name()="package"][*[local-name()="name"][text()="_PKG_"]][*[local-name()="arch"][text()="_ARCH_"]]/*[local-name()="checksum"]/text())'
for pkg in ${@:2}
do
Inform "Fetching... $pkg"
XPATH=${PACKAGE_XPATH_TPL/_PKG_/$pkg}
XPATH=${XPATH/_ARCH_/$ARCH}
Xpath_get $XPATH $TMP_PRIMARY
PKG_PATH=$XPATH_RESULT
XPATH=${PACKAGE_CHECKSUM_XPATH_TPL/_PKG_/$pkg}
XPATH=${XPATH/_ARCH_/$ARCH}
Xpath_get $XPATH $TMP_PRIMARY
CHECKSUM=$XPATH_RESULT
PKG_URL=$TARGET_URL/$PKG_PATH
PKG_FILE=$(basename $PKG_PATH)
PKG_PATH=$TMPDIR/$PKG_FILE
Debug "Download $PKG_URL to $PKG_PATH"
Fetch $PKG_URL $PKG_PATH true
echo "$CHECKSUM $PKG_PATH" | sha256sum -c - > /dev/null
if [ $? -ne 0 ]; then
Error "Fail to fetch $PKG_URL to $PKG_PATH"
Debug "Checksum = $CHECKSUM"
exit 1
fi
done
}
Inform "Initialize ${TIZEN_ARCH} base"
fetch_tizen_pkgs_init standard Tizen-Base
Inform "fetch common packages"
fetch_tizen_pkgs ${TIZEN_ARCH} gcc gcc-devel-static glibc glibc-devel libicu libicu-devel libatomic linux-glibc-devel keyutils keyutils-devel libkeyutils
Inform "fetch coreclr packages"
fetch_tizen_pkgs ${TIZEN_ARCH} lldb lldb-devel libgcc libstdc++ libstdc++-devel libunwind libunwind-devel lttng-ust-devel lttng-ust userspace-rcu-devel userspace-rcu
Inform "fetch corefx packages"
fetch_tizen_pkgs ${TIZEN_ARCH} libcom_err libcom_err-devel zlib zlib-devel libopenssl11 libopenssl1.1-devel krb5 krb5-devel
Inform "Initialize standard unified"
fetch_tizen_pkgs_init standard Tizen-Unified
Inform "fetch corefx packages"
fetch_tizen_pkgs ${TIZEN_ARCH} gssdp gssdp-devel tizen-release

111
eng/common/cross/toolchain.cmake

@ -1,5 +1,12 @@
set(CROSS_ROOTFS $ENV{ROOTFS_DIR}) set(CROSS_ROOTFS $ENV{ROOTFS_DIR})
# reset platform variables (e.g. cmake 3.25 sets LINUX=1)
unset(LINUX)
unset(FREEBSD)
unset(ILLUMOS)
unset(ANDROID)
unset(TIZEN)
set(TARGET_ARCH_NAME $ENV{TARGET_BUILD_ARCH}) set(TARGET_ARCH_NAME $ENV{TARGET_BUILD_ARCH})
if(EXISTS ${CROSS_ROOTFS}/bin/freebsd-version) if(EXISTS ${CROSS_ROOTFS}/bin/freebsd-version)
set(CMAKE_SYSTEM_NAME FreeBSD) set(CMAKE_SYSTEM_NAME FreeBSD)
@ -7,6 +14,8 @@ if(EXISTS ${CROSS_ROOTFS}/bin/freebsd-version)
elseif(EXISTS ${CROSS_ROOTFS}/usr/platform/i86pc) elseif(EXISTS ${CROSS_ROOTFS}/usr/platform/i86pc)
set(CMAKE_SYSTEM_NAME SunOS) set(CMAKE_SYSTEM_NAME SunOS)
set(ILLUMOS 1) set(ILLUMOS 1)
elseif(EXISTS ${CROSS_ROOTFS}/boot/system/develop/headers/config/HaikuConfig.h)
set(CMAKE_SYSTEM_NAME Haiku)
else() else()
set(CMAKE_SYSTEM_NAME Linux) set(CMAKE_SYSTEM_NAME Linux)
set(LINUX 1) set(LINUX 1)
@ -19,13 +28,7 @@ elseif(EXISTS ${CROSS_ROOTFS}/android_platform)
set(ANDROID 1) set(ANDROID 1)
endif() endif()
if(TARGET_ARCH_NAME STREQUAL "armel") if(TARGET_ARCH_NAME STREQUAL "arm")
set(CMAKE_SYSTEM_PROCESSOR armv7l)
set(TOOLCHAIN "arm-linux-gnueabi")
if(TIZEN)
set(TIZEN_TOOLCHAIN "armv7l-tizen-linux-gnueabi/9.2.0")
endif()
elseif(TARGET_ARCH_NAME STREQUAL "arm")
set(CMAKE_SYSTEM_PROCESSOR armv7l) set(CMAKE_SYSTEM_PROCESSOR armv7l)
if(EXISTS ${CROSS_ROOTFS}/usr/lib/gcc/armv7-alpine-linux-musleabihf) if(EXISTS ${CROSS_ROOTFS}/usr/lib/gcc/armv7-alpine-linux-musleabihf)
set(TOOLCHAIN "armv7-alpine-linux-musleabihf") set(TOOLCHAIN "armv7-alpine-linux-musleabihf")
@ -37,43 +40,62 @@ elseif(TARGET_ARCH_NAME STREQUAL "arm")
if(TIZEN) if(TIZEN)
set(TIZEN_TOOLCHAIN "armv7hl-tizen-linux-gnueabihf/9.2.0") set(TIZEN_TOOLCHAIN "armv7hl-tizen-linux-gnueabihf/9.2.0")
endif() endif()
elseif(TARGET_ARCH_NAME STREQUAL "armv6")
set(CMAKE_SYSTEM_PROCESSOR armv6l)
if(EXISTS ${CROSS_ROOTFS}/usr/lib/gcc/armv6-alpine-linux-musleabihf)
set(TOOLCHAIN "armv6-alpine-linux-musleabihf")
else()
set(TOOLCHAIN "arm-linux-gnueabihf")
endif()
elseif(TARGET_ARCH_NAME STREQUAL "arm64") elseif(TARGET_ARCH_NAME STREQUAL "arm64")
set(CMAKE_SYSTEM_PROCESSOR aarch64) set(CMAKE_SYSTEM_PROCESSOR aarch64)
if(EXISTS ${CROSS_ROOTFS}/usr/lib/gcc/aarch64-alpine-linux-musl) if(EXISTS ${CROSS_ROOTFS}/usr/lib/gcc/aarch64-alpine-linux-musl)
set(TOOLCHAIN "aarch64-alpine-linux-musl") set(TOOLCHAIN "aarch64-alpine-linux-musl")
else() elseif(LINUX)
set(TOOLCHAIN "aarch64-linux-gnu") set(TOOLCHAIN "aarch64-linux-gnu")
if(TIZEN)
set(TIZEN_TOOLCHAIN "aarch64-tizen-linux-gnu/9.2.0")
endif()
elseif(FREEBSD)
set(triple "aarch64-unknown-freebsd12")
endif() endif()
elseif(TARGET_ARCH_NAME STREQUAL "armel")
set(CMAKE_SYSTEM_PROCESSOR armv7l)
set(TOOLCHAIN "arm-linux-gnueabi")
if(TIZEN) if(TIZEN)
set(TIZEN_TOOLCHAIN "aarch64-tizen-linux-gnu/9.2.0") set(TIZEN_TOOLCHAIN "armv7l-tizen-linux-gnueabi/9.2.0")
endif()
elseif(TARGET_ARCH_NAME STREQUAL "armv6")
set(CMAKE_SYSTEM_PROCESSOR armv6l)
if(EXISTS ${CROSS_ROOTFS}/usr/lib/gcc/armv6-alpine-linux-musleabihf)
set(TOOLCHAIN "armv6-alpine-linux-musleabihf")
else()
set(TOOLCHAIN "arm-linux-gnueabihf")
endif() endif()
elseif(TARGET_ARCH_NAME STREQUAL "ppc64le") elseif(TARGET_ARCH_NAME STREQUAL "ppc64le")
set(CMAKE_SYSTEM_PROCESSOR ppc64le) set(CMAKE_SYSTEM_PROCESSOR ppc64le)
set(TOOLCHAIN "powerpc64le-linux-gnu") set(TOOLCHAIN "powerpc64le-linux-gnu")
elseif(TARGET_ARCH_NAME STREQUAL "riscv64")
set(CMAKE_SYSTEM_PROCESSOR riscv64)
set(TOOLCHAIN "riscv64-linux-gnu")
elseif(TARGET_ARCH_NAME STREQUAL "s390x") elseif(TARGET_ARCH_NAME STREQUAL "s390x")
set(CMAKE_SYSTEM_PROCESSOR s390x) set(CMAKE_SYSTEM_PROCESSOR s390x)
set(TOOLCHAIN "s390x-linux-gnu") set(TOOLCHAIN "s390x-linux-gnu")
elseif(TARGET_ARCH_NAME STREQUAL "x64")
set(CMAKE_SYSTEM_PROCESSOR x86_64)
if(LINUX)
set(TOOLCHAIN "x86_64-linux-gnu")
if(TIZEN)
set(TIZEN_TOOLCHAIN "x86_64-tizen-linux-gnu/9.2.0")
endif()
elseif(FREEBSD)
set(triple "x86_64-unknown-freebsd12")
elseif(ILLUMOS)
set(TOOLCHAIN "x86_64-illumos")
elseif(HAIKU)
set(TOOLCHAIN "x64_64-unknown-haiku")
endif()
elseif(TARGET_ARCH_NAME STREQUAL "x86") elseif(TARGET_ARCH_NAME STREQUAL "x86")
set(CMAKE_SYSTEM_PROCESSOR i686) set(CMAKE_SYSTEM_PROCESSOR i686)
set(TOOLCHAIN "i686-linux-gnu") set(TOOLCHAIN "i686-linux-gnu")
if(TIZEN) if(TIZEN)
set(TIZEN_TOOLCHAIN "i586-tizen-linux-gnu/9.2.0") set(TIZEN_TOOLCHAIN "i586-tizen-linux-gnu/9.2.0")
endif() endif()
elseif (FREEBSD)
set(CMAKE_SYSTEM_PROCESSOR "x86_64")
set(triple "x86_64-unknown-freebsd12")
elseif (ILLUMOS)
set(CMAKE_SYSTEM_PROCESSOR "x86_64")
set(TOOLCHAIN "x86_64-illumos")
else() else()
message(FATAL_ERROR "Arch is ${TARGET_ARCH_NAME}. Only armel, arm, armv6, arm64, ppc64le, s390x and x86 are supported!") message(FATAL_ERROR "Arch is ${TARGET_ARCH_NAME}. Only arm, arm64, armel, armv6, ppc64le, riscv64, s390x, x64 and x86 are supported!")
endif() endif()
if(DEFINED ENV{TOOLCHAIN}) if(DEFINED ENV{TOOLCHAIN})
@ -98,6 +120,10 @@ if(TIZEN)
include_directories(SYSTEM ${CROSS_ROOTFS}/usr/lib/gcc/${TIZEN_TOOLCHAIN}/include/c++/) include_directories(SYSTEM ${CROSS_ROOTFS}/usr/lib/gcc/${TIZEN_TOOLCHAIN}/include/c++/)
include_directories(SYSTEM ${CROSS_ROOTFS}/usr/lib/gcc/${TIZEN_TOOLCHAIN}/include/c++/i586-tizen-linux-gnu) include_directories(SYSTEM ${CROSS_ROOTFS}/usr/lib/gcc/${TIZEN_TOOLCHAIN}/include/c++/i586-tizen-linux-gnu)
endif() endif()
if(TARGET_ARCH_NAME STREQUAL "x64")
include_directories(SYSTEM ${CROSS_ROOTFS}/usr/lib64/gcc/${TIZEN_TOOLCHAIN}/include/c++/)
include_directories(SYSTEM ${CROSS_ROOTFS}/usr/lib64/gcc/${TIZEN_TOOLCHAIN}/include/c++/x86_64-tizen-linux-gnu)
endif()
endif() endif()
if(ANDROID) if(ANDROID)
@ -159,6 +185,41 @@ elseif(ILLUMOS)
set(CMAKE_C_STANDARD_LIBRARIES "${CMAKE_C_STANDARD_LIBRARIES} -lssp") set(CMAKE_C_STANDARD_LIBRARIES "${CMAKE_C_STANDARD_LIBRARIES} -lssp")
set(CMAKE_CXX_STANDARD_LIBRARIES "${CMAKE_CXX_STANDARD_LIBRARIES} -lssp") set(CMAKE_CXX_STANDARD_LIBRARIES "${CMAKE_CXX_STANDARD_LIBRARIES} -lssp")
elseif(HAIKU)
set(CMAKE_SYSROOT "${CROSS_ROOTFS}")
set(TOOLSET_PREFIX ${TOOLCHAIN}-)
function(locate_toolchain_exec exec var)
string(TOUPPER ${exec} EXEC_UPPERCASE)
if(NOT "$ENV{CLR_${EXEC_UPPERCASE}}" STREQUAL "")
set(${var} "$ENV{CLR_${EXEC_UPPERCASE}}" PARENT_SCOPE)
return()
endif()
set(SEARCH_PATH "${CROSS_ROOTFS}/generated/cross-tools-x86_64/bin")
find_program(EXEC_LOCATION_${exec}
PATHS ${SEARCH_PATH}
NAMES
"${TOOLSET_PREFIX}${exec}${CLR_CMAKE_COMPILER_FILE_NAME_VERSION}"
"${TOOLSET_PREFIX}${exec}")
if (EXEC_LOCATION_${exec} STREQUAL "EXEC_LOCATION_${exec}-NOTFOUND")
message(FATAL_ERROR "Unable to find toolchain executable. Name: ${exec}, Prefix: ${TOOLSET_PREFIX}.")
endif()
set(${var} ${EXEC_LOCATION_${exec}} PARENT_SCOPE)
endfunction()
set(CMAKE_SYSTEM_PREFIX_PATH "${CROSS_ROOTFS}")
locate_toolchain_exec(gcc CMAKE_C_COMPILER)
locate_toolchain_exec(g++ CMAKE_CXX_COMPILER)
set(CMAKE_C_STANDARD_LIBRARIES "${CMAKE_C_STANDARD_LIBRARIES} -lssp")
set(CMAKE_CXX_STANDARD_LIBRARIES "${CMAKE_CXX_STANDARD_LIBRARIES} -lssp")
# let CMake set up the correct search paths
include(Platform/Haiku)
else() else()
set(CMAKE_SYSROOT "${CROSS_ROOTFS}") set(CMAKE_SYSROOT "${CROSS_ROOTFS}")
@ -191,7 +252,7 @@ if(TARGET_ARCH_NAME MATCHES "^(arm|armel)$")
add_toolchain_linker_flag("-L${CROSS_ROOTFS}/usr/lib") add_toolchain_linker_flag("-L${CROSS_ROOTFS}/usr/lib")
add_toolchain_linker_flag("-L${CROSS_ROOTFS}/usr/lib/gcc/${TIZEN_TOOLCHAIN}") add_toolchain_linker_flag("-L${CROSS_ROOTFS}/usr/lib/gcc/${TIZEN_TOOLCHAIN}")
endif() endif()
elseif(TARGET_ARCH_NAME STREQUAL "arm64") elseif(TARGET_ARCH_NAME MATCHES "^(arm64|x64)$")
if(TIZEN) if(TIZEN)
add_toolchain_linker_flag("-B${CROSS_ROOTFS}/usr/lib64/gcc/${TIZEN_TOOLCHAIN}") add_toolchain_linker_flag("-B${CROSS_ROOTFS}/usr/lib64/gcc/${TIZEN_TOOLCHAIN}")
add_toolchain_linker_flag("-L${CROSS_ROOTFS}/lib64") add_toolchain_linker_flag("-L${CROSS_ROOTFS}/lib64")
@ -218,7 +279,7 @@ endif()
# Specify compile options # Specify compile options
if((TARGET_ARCH_NAME MATCHES "^(arm|armv6|armel|arm64|ppc64le|s390x)$" AND NOT ANDROID) OR ILLUMOS) if((TARGET_ARCH_NAME MATCHES "^(arm|arm64|armel|armv6|ppc64le|riscv64|s390x)$" AND NOT ANDROID AND NOT FREEBSD) OR ILLUMOS OR HAIKU)
set(CMAKE_C_COMPILER_TARGET ${TOOLCHAIN}) set(CMAKE_C_COMPILER_TARGET ${TOOLCHAIN})
set(CMAKE_CXX_COMPILER_TARGET ${TOOLCHAIN}) set(CMAKE_CXX_COMPILER_TARGET ${TOOLCHAIN})
set(CMAKE_ASM_COMPILER_TARGET ${TOOLCHAIN}) set(CMAKE_ASM_COMPILER_TARGET ${TOOLCHAIN})

47
eng/common/generate-locproject.ps1

@ -10,9 +10,7 @@ Param(
Set-StrictMode -Version 2.0 Set-StrictMode -Version 2.0
$ErrorActionPreference = "Stop" $ErrorActionPreference = "Stop"
. $PSScriptRoot\tools.ps1 . $PSScriptRoot\pipeline-logging-functions.ps1
Import-Module -Name (Join-Path $PSScriptRoot 'native\CommonLibrary.psm1')
$exclusionsFilePath = "$SourcesDirectory\eng\Localize\LocExclusions.json" $exclusionsFilePath = "$SourcesDirectory\eng\Localize\LocExclusions.json"
$exclusions = @{ Exclusions = @() } $exclusions = @{ Exclusions = @() }
@ -28,13 +26,15 @@ $jsonFiles = @()
$jsonTemplateFiles = Get-ChildItem -Recurse -Path "$SourcesDirectory" | Where-Object { $_.FullName -Match "\.template\.config\\localize\\.+\.en\.json" } # .NET templating pattern $jsonTemplateFiles = Get-ChildItem -Recurse -Path "$SourcesDirectory" | Where-Object { $_.FullName -Match "\.template\.config\\localize\\.+\.en\.json" } # .NET templating pattern
$jsonTemplateFiles | ForEach-Object { $jsonTemplateFiles | ForEach-Object {
$null = $_.Name -Match "(.+)\.[\w-]+\.json" # matches '[filename].[langcode].json $null = $_.Name -Match "(.+)\.[\w-]+\.json" # matches '[filename].[langcode].json
$destinationFile = "$($_.Directory.FullName)\$($Matches.1).json" $destinationFile = "$($_.Directory.FullName)\$($Matches.1).json"
$jsonFiles += Copy-Item "$($_.FullName)" -Destination $destinationFile -PassThru $jsonFiles += Copy-Item "$($_.FullName)" -Destination $destinationFile -PassThru
} }
$jsonWinformsTemplateFiles = Get-ChildItem -Recurse -Path "$SourcesDirectory" | Where-Object { $_.FullName -Match "en\\strings\.json" } # current winforms pattern $jsonWinformsTemplateFiles = Get-ChildItem -Recurse -Path "$SourcesDirectory" | Where-Object { $_.FullName -Match "en\\strings\.json" } # current winforms pattern
$wxlFiles = Get-ChildItem -Recurse -Path "$SourcesDirectory" | Where-Object { $_.FullName -Match "\\.+\.wxl" -And -Not( $_.Directory.Name -Match "\d{4}" ) } # localized files live in four digit lang ID directories; this excludes them
$xlfFiles = @() $xlfFiles = @()
$allXlfFiles = Get-ChildItem -Recurse -Path "$SourcesDirectory\*\*.xlf" $allXlfFiles = Get-ChildItem -Recurse -Path "$SourcesDirectory\*\*.xlf"
@ -46,7 +46,7 @@ if ($allXlfFiles) {
} }
$langXlfFiles | ForEach-Object { $langXlfFiles | ForEach-Object {
$null = $_.Name -Match "(.+)\.[\w-]+\.xlf" # matches '[filename].[langcode].xlf $null = $_.Name -Match "(.+)\.[\w-]+\.xlf" # matches '[filename].[langcode].xlf
$destinationFile = "$($_.Directory.FullName)\$($Matches.1).xlf" $destinationFile = "$($_.Directory.FullName)\$($Matches.1).xlf"
$xlfFiles += Copy-Item "$($_.FullName)" -Destination $destinationFile -PassThru $xlfFiles += Copy-Item "$($_.FullName)" -Destination $destinationFile -PassThru
} }
@ -59,10 +59,10 @@ $locJson = @{
LanguageSet = $LanguageSet LanguageSet = $LanguageSet
LocItems = @( LocItems = @(
$locFiles | ForEach-Object { $locFiles | ForEach-Object {
$outputPath = "$(($_.DirectoryName | Resolve-Path -Relative) + "\")" $outputPath = "$(($_.DirectoryName | Resolve-Path -Relative) + "\")"
$continue = $true $continue = $true
foreach ($exclusion in $exclusions.Exclusions) { foreach ($exclusion in $exclusions.Exclusions) {
if ($outputPath.Contains($exclusion)) if ($_.FullName.Contains($exclusion))
{ {
$continue = $false $continue = $false
} }
@ -79,8 +79,7 @@ $locJson = @{
CopyOption = "LangIDOnPath" CopyOption = "LangIDOnPath"
OutputPath = "$($_.Directory.Parent.FullName | Resolve-Path -Relative)\" OutputPath = "$($_.Directory.Parent.FullName | Resolve-Path -Relative)\"
} }
} } else {
else {
return @{ return @{
SourceFile = $sourceFile SourceFile = $sourceFile
CopyOption = "LangIDOnName" CopyOption = "LangIDOnName"
@ -90,6 +89,32 @@ $locJson = @{
} }
} }
) )
},
@{
LanguageSet = $LanguageSet
CloneLanguageSet = "WiX_CloneLanguages"
LssFiles = @( "wxl_loc.lss" )
LocItems = @(
$wxlFiles | ForEach-Object {
$outputPath = "$($_.Directory.FullName | Resolve-Path -Relative)\"
$continue = $true
foreach ($exclusion in $exclusions.Exclusions) {
if ($_.FullName.Contains($exclusion))
{
$continue = $false
}
}
$sourceFile = ($_.FullName | Resolve-Path -Relative)
if ($continue)
{
return @{
SourceFile = $sourceFile
CopyOption = "LangIDOnPath"
OutputPath = $outputPath
}
}
}
)
} }
) )
} }
@ -108,10 +133,10 @@ else {
if ((Get-FileHash "$SourcesDirectory\eng\Localize\LocProject-generated.json").Hash -ne (Get-FileHash "$SourcesDirectory\eng\Localize\LocProject.json").Hash) { if ((Get-FileHash "$SourcesDirectory\eng\Localize\LocProject-generated.json").Hash -ne (Get-FileHash "$SourcesDirectory\eng\Localize\LocProject.json").Hash) {
Write-PipelineTelemetryError -Category "OneLocBuild" -Message "Existing LocProject.json differs from generated LocProject.json. Download LocProject-generated.json and compare them." Write-PipelineTelemetryError -Category "OneLocBuild" -Message "Existing LocProject.json differs from generated LocProject.json. Download LocProject-generated.json and compare them."
exit 1 exit 1
} }
else { else {
Write-Host "Generated LocProject.json and current LocProject.json are identical." Write-Host "Generated LocProject.json and current LocProject.json are identical."
} }
} }

2
eng/common/generate-sbom-prep.ps1

@ -2,6 +2,8 @@ Param(
[Parameter(Mandatory=$true)][string] $ManifestDirPath # Manifest directory where sbom will be placed [Parameter(Mandatory=$true)][string] $ManifestDirPath # Manifest directory where sbom will be placed
) )
. $PSScriptRoot\pipeline-logging-functions.ps1
Write-Host "Creating dir $ManifestDirPath" Write-Host "Creating dir $ManifestDirPath"
# create directory for sbom manifest to be placed # create directory for sbom manifest to be placed
if (!(Test-Path -path $ManifestDirPath)) if (!(Test-Path -path $ManifestDirPath))

12
eng/common/generate-sbom-prep.sh

@ -2,6 +2,18 @@
source="${BASH_SOURCE[0]}" source="${BASH_SOURCE[0]}"
# resolve $SOURCE until the file is no longer a symlink
while [[ -h $source ]]; do
scriptroot="$( cd -P "$( dirname "$source" )" && pwd )"
source="$(readlink "$source")"
# if $source was a relative symlink, we need to resolve it relative to the path where the
# symlink file was located
[[ $source != /* ]] && source="$scriptroot/$source"
done
scriptroot="$( cd -P "$( dirname "$source" )" && pwd )"
. $scriptroot/pipeline-logging-functions.sh
manifest_dir=$1 manifest_dir=$1
if [ ! -d "$manifest_dir" ] ; then if [ ! -d "$manifest_dir" ] ; then

10
eng/common/init-tools-native.ps1

@ -87,6 +87,7 @@ try {
$NativeTools.PSObject.Properties | ForEach-Object { $NativeTools.PSObject.Properties | ForEach-Object {
$ToolName = $_.Name $ToolName = $_.Name
$ToolVersion = $_.Value $ToolVersion = $_.Value
$InstalledTools = @{}
if ((Get-Command "$ToolName" -ErrorAction SilentlyContinue) -eq $null) { if ((Get-Command "$ToolName" -ErrorAction SilentlyContinue) -eq $null) {
if ($ToolVersion -eq "latest") { if ($ToolVersion -eq "latest") {
@ -97,11 +98,12 @@ try {
Write-Error "Arcade tools directory '$ArcadeToolsDirectory' was not found; artifacts were not properly installed." Write-Error "Arcade tools directory '$ArcadeToolsDirectory' was not found; artifacts were not properly installed."
exit 1 exit 1
} }
$ToolDirectory = (Get-ChildItem -Path "$ArcadeToolsDirectory" -Filter "$ToolName-$ToolVersion*" | Sort-Object -Descending)[0] $ToolDirectories = (Get-ChildItem -Path "$ArcadeToolsDirectory" -Filter "$ToolName-$ToolVersion*" | Sort-Object -Descending)
if ([string]::IsNullOrWhiteSpace($ToolDirectory)) { if ($ToolDirectories -eq $null) {
Write-Error "Unable to find directory for $ToolName $ToolVersion; please make sure the tool is installed on this image." Write-Error "Unable to find directory for $ToolName $ToolVersion; please make sure the tool is installed on this image."
exit 1 exit 1
} }
$ToolDirectory = $ToolDirectories[0]
$BinPathFile = "$($ToolDirectory.FullName)\binpath.txt" $BinPathFile = "$($ToolDirectory.FullName)\binpath.txt"
if (-not (Test-Path -Path "$BinPathFile")) { if (-not (Test-Path -Path "$BinPathFile")) {
Write-Error "Unable to find binpath.txt in '$($ToolDirectory.FullName)' ($ToolName $ToolVersion); artifact is either installed incorrectly or is not a bootstrappable tool." Write-Error "Unable to find binpath.txt in '$($ToolDirectory.FullName)' ($ToolName $ToolVersion); artifact is either installed incorrectly or is not a bootstrappable tool."
@ -111,9 +113,11 @@ try {
$ToolPath = Convert-Path -Path $BinPath $ToolPath = Convert-Path -Path $BinPath
Write-Host "Adding $ToolName to the path ($ToolPath)..." Write-Host "Adding $ToolName to the path ($ToolPath)..."
Write-Host "##vso[task.prependpath]$ToolPath" Write-Host "##vso[task.prependpath]$ToolPath"
$env:PATH = "$ToolPath;$env:PATH"
$InstalledTools += @{ $ToolName = $ToolDirectory.FullName }
} }
} }
exit 0 return $InstalledTools
} else { } else {
$NativeTools.PSObject.Properties | ForEach-Object { $NativeTools.PSObject.Properties | ForEach-Object {
$ToolName = $_.Name $ToolName = $_.Name

83
eng/common/native/init-compiler.sh

@ -1,30 +1,25 @@
#!/usr/bin/env bash #!/bin/sh
# #
# This file detects the C/C++ compiler and exports it to the CC/CXX environment variables # This file detects the C/C++ compiler and exports it to the CC/CXX environment variables
# #
# NOTE: some scripts source this file and rely on stdout being empty, make sure to not output anything here! # NOTE: some scripts source this file and rely on stdout being empty, make sure to not output anything here!
if [[ "$#" -lt 3 ]]; then if [ -z "$build_arch" ] || [ -z "$compiler" ]; then
echo "Usage..." echo "Usage..."
echo "init-compiler.sh <script directory> <Architecture> <compiler>" echo "build_arch=<ARCH> compiler=<NAME> init-compiler.sh"
echo "Specify the script directory."
echo "Specify the target architecture." echo "Specify the target architecture."
echo "Specify the name of compiler (clang or gcc)." echo "Specify the name of compiler (clang or gcc)."
exit 1 exit 1
fi fi
nativescriptroot="$1"
build_arch="$2"
compiler="$3"
case "$compiler" in case "$compiler" in
clang*|-clang*|--clang*) clang*|-clang*|--clang*)
# clangx.y or clang-x.y # clangx.y or clang-x.y
version="$(echo "$compiler" | tr -d '[:alpha:]-=')" version="$(echo "$compiler" | tr -d '[:alpha:]-=')"
parts=(${version//./ }) majorVersion="${version%%.*}"
majorVersion="${parts[0]}" [ -z "${version##*.*}" ] && minorVersion="${version#*.}"
minorVersion="${parts[1]}"
if [[ -z "$minorVersion" && "$majorVersion" -le 6 ]]; then if [ -z "$minorVersion" ] && [ -n "$majorVersion" ] && [ "$majorVersion" -le 6 ]; then
minorVersion=0; minorVersion=0;
fi fi
compiler=clang compiler=clang
@ -33,23 +28,20 @@ case "$compiler" in
gcc*|-gcc*|--gcc*) gcc*|-gcc*|--gcc*)
# gccx.y or gcc-x.y # gccx.y or gcc-x.y
version="$(echo "$compiler" | tr -d '[:alpha:]-=')" version="$(echo "$compiler" | tr -d '[:alpha:]-=')"
parts=(${version//./ }) majorVersion="${version%%.*}"
majorVersion="${parts[0]}" [ -z "${version##*.*}" ] && minorVersion="${version#*.}"
minorVersion="${parts[1]}"
compiler=gcc compiler=gcc
;; ;;
esac esac
cxxCompiler="$compiler++" cxxCompiler="$compiler++"
. "$nativescriptroot"/../pipeline-logging-functions.sh
# clear the existing CC and CXX from environment # clear the existing CC and CXX from environment
CC= CC=
CXX= CXX=
LDFLAGS= LDFLAGS=
if [[ "$compiler" == "gcc" ]]; then cxxCompiler="g++"; fi if [ "$compiler" = "gcc" ]; then cxxCompiler="g++"; fi
check_version_exists() { check_version_exists() {
desired_version=-1 desired_version=-1
@ -66,40 +58,41 @@ check_version_exists() {
echo "$desired_version" echo "$desired_version"
} }
if [[ -z "$CLR_CC" ]]; then if [ -z "$CLR_CC" ]; then
# Set default versions # Set default versions
if [[ -z "$majorVersion" ]]; then if [ -z "$majorVersion" ]; then
# note: gcc (all versions) and clang versions higher than 6 do not have minor version in file name, if it is zero. # note: gcc (all versions) and clang versions higher than 6 do not have minor version in file name, if it is zero.
if [[ "$compiler" == "clang" ]]; then versions=( 13 12 11 10 9 8 7 6.0 5.0 4.0 3.9 3.8 3.7 3.6 3.5 ) if [ "$compiler" = "clang" ]; then versions="15 14 13 12 11 10 9 8 7 6.0 5.0 4.0 3.9 3.8 3.7 3.6 3.5"
elif [[ "$compiler" == "gcc" ]]; then versions=( 12 11 10 9 8 7 6 5 4.9 ); fi elif [ "$compiler" = "gcc" ]; then versions="12 11 10 9 8 7 6 5 4.9"; fi
for version in "${versions[@]}"; do for version in $versions; do
parts=(${version//./ }) _major="${version%%.*}"
desired_version="$(check_version_exists "${parts[0]}" "${parts[1]}")" [ -z "${version##*.*}" ] && _minor="${version#*.}"
if [[ "$desired_version" != "-1" ]]; then majorVersion="${parts[0]}"; break; fi desired_version="$(check_version_exists "$_major" "$_minor")"
if [ "$desired_version" != "-1" ]; then majorVersion="$_major"; break; fi
done done
if [[ -z "$majorVersion" ]]; then if [ -z "$majorVersion" ]; then
if command -v "$compiler" > /dev/null; then if command -v "$compiler" > /dev/null; then
if [[ "$(uname)" != "Darwin" ]]; then if [ "$(uname)" != "Darwin" ]; then
Write-PipelineTelemetryError -category "Build" -type "warning" "Specific version of $compiler not found, falling back to use the one in PATH." echo "Warning: Specific version of $compiler not found, falling back to use the one in PATH."
fi fi
CC="$(command -v "$compiler")" CC="$(command -v "$compiler")"
CXX="$(command -v "$cxxCompiler")" CXX="$(command -v "$cxxCompiler")"
else else
Write-PipelineTelemetryError -category "Build" "No usable version of $compiler found." echo "No usable version of $compiler found."
exit 1 exit 1
fi fi
else else
if [[ "$compiler" == "clang" && "$majorVersion" -lt 5 ]]; then if [ "$compiler" = "clang" ] && [ "$majorVersion" -lt 5 ]; then
if [[ "$build_arch" == "arm" || "$build_arch" == "armel" ]]; then if [ "$build_arch" = "arm" ] || [ "$build_arch" = "armel" ]; then
if command -v "$compiler" > /dev/null; then if command -v "$compiler" > /dev/null; then
Write-PipelineTelemetryError -category "Build" -type "warning" "Found clang version $majorVersion which is not supported on arm/armel architectures, falling back to use clang from PATH." echo "Warning: Found clang version $majorVersion which is not supported on arm/armel architectures, falling back to use clang from PATH."
CC="$(command -v "$compiler")" CC="$(command -v "$compiler")"
CXX="$(command -v "$cxxCompiler")" CXX="$(command -v "$cxxCompiler")"
else else
Write-PipelineTelemetryError -category "Build" "Found clang version $majorVersion which is not supported on arm/armel architectures, and there is no clang in PATH." echo "Found clang version $majorVersion which is not supported on arm/armel architectures, and there is no clang in PATH."
exit 1 exit 1
fi fi
fi fi
@ -107,33 +100,33 @@ if [[ -z "$CLR_CC" ]]; then
fi fi
else else
desired_version="$(check_version_exists "$majorVersion" "$minorVersion")" desired_version="$(check_version_exists "$majorVersion" "$minorVersion")"
if [[ "$desired_version" == "-1" ]]; then if [ "$desired_version" = "-1" ]; then
Write-PipelineTelemetryError -category "Build" "Could not find specific version of $compiler: $majorVersion $minorVersion." echo "Could not find specific version of $compiler: $majorVersion $minorVersion."
exit 1 exit 1
fi fi
fi fi
if [[ -z "$CC" ]]; then if [ -z "$CC" ]; then
CC="$(command -v "$compiler$desired_version")" CC="$(command -v "$compiler$desired_version")"
CXX="$(command -v "$cxxCompiler$desired_version")" CXX="$(command -v "$cxxCompiler$desired_version")"
if [[ -z "$CXX" ]]; then CXX="$(command -v "$cxxCompiler")"; fi if [ -z "$CXX" ]; then CXX="$(command -v "$cxxCompiler")"; fi
fi fi
else else
if [[ ! -f "$CLR_CC" ]]; then if [ ! -f "$CLR_CC" ]; then
Write-PipelineTelemetryError -category "Build" "CLR_CC is set but path '$CLR_CC' does not exist" echo "CLR_CC is set but path '$CLR_CC' does not exist"
exit 1 exit 1
fi fi
CC="$CLR_CC" CC="$CLR_CC"
CXX="$CLR_CXX" CXX="$CLR_CXX"
fi fi
if [[ -z "$CC" ]]; then if [ -z "$CC" ]; then
Write-PipelineTelemetryError -category "Build" "Unable to find $compiler." echo "Unable to find $compiler."
exit 1 exit 1
fi fi
# Only lld version >= 9 can be considered stable # Only lld version >= 9 can be considered stable. lld doesn't support s390x.
if [[ "$compiler" == "clang" && "$majorVersion" -ge 9 ]]; then if [ "$compiler" = "clang" ] && [ -n "$majorVersion" ] && [ "$majorVersion" -ge 9 ] && [ "$build_arch" != "s390x" ]; then
if "$CC" -fuse-ld=lld -Wl,--version >/dev/null 2>&1; then if "$CC" -fuse-ld=lld -Wl,--version >/dev/null 2>&1; then
LDFLAGS="-fuse-ld=lld" LDFLAGS="-fuse-ld=lld"
fi fi

2
eng/common/sdk-task.ps1

@ -64,7 +64,7 @@ try {
$GlobalJson.tools | Add-Member -Name "vs" -Value (ConvertFrom-Json "{ `"version`": `"16.5`" }") -MemberType NoteProperty $GlobalJson.tools | Add-Member -Name "vs" -Value (ConvertFrom-Json "{ `"version`": `"16.5`" }") -MemberType NoteProperty
} }
if( -not ($GlobalJson.tools.PSObject.Properties.Name -match "xcopy-msbuild" )) { if( -not ($GlobalJson.tools.PSObject.Properties.Name -match "xcopy-msbuild" )) {
$GlobalJson.tools | Add-Member -Name "xcopy-msbuild" -Value "17.1.0" -MemberType NoteProperty $GlobalJson.tools | Add-Member -Name "xcopy-msbuild" -Value "17.4.1" -MemberType NoteProperty
} }
if ($GlobalJson.tools."xcopy-msbuild".Trim() -ine "none") { if ($GlobalJson.tools."xcopy-msbuild".Trim() -ine "none") {
$xcopyMSBuildToolsFolder = InitializeXCopyMSBuild $GlobalJson.tools."xcopy-msbuild" -install $true $xcopyMSBuildToolsFolder = InitializeXCopyMSBuild $GlobalJson.tools."xcopy-msbuild" -install $true

5
eng/common/sdl/NuGet.config

@ -7,6 +7,11 @@
<clear /> <clear />
<add key="guardian" value="https://securitytools.pkgs.visualstudio.com/_packaging/Guardian/nuget/v3/index.json" /> <add key="guardian" value="https://securitytools.pkgs.visualstudio.com/_packaging/Guardian/nuget/v3/index.json" />
</packageSources> </packageSources>
<packageSourceMapping>
<packageSource key="guardian">
<package pattern="microsoft.guardian.cli" />
</packageSource>
</packageSourceMapping>
<disabledPackageSources> <disabledPackageSources>
<clear /> <clear />
</disabledPackageSources> </disabledPackageSources>

38
eng/common/sdl/sdl.ps1

@ -0,0 +1,38 @@
function Install-Gdn {
param(
[Parameter(Mandatory=$true)]
[string]$Path,
# If omitted, install the latest version of Guardian, otherwise install that specific version.
[string]$Version
)
$ErrorActionPreference = 'Stop'
Set-StrictMode -Version 2.0
$disableConfigureToolsetImport = $true
$global:LASTEXITCODE = 0
# `tools.ps1` checks $ci to perform some actions. Since the SDL
# scripts don't necessarily execute in the same agent that run the
# build.ps1/sh script this variable isn't automatically set.
$ci = $true
. $PSScriptRoot\..\tools.ps1
$argumentList = @("install", "Microsoft.Guardian.Cli", "-Source https://securitytools.pkgs.visualstudio.com/_packaging/Guardian/nuget/v3/index.json", "-OutputDirectory $Path", "-NonInteractive", "-NoCache")
if ($Version) {
$argumentList += "-Version $Version"
}
Start-Process nuget -Verbose -ArgumentList $argumentList -NoNewWindow -Wait
$gdnCliPath = Get-ChildItem -Filter guardian.cmd -Recurse -Path $Path
if (!$gdnCliPath)
{
Write-PipelineTelemetryError -Category 'Sdl' -Message 'Failure installing Guardian'
}
return $gdnCliPath.FullName
}

11
eng/common/templates/job/execute-sdl.yml

@ -34,7 +34,7 @@ jobs:
- job: Run_SDL - job: Run_SDL
dependsOn: ${{ parameters.dependsOn }} dependsOn: ${{ parameters.dependsOn }}
displayName: Run SDL tool displayName: Run SDL tool
condition: eq( ${{ parameters.enable }}, 'true') condition: and(succeededOrFailed(), eq( ${{ parameters.enable }}, 'true'))
variables: variables:
- group: DotNet-VSTS-Bot - group: DotNet-VSTS-Bot
- name: AzDOProjectName - name: AzDOProjectName
@ -46,6 +46,7 @@ jobs:
- template: /eng/common/templates/variables/sdl-variables.yml - template: /eng/common/templates/variables/sdl-variables.yml
- name: GuardianVersion - name: GuardianVersion
value: ${{ coalesce(parameters.overrideGuardianVersion, '$(DefaultGuardianVersion)') }} value: ${{ coalesce(parameters.overrideGuardianVersion, '$(DefaultGuardianVersion)') }}
- template: /eng/common/templates/variables/pool-providers.yml
pool: pool:
# We don't use the collection uri here because it might vary (.visualstudio.com vs. dev.azure.com) # We don't use the collection uri here because it might vary (.visualstudio.com vs. dev.azure.com)
${{ if eq(variables['System.TeamProject'], 'DevDiv') }}: ${{ if eq(variables['System.TeamProject'], 'DevDiv') }}:
@ -53,13 +54,15 @@ jobs:
demands: Cmd demands: Cmd
# If it's not devdiv, it's dnceng # If it's not devdiv, it's dnceng
${{ if ne(variables['System.TeamProject'], 'DevDiv') }}: ${{ if ne(variables['System.TeamProject'], 'DevDiv') }}:
name: NetCore1ESPool-Internal name: $(DncEngInternalBuildPool)
demands: ImageOverride -equals Build.Server.Amd64.VS2019 demands: ImageOverride -equals windows.vs2019.amd64
steps: steps:
- checkout: self - checkout: self
clean: true clean: true
- template: /eng/common/templates/post-build/setup-maestro-vars.yml # If the template caller didn't provide an AzDO parameter, set them all up as Maestro vars.
- ${{ if not(and(parameters.AzDOProjectName, parameters.AzDOPipelineId, parameters.AzDOBuildId)) }}:
- template: /eng/common/templates/post-build/setup-maestro-vars.yml
- ${{ if ne(parameters.downloadArtifacts, 'false')}}: - ${{ if ne(parameters.downloadArtifacts, 'false')}}:
- ${{ if ne(parameters.artifactNames, '') }}: - ${{ if ne(parameters.artifactNames, '') }}:

47
eng/common/templates/job/job.yml

@ -24,7 +24,9 @@ parameters:
enablePublishBuildAssets: false enablePublishBuildAssets: false
enablePublishTestResults: false enablePublishTestResults: false
enablePublishUsingPipelines: false enablePublishUsingPipelines: false
enableBuildRetry: false
disableComponentGovernance: false disableComponentGovernance: false
componentGovernanceIgnoreDirectories: ''
mergeTestResults: false mergeTestResults: false
testRunTitle: '' testRunTitle: ''
testResultsFormat: '' testResultsFormat: ''
@ -140,11 +142,14 @@ jobs:
languages: ${{ coalesce(parameters.richCodeNavigationLanguage, 'csharp') }} languages: ${{ coalesce(parameters.richCodeNavigationLanguage, 'csharp') }}
environment: ${{ coalesce(parameters.richCodeNavigationEnvironment, 'production') }} environment: ${{ coalesce(parameters.richCodeNavigationEnvironment, 'production') }}
richNavLogOutputDirectory: $(Build.SourcesDirectory)/artifacts/bin richNavLogOutputDirectory: $(Build.SourcesDirectory)/artifacts/bin
uploadRichNavArtifacts: ${{ coalesce(parameters.richCodeNavigationUploadArtifacts, false) }}
continueOnError: true continueOnError: true
- ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest'), ne(parameters.disableComponentGovernance, 'true')) }}: - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest'), ne(parameters.disableComponentGovernance, 'true')) }}:
- task: ComponentGovernanceComponentDetection@0 - task: ComponentGovernanceComponentDetection@0
continueOnError: true continueOnError: true
inputs:
ignoreDirectories: ${{ parameters.componentGovernanceIgnoreDirectories }}
- ${{ if eq(parameters.enableMicrobuild, 'true') }}: - ${{ if eq(parameters.enableMicrobuild, 'true') }}:
- ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}:
@ -183,24 +188,6 @@ jobs:
displayName: Publish logs displayName: Publish logs
continueOnError: true continueOnError: true
condition: always() condition: always()
- ${{ if or(eq(parameters.artifacts.publish.manifests, 'true'), ne(parameters.artifacts.publish.manifests, '')) }}:
- ${{ if and(ne(parameters.enablePublishUsingPipelines, 'true'), eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}:
- task: CopyFiles@2
displayName: Gather Asset Manifests
inputs:
SourceFolder: '$(Build.SourcesDirectory)/artifacts/log/$(_BuildConfig)/AssetManifest'
TargetFolder: '$(Build.ArtifactStagingDirectory)/AssetManifests'
continueOnError: ${{ parameters.continueOnError }}
condition: and(succeeded(), eq(variables['_DotNetPublishToBlobFeed'], 'true'))
- task: PublishBuildArtifacts@1
displayName: Push Asset Manifests
inputs:
PathtoPublish: '$(Build.ArtifactStagingDirectory)/AssetManifests'
PublishLocation: Container
ArtifactName: AssetManifests
continueOnError: ${{ parameters.continueOnError }}
condition: and(succeeded(), eq(variables['_DotNetPublishToBlobFeed'], 'true'))
- ${{ if ne(parameters.enablePublishBuildArtifacts, 'false') }}: - ${{ if ne(parameters.enablePublishBuildArtifacts, 'false') }}:
- task: PublishBuildArtifacts@1 - task: PublishBuildArtifacts@1
@ -234,28 +221,16 @@ jobs:
mergeTestResults: ${{ parameters.mergeTestResults }} mergeTestResults: ${{ parameters.mergeTestResults }}
continueOnError: true continueOnError: true
condition: always() condition: always()
- ${{ if and(eq(parameters.enablePublishBuildAssets, true), ne(parameters.enablePublishUsingPipelines, 'true'), eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}:
- task: CopyFiles@2
displayName: Gather Asset Manifests
inputs:
SourceFolder: '$(Build.SourcesDirectory)/artifacts/log/$(_BuildConfig)/AssetManifest'
TargetFolder: '$(Build.StagingDirectory)/AssetManifests'
continueOnError: ${{ parameters.continueOnError }}
condition: and(succeeded(), eq(variables['_DotNetPublishToBlobFeed'], 'true'))
- task: PublishBuildArtifacts@1
displayName: Push Asset Manifests
inputs:
PathtoPublish: '$(Build.StagingDirectory)/AssetManifests'
PublishLocation: Container
ArtifactName: AssetManifests
continueOnError: ${{ parameters.continueOnError }}
condition: and(succeeded(), eq(variables['_DotNetPublishToBlobFeed'], 'true'))
- ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest'), eq(parameters.enableSbom, 'true')) }}: - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest'), eq(parameters.enableSbom, 'true')) }}:
- template: /eng/common/templates/steps/generate-sbom.yml - template: /eng/common/templates/steps/generate-sbom.yml
parameters: parameters:
PackageVersion: ${{ parameters.packageVersion}} PackageVersion: ${{ parameters.packageVersion}}
BuildDropPath: ${{ parameters.buildDropPath }} BuildDropPath: ${{ parameters.buildDropPath }}
IgnoreDirectories: ${{ parameters.componentGovernanceIgnoreDirectories }}
- ${{ if eq(parameters.enableBuildRetry, 'true') }}:
- publish: $(Build.SourcesDirectory)\eng\common\BuildConfiguration
artifact: BuildConfiguration
displayName: Publish build retry configuration
continueOnError: true

45
eng/common/templates/job/onelocbuild.yml

@ -14,6 +14,7 @@ parameters:
ReusePr: true ReusePr: true
UseLfLineEndings: true UseLfLineEndings: true
UseCheckedInLocProjectJson: false UseCheckedInLocProjectJson: false
SkipLocProjectJsonGeneration: false
LanguageSet: VS_Main_Languages LanguageSet: VS_Main_Languages
LclSource: lclFilesInRepo LclSource: lclFilesInRepo
LclPackageId: '' LclPackageId: ''
@ -22,13 +23,25 @@ parameters:
MirrorRepo: '' MirrorRepo: ''
MirrorBranch: main MirrorBranch: main
condition: '' condition: ''
JobNameSuffix: ''
jobs: jobs:
- job: OneLocBuild - job: OneLocBuild${{ parameters.JobNameSuffix }}
dependsOn: ${{ parameters.dependsOn }} dependsOn: ${{ parameters.dependsOn }}
displayName: OneLocBuild displayName: OneLocBuild${{ parameters.JobNameSuffix }}
variables:
- group: OneLocBuildVariables # Contains the CeapexPat and GithubPat
- name: _GenerateLocProjectArguments
value: -SourcesDirectory ${{ parameters.SourcesDirectory }}
-LanguageSet "${{ parameters.LanguageSet }}"
-CreateNeutralXlfs
- ${{ if eq(parameters.UseCheckedInLocProjectJson, 'true') }}:
- name: _GenerateLocProjectArguments
value: ${{ variables._GenerateLocProjectArguments }} -UseCheckedInLocProjectJson
- template: /eng/common/templates/variables/pool-providers.yml
${{ if ne(parameters.pool, '') }}: ${{ if ne(parameters.pool, '') }}:
pool: ${{ parameters.pool }} pool: ${{ parameters.pool }}
@ -40,27 +53,17 @@ jobs:
demands: Cmd demands: Cmd
# If it's not devdiv, it's dnceng # If it's not devdiv, it's dnceng
${{ if ne(variables['System.TeamProject'], 'DevDiv') }}: ${{ if ne(variables['System.TeamProject'], 'DevDiv') }}:
name: NetCore1ESPool-Internal name: $(DncEngInternalBuildPool)
demands: ImageOverride -equals Build.Server.Amd64.VS2019 demands: ImageOverride -equals windows.vs2019.amd64
variables:
- group: OneLocBuildVariables # Contains the CeapexPat and GithubPat
- name: _GenerateLocProjectArguments
value: -SourcesDirectory ${{ parameters.SourcesDirectory }}
-LanguageSet "${{ parameters.LanguageSet }}"
-CreateNeutralXlfs
- ${{ if eq(parameters.UseCheckedInLocProjectJson, 'true') }}:
- name: _GenerateLocProjectArguments
value: ${{ variables._GenerateLocProjectArguments }} -UseCheckedInLocProjectJson
steps: steps:
- task: Powershell@2 - ${{ if ne(parameters.SkipLocProjectJsonGeneration, 'true') }}:
inputs: - task: Powershell@2
filePath: $(Build.SourcesDirectory)/eng/common/generate-locproject.ps1 inputs:
arguments: $(_GenerateLocProjectArguments) filePath: $(Build.SourcesDirectory)/eng/common/generate-locproject.ps1
displayName: Generate LocProject.json arguments: $(_GenerateLocProjectArguments)
condition: ${{ parameters.condition }} displayName: Generate LocProject.json
condition: ${{ parameters.condition }}
- task: OneLocBuild@2 - task: OneLocBuild@2
displayName: OneLocBuild displayName: OneLocBuild

20
eng/common/templates/job/publish-build-assets.yml

@ -34,15 +34,15 @@ jobs:
- job: Asset_Registry_Publish - job: Asset_Registry_Publish
dependsOn: ${{ parameters.dependsOn }} dependsOn: ${{ parameters.dependsOn }}
timeoutInMinutes: 150
${{ if eq(parameters.publishAssetsImmediately, 'true') }}: ${{ if eq(parameters.publishAssetsImmediately, 'true') }}:
displayName: Publish Assets displayName: Publish Assets
${{ else }}: ${{ else }}:
displayName: Publish to Build Asset Registry displayName: Publish to Build Asset Registry
pool: ${{ parameters.pool }}
variables: variables:
- template: /eng/common/templates/variables/pool-providers.yml
- ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}:
- group: Publish-Build-Assets - group: Publish-Build-Assets
- group: AzureDevOps-Artifact-Feeds-Pats - group: AzureDevOps-Artifact-Feeds-Pats
@ -51,6 +51,16 @@ jobs:
- ${{ if eq(parameters.publishAssetsImmediately, 'true') }}: - ${{ if eq(parameters.publishAssetsImmediately, 'true') }}:
- template: /eng/common/templates/post-build/common-variables.yml - template: /eng/common/templates/post-build/common-variables.yml
pool:
# We don't use the collection uri here because it might vary (.visualstudio.com vs. dev.azure.com)
${{ if eq(variables['System.TeamProject'], 'DevDiv') }}:
name: VSEngSS-MicroBuild2022-1ES
demands: Cmd
# If it's not devdiv, it's dnceng
${{ if ne(variables['System.TeamProject'], 'DevDiv') }}:
name: $(DncEngInternalBuildPool)
demands: ImageOverride -equals windows.vs2019.amd64
steps: steps:
- ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}:
- task: DownloadBuildArtifacts@0 - task: DownloadBuildArtifacts@0
@ -64,12 +74,6 @@ jobs:
- task: NuGetAuthenticate@0 - task: NuGetAuthenticate@0
- task: PowerShell@2
displayName: Enable cross-org NuGet feed authentication
inputs:
filePath: $(Build.SourcesDirectory)/eng/common/enable-cross-org-publishing.ps1
arguments: -token $(dn-bot-all-orgs-artifact-feeds-rw)
- task: PowerShell@2 - task: PowerShell@2
displayName: Publish Build Assets displayName: Publish Build Assets
inputs: inputs:

15
eng/common/templates/job/source-build.yml

@ -44,13 +44,24 @@ jobs:
${{ if eq(parameters.platform.pool, '') }}: ${{ if eq(parameters.platform.pool, '') }}:
# The default VM host AzDO pool. This should be capable of running Docker containers: almost all # The default VM host AzDO pool. This should be capable of running Docker containers: almost all
# source-build builds run in Docker, including the default managed platform. # source-build builds run in Docker, including the default managed platform.
# /eng/common/templates/variables/pool-providers.yml can't be used here (some customers declare variables already), so duplicate its logic
pool: pool:
${{ if eq(variables['System.TeamProject'], 'public') }}: # Main environments
${{ if and(eq(variables['System.TeamProject'], 'public'), ne(contains(coalesce(variables['System.PullRequest.TargetBranch'], variables['Build.SourceBranch'], 'refs/heads/main'), 'release'), true)) }}:
name: NetCore-Public name: NetCore-Public
demands: ImageOverride -equals Build.Ubuntu.1804.Amd64.Open demands: ImageOverride -equals Build.Ubuntu.1804.Amd64.Open
${{ if eq(variables['System.TeamProject'], 'internal') }}: ${{ if and(eq(variables['System.TeamProject'], 'internal'), ne(contains(coalesce(variables['System.PullRequest.TargetBranch'], variables['Build.SourceBranch'], 'refs/heads/main'), 'release'), true)) }}:
name: NetCore1ESPool-Internal name: NetCore1ESPool-Internal
demands: ImageOverride -equals Build.Ubuntu.1804.Amd64 demands: ImageOverride -equals Build.Ubuntu.1804.Amd64
# Servicing build environments
${{ if and(eq(variables['System.TeamProject'], 'public'), contains(coalesce(variables['System.PullRequest.TargetBranch'], variables['Build.SourceBranch'], 'refs/heads/main'), 'release')) }}:
name: NetCore-Svc-Public
demands: ImageOverride -equals Build.Ubuntu.1804.Amd64.Open
${{ if and(eq(variables['System.TeamProject'], 'internal'), contains(coalesce(variables['System.PullRequest.TargetBranch'], variables['Build.SourceBranch'], 'refs/heads/main'), 'release')) }}:
name: NetCore1ESPool-Svc-Internal
demands: ImageOverride -equals Build.Ubuntu.1804.Amd64
${{ if ne(parameters.platform.pool, '') }}: ${{ if ne(parameters.platform.pool, '') }}:
pool: ${{ parameters.platform.pool }} pool: ${{ parameters.platform.pool }}

11
eng/common/templates/job/source-index-stage1.yml

@ -1,6 +1,6 @@
parameters: parameters:
runAsPublic: false runAsPublic: false
sourceIndexPackageVersion: 1.0.1-20210614.1 sourceIndexPackageVersion: 1.0.1-20220804.1
sourceIndexPackageSource: https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools/nuget/v3/index.json sourceIndexPackageSource: https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools/nuget/v3/index.json
sourceIndexBuildCommand: powershell -NoLogo -NoProfile -ExecutionPolicy Bypass -Command "eng/common/build.ps1 -restore -build -binarylog -ci" sourceIndexBuildCommand: powershell -NoLogo -NoProfile -ExecutionPolicy Bypass -Command "eng/common/build.ps1 -restore -build -binarylog -ci"
preSteps: [] preSteps: []
@ -22,17 +22,18 @@ jobs:
value: ${{ parameters.binlogPath }} value: ${{ parameters.binlogPath }}
- ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}:
- group: source-dot-net stage1 variables - group: source-dot-net stage1 variables
- template: /eng/common/templates/variables/pool-providers.yml
${{ if ne(parameters.pool, '') }}: ${{ if ne(parameters.pool, '') }}:
pool: ${{ parameters.pool }} pool: ${{ parameters.pool }}
${{ if eq(parameters.pool, '') }}: ${{ if eq(parameters.pool, '') }}:
pool: pool:
${{ if eq(variables['System.TeamProject'], 'public') }}: ${{ if eq(variables['System.TeamProject'], 'public') }}:
name: NetCore-Public name: $(DncEngPublicBuildPool)
demands: ImageOverride -equals Build.Server.Amd64.VS2019.Open demands: ImageOverride -equals windows.vs2019.amd64.open
${{ if eq(variables['System.TeamProject'], 'internal') }}: ${{ if eq(variables['System.TeamProject'], 'internal') }}:
name: NetCore1ESPool-Internal name: $(DncEngInternalBuildPool)
demands: ImageOverride -equals Build.Server.Amd64.VS2019 demands: ImageOverride -equals windows.vs2019.amd64
steps: steps:
- ${{ each preStep in parameters.preSteps }}: - ${{ each preStep in parameters.preSteps }}:

9
eng/common/templates/jobs/jobs.yml

@ -88,15 +88,6 @@ jobs:
- ${{ job.job }} - ${{ job.job }}
- ${{ if eq(parameters.enableSourceBuild, true) }}: - ${{ if eq(parameters.enableSourceBuild, true) }}:
- Source_Build_Complete - Source_Build_Complete
pool:
# We don't use the collection uri here because it might vary (.visualstudio.com vs. dev.azure.com)
${{ if eq(variables['System.TeamProject'], 'DevDiv') }}:
name: VSEngSS-MicroBuild2022-1ES
demands: Cmd
# If it's not devdiv, it's dnceng
${{ if ne(variables['System.TeamProject'], 'DevDiv') }}:
name: NetCore1ESPool-Internal
demands: ImageOverride -equals Build.Server.Amd64.VS2019
runAsPublic: ${{ parameters.runAsPublic }} runAsPublic: ${{ parameters.runAsPublic }}
publishUsingPipelines: ${{ parameters.enablePublishUsingPipelines }} publishUsingPipelines: ${{ parameters.enablePublishUsingPipelines }}

2
eng/common/templates/jobs/source-build.yml

@ -14,7 +14,7 @@ parameters:
# This is the default platform provided by Arcade, intended for use by a managed-only repo. # This is the default platform provided by Arcade, intended for use by a managed-only repo.
defaultManagedPlatform: defaultManagedPlatform:
name: 'Managed' name: 'Managed'
container: 'mcr.microsoft.com/dotnet-buildtools/prereqs:centos-7-3e800f1-20190501005343' container: 'mcr.microsoft.com/dotnet-buildtools/prereqs:centos-stream8'
# Defines the platforms on which to run build jobs. One job is created for each platform, and the # Defines the platforms on which to run build jobs. One job is created for each platform, and the
# object in this array is sent to the job template as 'platform'. If no platforms are specified, # object in this array is sent to the job template as 'platform'. If no platforms are specified,

4
eng/common/templates/post-build/common-variables.yml

@ -1,8 +1,4 @@
variables: variables:
- group: AzureDevOps-Artifact-Feeds-Pats
- group: DotNet-Blob-Feed
- group: DotNet-DotNetCli-Storage
- group: DotNet-MSRC-Storage
- group: Publish-Build-Assets - group: Publish-Build-Assets
# Whether the build is internal or not # Whether the build is internal or not

30
eng/common/templates/post-build/post-build.yml

@ -49,6 +49,7 @@ parameters:
type: object type: object
default: default:
enable: false enable: false
publishGdn: false
continueOnError: false continueOnError: false
params: '' params: ''
artifactNames: '' artifactNames: ''
@ -94,10 +95,11 @@ stages:
displayName: Validate Build Assets displayName: Validate Build Assets
variables: variables:
- template: common-variables.yml - template: common-variables.yml
- template: /eng/common/templates/variables/pool-providers.yml
jobs: jobs:
- job: - job:
displayName: NuGet Validation displayName: NuGet Validation
condition: eq( ${{ parameters.enableNugetValidation }}, 'true') condition: and(succeededOrFailed(), eq( ${{ parameters.enableNugetValidation }}, 'true'))
pool: pool:
# We don't use the collection uri here because it might vary (.visualstudio.com vs. dev.azure.com) # We don't use the collection uri here because it might vary (.visualstudio.com vs. dev.azure.com)
${{ if eq(variables['System.TeamProject'], 'DevDiv') }}: ${{ if eq(variables['System.TeamProject'], 'DevDiv') }}:
@ -105,8 +107,8 @@ stages:
demands: Cmd demands: Cmd
# If it's not devdiv, it's dnceng # If it's not devdiv, it's dnceng
${{ else }}: ${{ else }}:
name: NetCore1ESPool-Internal name: $(DncEngInternalBuildPool)
demands: ImageOverride -equals Build.Server.Amd64.VS2019 demands: ImageOverride -equals windows.vs2019.amd64
steps: steps:
- template: setup-maestro-vars.yml - template: setup-maestro-vars.yml
@ -142,8 +144,8 @@ stages:
demands: Cmd demands: Cmd
# If it's not devdiv, it's dnceng # If it's not devdiv, it's dnceng
${{ else }}: ${{ else }}:
name: NetCore1ESPool-Internal name: $(DncEngInternalBuildPool)
demands: ImageOverride -equals Build.Server.Amd64.VS2019 demands: ImageOverride -equals windows.vs2019.amd64
steps: steps:
- template: setup-maestro-vars.yml - template: setup-maestro-vars.yml
parameters: parameters:
@ -170,12 +172,6 @@ stages:
- task: NuGetAuthenticate@0 - task: NuGetAuthenticate@0
displayName: 'Authenticate to AzDO Feeds' displayName: 'Authenticate to AzDO Feeds'
- task: PowerShell@2
displayName: Enable cross-org publishing
inputs:
filePath: eng\common\enable-cross-org-publishing.ps1
arguments: -token $(dn-bot-dnceng-artifact-feeds-rw)
# Signing validation will optionally work with the buildmanifest file which is downloaded from # Signing validation will optionally work with the buildmanifest file which is downloaded from
# Azure DevOps above. # Azure DevOps above.
- task: PowerShell@2 - task: PowerShell@2
@ -202,8 +198,8 @@ stages:
demands: Cmd demands: Cmd
# If it's not devdiv, it's dnceng # If it's not devdiv, it's dnceng
${{ else }}: ${{ else }}:
name: NetCore1ESPool-Internal name: $(DncEngInternalBuildPool)
demands: ImageOverride -equals Build.Server.Amd64.VS2019 demands: ImageOverride -equals windows.vs2019.amd64
steps: steps:
- template: setup-maestro-vars.yml - template: setup-maestro-vars.yml
parameters: parameters:
@ -235,6 +231,7 @@ stages:
- template: /eng/common/templates/job/execute-sdl.yml - template: /eng/common/templates/job/execute-sdl.yml
parameters: parameters:
enable: ${{ parameters.SDLValidationParameters.enable }} enable: ${{ parameters.SDLValidationParameters.enable }}
publishGuardianDirectoryToPipeline: ${{ parameters.SDLValidationParameters.publishGdn }}
additionalParameters: ${{ parameters.SDLValidationParameters.params }} additionalParameters: ${{ parameters.SDLValidationParameters.params }}
continueOnError: ${{ parameters.SDLValidationParameters.continueOnError }} continueOnError: ${{ parameters.SDLValidationParameters.continueOnError }}
artifactNames: ${{ parameters.SDLValidationParameters.artifactNames }} artifactNames: ${{ parameters.SDLValidationParameters.artifactNames }}
@ -249,6 +246,7 @@ stages:
displayName: Publish using Darc displayName: Publish using Darc
variables: variables:
- template: common-variables.yml - template: common-variables.yml
- template: /eng/common/templates/variables/pool-providers.yml
jobs: jobs:
- job: - job:
displayName: Publish Using Darc displayName: Publish Using Darc
@ -260,8 +258,8 @@ stages:
demands: Cmd demands: Cmd
# If it's not devdiv, it's dnceng # If it's not devdiv, it's dnceng
${{ else }}: ${{ else }}:
name: NetCore1ESPool-Internal name: $(DncEngInternalBuildPool)
demands: ImageOverride -equals Build.Server.Amd64.VS2019 demands: ImageOverride -equals windows.vs2019.amd64
steps: steps:
- template: setup-maestro-vars.yml - template: setup-maestro-vars.yml
parameters: parameters:
@ -280,4 +278,4 @@ stages:
-MaestroToken '$(MaestroApiAccessToken)' -MaestroToken '$(MaestroApiAccessToken)'
-WaitPublishingFinish true -WaitPublishingFinish true
-ArtifactsPublishingAdditionalParameters '${{ parameters.artifactsPublishingAdditionalParameters }}' -ArtifactsPublishingAdditionalParameters '${{ parameters.artifactsPublishingAdditionalParameters }}'
-SymbolPublishingAdditionalParameters '${{ parameters.symbolPublishingAdditionalParameters }}' -SymbolPublishingAdditionalParameters '${{ parameters.symbolPublishingAdditionalParameters }}'

60
eng/common/templates/steps/execute-sdl.yml

@ -8,29 +8,28 @@ parameters:
condition: '' condition: ''
steps: steps:
- ${{ if ne(parameters.overrideGuardianVersion, '') }}: - task: NuGetAuthenticate@1
- powershell: | inputs:
$content = Get-Content $(GuardianPackagesConfigFile) nuGetServiceConnections: GuardianConnect
Write-Host "packages.config content was:`n$content"
$content = $content.Replace('$(DefaultGuardianVersion)', '$(GuardianVersion)')
$content | Set-Content $(GuardianPackagesConfigFile)
Write-Host "packages.config content updated to:`n$content"
displayName: Use overridden Guardian version ${{ parameters.overrideGuardianVersion }}
- task: NuGetToolInstaller@1 - task: NuGetToolInstaller@1
displayName: 'Install NuGet.exe' displayName: 'Install NuGet.exe'
- task: NuGetCommand@2 - ${{ if ne(parameters.overrideGuardianVersion, '') }}:
displayName: 'Install Guardian' - pwsh: |
inputs: Set-Location -Path $(Build.SourcesDirectory)\eng\common\sdl
restoreSolution: $(Build.SourcesDirectory)\eng\common\sdl\packages.config . .\sdl.ps1
feedsToUse: config $guardianCliLocation = Install-Gdn -Path $(Build.SourcesDirectory)\.artifacts -Version ${{ parameters.overrideGuardianVersion }}
nugetConfigPath: $(Build.SourcesDirectory)\eng\common\sdl\NuGet.config Write-Host "##vso[task.setvariable variable=GuardianCliLocation]$guardianCliLocation"
externalFeedCredentials: GuardianConnect displayName: Install Guardian (Overridden)
restoreDirectory: $(Build.SourcesDirectory)\.packages
- ${{ if eq(parameters.overrideGuardianVersion, '') }}:
- pwsh: |
Set-Location -Path $(Build.SourcesDirectory)\eng\common\sdl
. .\sdl.ps1
$guardianCliLocation = Install-Gdn -Path $(Build.SourcesDirectory)\.artifacts
Write-Host "##vso[task.setvariable variable=GuardianCliLocation]$guardianCliLocation"
displayName: Install Guardian
- ${{ if ne(parameters.overrideParameters, '') }}: - ${{ if ne(parameters.overrideParameters, '') }}:
- powershell: ${{ parameters.executeAllSdlToolsScript }} ${{ parameters.overrideParameters }} - powershell: ${{ parameters.executeAllSdlToolsScript }} ${{ parameters.overrideParameters }}
@ -40,7 +39,7 @@ steps:
- ${{ if eq(parameters.overrideParameters, '') }}: - ${{ if eq(parameters.overrideParameters, '') }}:
- powershell: ${{ parameters.executeAllSdlToolsScript }} - powershell: ${{ parameters.executeAllSdlToolsScript }}
-GuardianPackageName Microsoft.Guardian.Cli.$(GuardianVersion) -GuardianCliLocation $(GuardianCliLocation)
-NugetPackageDirectory $(Build.SourcesDirectory)\.packages -NugetPackageDirectory $(Build.SourcesDirectory)\.packages
-AzureDevOpsAccessToken $(dn-bot-dotnet-build-rw-code-rw) -AzureDevOpsAccessToken $(dn-bot-dotnet-build-rw-code-rw)
${{ parameters.additionalParameters }} ${{ parameters.additionalParameters }}
@ -62,7 +61,28 @@ steps:
c c
i i
condition: succeededOrFailed() condition: succeededOrFailed()
- publish: $(Agent.BuildDirectory)/.gdn - publish: $(Agent.BuildDirectory)/.gdn
artifact: GuardianConfiguration artifact: GuardianConfiguration
displayName: Publish GuardianConfiguration displayName: Publish GuardianConfiguration
condition: succeededOrFailed()
# Publish the SARIF files in a container named CodeAnalysisLogs to enable integration
# with the "SARIF SAST Scans Tab" Azure DevOps extension
- task: CopyFiles@2
displayName: Copy SARIF files
inputs:
flattenFolders: true
sourceFolder: $(Agent.BuildDirectory)/.gdn/rc/
contents: '**/*.sarif'
targetFolder: $(Build.SourcesDirectory)/CodeAnalysisLogs
condition: succeededOrFailed()
# Use PublishBuildArtifacts because the SARIF extension only checks this case
# see microsoft/sarif-azuredevops-extension#4
- task: PublishBuildArtifacts@1
displayName: Publish SARIF files to CodeAnalysisLogs container
inputs:
pathToPublish: $(Build.SourcesDirectory)/CodeAnalysisLogs
artifactName: CodeAnalysisLogs
condition: succeededOrFailed() condition: succeededOrFailed()

4
eng/common/templates/steps/generate-sbom.yml

@ -2,12 +2,14 @@
# PackageName - The name of the package this SBOM represents. # PackageName - The name of the package this SBOM represents.
# PackageVersion - The version of the package this SBOM represents. # PackageVersion - The version of the package this SBOM represents.
# ManifestDirPath - The path of the directory where the generated manifest files will be placed # ManifestDirPath - The path of the directory where the generated manifest files will be placed
# IgnoreDirectories - Directories to ignore for SBOM generation. This will be passed through to the CG component detector.
parameters: parameters:
PackageVersion: 7.0.0 PackageVersion: 7.0.0
BuildDropPath: '$(Build.SourcesDirectory)/artifacts' BuildDropPath: '$(Build.SourcesDirectory)/artifacts'
PackageName: '.NET' PackageName: '.NET'
ManifestDirPath: $(Build.ArtifactStagingDirectory)/sbom ManifestDirPath: $(Build.ArtifactStagingDirectory)/sbom
IgnoreDirectories: ''
sbomContinueOnError: true sbomContinueOnError: true
steps: steps:
@ -34,6 +36,8 @@ steps:
BuildDropPath: ${{ parameters.buildDropPath }} BuildDropPath: ${{ parameters.buildDropPath }}
PackageVersion: ${{ parameters.packageVersion }} PackageVersion: ${{ parameters.packageVersion }}
ManifestDirPath: ${{ parameters.manifestDirPath }} ManifestDirPath: ${{ parameters.manifestDirPath }}
${{ if ne(parameters.IgnoreDirectories, '') }}:
AdditionalComponentDetectorArgs: '--IgnoreDirectories ${{ parameters.IgnoreDirectories }}'
- task: PublishPipelineArtifact@1 - task: PublishPipelineArtifact@1
displayName: Publish SBOM manifest displayName: Publish SBOM manifest

14
eng/common/templates/steps/source-build.yml

@ -63,11 +63,21 @@ steps:
targetRidArgs='/p:TargetRid=${{ parameters.platform.targetRID }}' targetRidArgs='/p:TargetRid=${{ parameters.platform.targetRID }}'
fi fi
runtimeOsArgs=
if [ '${{ parameters.platform.runtimeOS }}' != '' ]; then
runtimeOsArgs='/p:RuntimeOS=${{ parameters.platform.runtimeOS }}'
fi
publishArgs= publishArgs=
if [ '${{ parameters.platform.skipPublishValidation }}' != 'true' ]; then if [ '${{ parameters.platform.skipPublishValidation }}' != 'true' ]; then
publishArgs='--publish' publishArgs='--publish'
fi fi
assetManifestFileName=SourceBuild_RidSpecific.xml
if [ '${{ parameters.platform.name }}' != '' ]; then
assetManifestFileName=SourceBuild_${{ parameters.platform.name }}.xml
fi
${{ coalesce(parameters.platform.buildScript, './build.sh') }} --ci \ ${{ coalesce(parameters.platform.buildScript, './build.sh') }} --ci \
--configuration $buildConfig \ --configuration $buildConfig \
--restore --build --pack $publishArgs -bl \ --restore --build --pack $publishArgs -bl \
@ -75,8 +85,10 @@ steps:
$internalRuntimeDownloadArgs \ $internalRuntimeDownloadArgs \
$internalRestoreArgs \ $internalRestoreArgs \
$targetRidArgs \ $targetRidArgs \
$runtimeOsArgs \
/p:SourceBuildNonPortable=${{ parameters.platform.nonPortable }} \ /p:SourceBuildNonPortable=${{ parameters.platform.nonPortable }} \
/p:ArcadeBuildFromSource=true /p:ArcadeBuildFromSource=true \
/p:AssetManifestFileName=$assetManifestFileName
displayName: Build displayName: Build
# Upload build logs for diagnosis. # Upload build logs for diagnosis.

48
eng/common/templates/variables/pool-providers.yml

@ -0,0 +1,48 @@
# Select a pool provider based off branch name. Anything with branch name containing 'release' must go into an -Svc pool,
# otherwise it should go into the "normal" pools. This separates out the queueing and billing of released branches.
# Motivation:
# Once a given branch of a repository's output has been officially "shipped" once, it is then considered to be COGS
# (Cost of goods sold) and should be moved to a servicing pool provider. This allows both separation of queueing
# (allowing release builds and main PR builds to not intefere with each other) and billing (required for COGS.
# Additionally, the pool provider name itself may be subject to change when the .NET Core Engineering Services
# team needs to move resources around and create new and potentially differently-named pools. Using this template
# file from an Arcade-ified repo helps guard against both having to update one's release/* branches and renaming.
# How to use:
# This yaml assumes your shipped product branches use the naming convention "release/..." (which many do).
# If we find alternate naming conventions in broad usage it can be added to the condition below.
#
# First, import the template in an arcade-ified repo to pick up the variables, e.g.:
#
# variables:
# - template: eng/common/templates/variables/pool-providers.yml
#
# ... then anywhere specifying the pool provider use the runtime variables,
# $(DncEngInternalBuildPool) and $ (DncEngPublicBuildPool), e.g.:
#
# pool:
# name: $(DncEngInternalBuildPool)
# demands: ImageOverride -equals windows.vs2019.amd64
variables:
# Coalesce the target and source branches so we know when a PR targets a release branch
# If these variables are somehow missing, fall back to main (tends to have more capacity)
- name: BranchNameForPoolSelection
value: ${{ coalesce(variables['System.PullRequest.TargetBranch'], variables['Build.SourceBranch'], 'refs/heads/main') }}
# Any new -Svc alternative pools should have variables added here to allow for splitting work
# Main branch pools
- ${{ if ne(contains(variables['BranchNameForPoolSelection'], 'release'), true) }}:
- name: DncEngPublicBuildPool
value: NetCore-Public
- name: DncEngInternalBuildPool
value: NetCore1ESPool-Internal
# Release branch pools
- ${{ if contains(variables['BranchNameForPoolSelection'], 'release') }}:
- name: DncEngPublicBuildPool
value: NetCore-Svc-Public
- name: DncEngInternalBuildPool
value: NetCore1ESPool-Svc-Internal

19
eng/common/tools.ps1

@ -365,10 +365,17 @@ function InitializeVisualStudioMSBuild([bool]$install, [object]$vsRequirements =
# If the version of msbuild is going to be xcopied, # If the version of msbuild is going to be xcopied,
# use this version. Version matches a package here: # use this version. Version matches a package here:
# https://dev.azure.com/dnceng/public/_packaging?_a=package&feed=dotnet-eng&package=RoslynTools.MSBuild&protocolType=NuGet&version=17.1.0&view=overview # https://dev.azure.com/dnceng/public/_packaging?_a=package&feed=dotnet-eng&package=RoslynTools.MSBuild&protocolType=NuGet&version=17.4.1&view=overview
$defaultXCopyMSBuildVersion = '17.1.0' $defaultXCopyMSBuildVersion = '17.4.1'
if (!$vsRequirements) { $vsRequirements = $GlobalJson.tools.vs } if (!$vsRequirements) {
if (Get-Member -InputObject $GlobalJson.tools -Name 'vs') {
$vsRequirements = $GlobalJson.tools.vs
}
else {
$vsRequirements = New-Object PSObject -Property @{ version = $vsMinVersionReqdStr }
}
}
$vsMinVersionStr = if ($vsRequirements.version) { $vsRequirements.version } else { $vsMinVersionReqdStr } $vsMinVersionStr = if ($vsRequirements.version) { $vsRequirements.version } else { $vsMinVersionReqdStr }
$vsMinVersion = [Version]::new($vsMinVersionStr) $vsMinVersion = [Version]::new($vsMinVersionStr)
@ -406,6 +413,7 @@ function InitializeVisualStudioMSBuild([bool]$install, [object]$vsRequirements =
if($vsMinVersion -lt $vsMinVersionReqd){ if($vsMinVersion -lt $vsMinVersionReqd){
Write-Host "Using xcopy-msbuild version of $defaultXCopyMSBuildVersion since VS version $vsMinVersionStr provided in global.json is not compatible" Write-Host "Using xcopy-msbuild version of $defaultXCopyMSBuildVersion since VS version $vsMinVersionStr provided in global.json is not compatible"
$xcopyMSBuildVersion = $defaultXCopyMSBuildVersion $xcopyMSBuildVersion = $defaultXCopyMSBuildVersion
$vsMajorVersion = $xcopyMSBuildVersion.Split('.')[0]
} }
else{ else{
# If the VS version IS compatible, look for an xcopy msbuild package # If the VS version IS compatible, look for an xcopy msbuild package
@ -635,7 +643,7 @@ function InitializeNativeTools() {
InstallDirectory = "$ToolsDir" InstallDirectory = "$ToolsDir"
} }
} }
if (Test-Path variable:NativeToolsOnMachine) { if ($env:NativeToolsOnMachine) {
Write-Host "Variable NativeToolsOnMachine detected, enabling native tool path promotion..." Write-Host "Variable NativeToolsOnMachine detected, enabling native tool path promotion..."
$nativeArgs += @{ PathPromotion = $true } $nativeArgs += @{ PathPromotion = $true }
} }
@ -807,7 +815,8 @@ function MSBuild-Core() {
Write-Host "See log: $buildLog" -ForegroundColor DarkGray Write-Host "See log: $buildLog" -ForegroundColor DarkGray
} }
if ($ci) { # When running on Azure Pipelines, override the returned exit code to avoid double logging.
if ($ci -and $env:SYSTEM_TEAMPROJECT -ne $null) {
Write-PipelineSetResult -Result "Failed" -Message "msbuild execution failed." Write-PipelineSetResult -Result "Failed" -Message "msbuild execution failed."
# Exiting with an exit code causes the azure pipelines task to log yet another "noise" error # Exiting with an exit code causes the azure pipelines task to log yet another "noise" error
# The above Write-PipelineSetResult will cause the task to be marked as failure without adding yet another error # The above Write-PipelineSetResult will cause the task to be marked as failure without adding yet another error

24
eng/common/tools.sh

@ -416,13 +416,6 @@ function MSBuild {
export NUGET_PLUGIN_REQUEST_TIMEOUT_IN_SECONDS=20 export NUGET_PLUGIN_REQUEST_TIMEOUT_IN_SECONDS=20
Write-PipelineSetVariable -name "NUGET_PLUGIN_HANDSHAKE_TIMEOUT_IN_SECONDS" -value "20" Write-PipelineSetVariable -name "NUGET_PLUGIN_HANDSHAKE_TIMEOUT_IN_SECONDS" -value "20"
Write-PipelineSetVariable -name "NUGET_PLUGIN_REQUEST_TIMEOUT_IN_SECONDS" -value "20" Write-PipelineSetVariable -name "NUGET_PLUGIN_REQUEST_TIMEOUT_IN_SECONDS" -value "20"
export NUGET_ENABLE_EXPERIMENTAL_HTTP_RETRY=true
export NUGET_EXPERIMENTAL_MAX_NETWORK_TRY_COUNT=6
export NUGET_EXPERIMENTAL_NETWORK_RETRY_DELAY_MILLISECONDS=1000
Write-PipelineSetVariable -name "NUGET_ENABLE_EXPERIMENTAL_HTTP_RETRY" -value "true"
Write-PipelineSetVariable -name "NUGET_EXPERIMENTAL_MAX_NETWORK_TRY_COUNT" -value "6"
Write-PipelineSetVariable -name "NUGET_EXPERIMENTAL_NETWORK_RETRY_DELAY_MILLISECONDS" -value "1000"
fi fi
local toolset_dir="${_InitializeToolset%/*}" local toolset_dir="${_InitializeToolset%/*}"
@ -479,7 +472,9 @@ function MSBuild-Core {
# We should not Write-PipelineTaskError here because that message shows up in the build summary # We should not Write-PipelineTaskError here because that message shows up in the build summary
# The build already logged an error, that's the reason it failed. Producing an error here only adds noise. # The build already logged an error, that's the reason it failed. Producing an error here only adds noise.
echo "Build failed with exit code $exit_code. Check errors above." echo "Build failed with exit code $exit_code. Check errors above."
if [[ "$ci" == "true" ]]; then
# When running on Azure Pipelines, override the returned exit code to avoid double logging.
if [[ "$ci" == "true" && -n ${SYSTEM_TEAMPROJECT:-} ]]; then
Write-PipelineSetResult -result "Failed" -message "msbuild execution failed." Write-PipelineSetResult -result "Failed" -message "msbuild execution failed."
# Exiting with an exit code causes the azure pipelines task to log yet another "noise" error # Exiting with an exit code causes the azure pipelines task to log yet another "noise" error
# The above Write-PipelineSetResult will cause the task to be marked as failure without adding yet another error # The above Write-PipelineSetResult will cause the task to be marked as failure without adding yet another error
@ -493,6 +488,17 @@ function MSBuild-Core {
RunBuildTool "$_InitializeBuildToolCommand" /m /nologo /clp:Summary /v:$verbosity /nr:$node_reuse $warnaserror_switch /p:TreatWarningsAsErrors=$warn_as_error /p:ContinuousIntegrationBuild=$ci "$@" RunBuildTool "$_InitializeBuildToolCommand" /m /nologo /clp:Summary /v:$verbosity /nr:$node_reuse $warnaserror_switch /p:TreatWarningsAsErrors=$warn_as_error /p:ContinuousIntegrationBuild=$ci "$@"
} }
function GetDarc {
darc_path="$temp_dir/darc"
version="$1"
if [[ -n "$version" ]]; then
version="--darcversion $version"
fi
"$eng_root/common/darc-init.sh" --toolpath "$darc_path" $version
}
ResolvePath "${BASH_SOURCE[0]}" ResolvePath "${BASH_SOURCE[0]}"
_script_dir=`dirname "$_ResolvePath"` _script_dir=`dirname "$_ResolvePath"`
@ -511,7 +517,7 @@ global_json_file="${repo_root}global.json"
# determine if global.json contains a "runtimes" entry # determine if global.json contains a "runtimes" entry
global_json_has_runtimes=false global_json_has_runtimes=false
if command -v jq &> /dev/null; then if command -v jq &> /dev/null; then
if jq -er '. | select(has("runtimes"))' "$global_json_file" &> /dev/null; then if jq -e '.tools | has("runtimes")' "$global_json_file" &> /dev/null; then
global_json_has_runtimes=true global_json_has_runtimes=true
fi fi
elif [[ "$(cat "$global_json_file")" =~ \"runtimes\"[[:space:]\:]*\{ ]]; then elif [[ "$(cat "$global_json_file")" =~ \"runtimes\"[[:space:]\:]*\{ ]]; then

4
global.json

@ -3,7 +3,7 @@
"allowPrerelease": true "allowPrerelease": true
}, },
"tools": { "tools": {
"dotnet": "7.0.100-preview.2.22153.17", "dotnet": "7.0.100",
"runtimes": { "runtimes": {
"dotnet": [ "dotnet": [
"6.0.0" "6.0.0"
@ -14,6 +14,6 @@
} }
}, },
"msbuild-sdks": { "msbuild-sdks": {
"Microsoft.DotNet.Arcade.Sdk": "7.0.0-beta.22327.2" "Microsoft.DotNet.Arcade.Sdk": "8.0.0-beta.22621.1"
} }
} }

1
test/E2ETest/TyeRunTests.cs

@ -1,6 +1,7 @@
// Licensed to the .NET Foundation under one or more agreements. // Licensed to the .NET Foundation under one or more agreements.
// The .NET Foundation licenses this file to you under the MIT license. // The .NET Foundation licenses this file to you under the MIT license.
// See the LICENSE file in the project root for more information. // See the LICENSE file in the project root for more information.
#nullable disable
using System; using System;
using System.Collections.Generic; using System.Collections.Generic;

1
test/UnitTests/Microsoft.Tye.UnitTests.csproj

@ -10,7 +10,6 @@
</PropertyGroup> </PropertyGroup>
<ItemGroup> <ItemGroup>
<PackageReference Include="coverlet.collector" Version="1.0.1" />
<PackageReference Include="coverlet.collector" Version="1.0.1" /> <PackageReference Include="coverlet.collector" Version="1.0.1" />
<PackageReference Include="coverlet.msbuild" Version="2.8.0"> <PackageReference Include="coverlet.msbuild" Version="2.8.0">
<IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets> <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>

Loading…
Cancel
Save