Browse Source

Update Arcade and bump the .NET SDK to 6.0.100-rc.1.21458.32

pull/1335/head
Kévin Chalet 5 years ago
parent
commit
aaba2d0008
  1. 1
      Directory.Build.props
  2. 8
      eng/Version.Details.xml
  3. 5
      eng/common/SetupNugetSources.ps1
  4. 4
      eng/common/SetupNugetSources.sh
  5. 4
      eng/common/build.ps1
  6. 11
      eng/common/build.sh
  7. 2
      eng/common/cross/arm64/tizen-fetch.sh
  8. 43
      eng/common/cross/armel/armel.jessie.patch
  9. 2
      eng/common/cross/armel/tizen-fetch.sh
  10. 2
      eng/common/cross/build-android-rootfs.sh
  11. 56
      eng/common/cross/build-rootfs.sh
  12. 11
      eng/common/cross/s390x/sources.list.bionic
  13. 11
      eng/common/cross/toolchain.cmake
  14. 2
      eng/common/darc-init.sh
  15. 13
      eng/common/dotnet-install.sh
  16. 117
      eng/common/generate-locproject.ps1
  17. 113
      eng/common/init-tools-native.sh
  18. 8
      eng/common/internal-feed-operations.ps1
  19. 6
      eng/common/internal-feed-operations.sh
  20. 1
      eng/common/msbuild.ps1
  21. 2
      eng/common/msbuild.sh
  22. 2
      eng/common/native/CommonLibrary.psm1
  23. 6
      eng/common/native/install-cmake-test.sh
  24. 6
      eng/common/native/install-cmake.sh
  25. 2
      eng/common/native/install-tool.ps1
  26. 104
      eng/common/pipeline-logging-functions.ps1
  27. 34
      eng/common/pipeline-logging-functions.sh
  28. 6
      eng/common/post-build/post-build-utils.ps1
  29. 14
      eng/common/post-build/publish-using-darc.ps1
  30. 95
      eng/common/post-build/sourcelink-validation.ps1
  31. 182
      eng/common/post-build/symbols-validation.ps1
  32. 8
      eng/common/sdk-task.ps1
  33. 6
      eng/common/sdl/execute-all-sdl-tools.ps1
  34. 16
      eng/common/sdl/init-sdl.ps1
  35. 2
      eng/common/sdl/packages.config
  36. 2
      eng/common/sdl/run-sdl.ps1
  37. 4
      eng/common/templates/job/execute-sdl.yml
  38. 18
      eng/common/templates/job/job.yml
  39. 93
      eng/common/templates/job/onelocbuild.yml
  40. 8
      eng/common/templates/job/publish-build-assets.yml
  41. 11
      eng/common/templates/job/source-build.yml
  42. 62
      eng/common/templates/job/source-index-stage1.yml
  43. 29
      eng/common/templates/jobs/jobs.yml
  44. 12
      eng/common/templates/jobs/source-build.yml
  45. 1
      eng/common/templates/phases/publish-build-assets.yml
  46. 8
      eng/common/templates/post-build/channels/generic-internal-channel.yml
  47. 8
      eng/common/templates/post-build/channels/generic-public-channel.yml
  48. 391
      eng/common/templates/post-build/post-build.yml
  49. 1
      eng/common/templates/post-build/setup-maestro-vars.yml
  50. 4
      eng/common/templates/steps/send-to-helix.yml
  51. 7
      eng/common/templates/steps/source-build.yml
  52. 164
      eng/common/tools.ps1
  53. 104
      eng/common/tools.sh
  54. 9
      global.json

1
Directory.Build.props

@ -42,6 +42,7 @@
<PropertyGroup> <PropertyGroup>
<RepoRelativeProjectDir>$([MSBuild]::MakeRelative($(RepoRoot), $(MSBuildProjectDirectory)))</RepoRelativeProjectDir> <RepoRelativeProjectDir>$([MSBuild]::MakeRelative($(RepoRoot), $(MSBuildProjectDirectory)))</RepoRelativeProjectDir>
<WarnOnPackingNonPackableProject>false</WarnOnPackingNonPackableProject> <WarnOnPackingNonPackableProject>false</WarnOnPackingNonPackableProject>
<CheckEolTargetFramework>false</CheckEolTargetFramework>
</PropertyGroup> </PropertyGroup>
<!-- <!--

8
eng/Version.Details.xml

@ -5,14 +5,14 @@
</ProductDependencies> </ProductDependencies>
<ToolsetDependencies> <ToolsetDependencies>
<Dependency Name="Microsoft.DotNet.Arcade.Sdk" Version="5.0.0-beta.20510.1"> <Dependency Name="Microsoft.DotNet.Arcade.Sdk" Version="6.0.0-beta.21316.1">
<Uri>https://github.com/dotnet/arcade</Uri> <Uri>https://github.com/dotnet/arcade</Uri>
<Sha>6813f5aa511a7a4498fa217a54219b5704a01f83</Sha> <Sha>bdcc258d0d4a547be51703795a140fa6b1f68c57</Sha>
</Dependency> </Dependency>
<Dependency Name="Microsoft.DotNet.Helix.Sdk" Version="5.0.0-beta.20510.1"> <Dependency Name="Microsoft.DotNet.Helix.Sdk" Version="6.0.0-beta.21316.1">
<Uri>https://github.com/dotnet/arcade</Uri> <Uri>https://github.com/dotnet/arcade</Uri>
<Sha>6813f5aa511a7a4498fa217a54219b5704a01f83</Sha> <Sha>bdcc258d0d4a547be51703795a140fa6b1f68c57</Sha>
</Dependency> </Dependency>
</ToolsetDependencies> </ToolsetDependencies>

5
eng/common/SetupNugetSources.ps1

@ -99,8 +99,9 @@ function InsertMaestroPrivateFeedCredentials($Sources, $Creds, $Username, $Passw
function EnablePrivatePackageSources($DisabledPackageSources) { function EnablePrivatePackageSources($DisabledPackageSources) {
$maestroPrivateSources = $DisabledPackageSources.SelectNodes("add[contains(@key,'darc-int')]") $maestroPrivateSources = $DisabledPackageSources.SelectNodes("add[contains(@key,'darc-int')]")
ForEach ($DisabledPackageSource in $maestroPrivateSources) { ForEach ($DisabledPackageSource in $maestroPrivateSources) {
Write-Host "`tEnsuring private source '$($DisabledPackageSource.key)' is enabled" Write-Host "`tEnsuring private source '$($DisabledPackageSource.key)' is enabled by deleting it from disabledPackageSource"
$DisabledPackageSource.SetAttribute("value", "false") # Due to https://github.com/NuGet/Home/issues/10291, we must actually remove the disabled entries
$DisabledPackageSources.RemoveChild($DisabledPackageSource)
} }
} }

4
eng/common/SetupNugetSources.sh

@ -158,8 +158,8 @@ if [ "$?" == "0" ]; then
for DisabledSourceName in ${DisabledDarcIntSources[@]} ; do for DisabledSourceName in ${DisabledDarcIntSources[@]} ; do
if [[ $DisabledSourceName == darc-int* ]] if [[ $DisabledSourceName == darc-int* ]]
then then
OldDisableValue="add key=\"$DisabledSourceName\" value=\"true\"" OldDisableValue="<add key=\"$DisabledSourceName\" value=\"true\" />"
NewDisableValue="add key=\"$DisabledSourceName\" value=\"false\"" NewDisableValue="<!-- Reenabled for build : $DisabledSourceName -->"
sed -i.bak "s|$OldDisableValue|$NewDisableValue|" $ConfigFile sed -i.bak "s|$OldDisableValue|$NewDisableValue|" $ConfigFile
echo "Neutralized disablePackageSources entry for '$DisabledSourceName'" echo "Neutralized disablePackageSources entry for '$DisabledSourceName'"
fi fi

4
eng/common/build.ps1

@ -7,7 +7,6 @@ Param(
[string] $msbuildEngine = $null, [string] $msbuildEngine = $null,
[bool] $warnAsError = $true, [bool] $warnAsError = $true,
[bool] $nodeReuse = $true, [bool] $nodeReuse = $true,
[bool] $useDefaultDotnetInstall = $false,
[switch][Alias('r')]$restore, [switch][Alias('r')]$restore,
[switch] $deployDeps, [switch] $deployDeps,
[switch][Alias('b')]$build, [switch][Alias('b')]$build,
@ -26,6 +25,7 @@ Param(
[switch] $prepareMachine, [switch] $prepareMachine,
[string] $runtimeSourceFeed = '', [string] $runtimeSourceFeed = '',
[string] $runtimeSourceFeedKey = '', [string] $runtimeSourceFeedKey = '',
[switch] $excludePrereleaseVS,
[switch] $help, [switch] $help,
[Parameter(ValueFromRemainingArguments=$true)][String[]]$properties [Parameter(ValueFromRemainingArguments=$true)][String[]]$properties
) )
@ -66,7 +66,7 @@ function Print-Usage() {
Write-Host " -prepareMachine Prepare machine for CI run, clean up processes after build" Write-Host " -prepareMachine Prepare machine for CI run, clean up processes after build"
Write-Host " -warnAsError <value> Sets warnaserror msbuild parameter ('true' or 'false')" Write-Host " -warnAsError <value> Sets warnaserror msbuild parameter ('true' or 'false')"
Write-Host " -msbuildEngine <value> Msbuild engine to use to run build ('dotnet', 'vs', or unspecified)." Write-Host " -msbuildEngine <value> Msbuild engine to use to run build ('dotnet', 'vs', or unspecified)."
Write-Host " -useDefaultDotnetInstall <value> Use dotnet-install.* scripts from public location as opposed to from eng common folder" Write-Host " -excludePrereleaseVS Set to exclude build engines in prerelease versions of Visual Studio"
Write-Host "" Write-Host ""
Write-Host "Command line arguments not listed above are passed thru to msbuild." Write-Host "Command line arguments not listed above are passed thru to msbuild."

11
eng/common/build.sh

@ -36,8 +36,6 @@ usage()
echo " --prepareMachine Prepare machine for CI run, clean up processes after build" echo " --prepareMachine Prepare machine for CI run, clean up processes after build"
echo " --nodeReuse <value> Sets nodereuse msbuild parameter ('true' or 'false')" echo " --nodeReuse <value> Sets nodereuse msbuild parameter ('true' or 'false')"
echo " --warnAsError <value> Sets warnaserror msbuild parameter ('true' or 'false')" echo " --warnAsError <value> Sets warnaserror msbuild parameter ('true' or 'false')"
echo " --useDefaultDotnetInstall <value> Use dotnet-install.* scripts from public location as opposed to from eng common folder"
echo "" echo ""
echo "Command line arguments not listed above are passed thru to msbuild." echo "Command line arguments not listed above are passed thru to msbuild."
echo "Arguments can also be passed in with a single hyphen." echo "Arguments can also be passed in with a single hyphen."
@ -80,11 +78,10 @@ prepare_machine=false
verbosity='minimal' verbosity='minimal'
runtime_source_feed='' runtime_source_feed=''
runtime_source_feed_key='' runtime_source_feed_key=''
use_default_dotnet_install=false
properties='' properties=''
while [[ $# > 0 ]]; do while [[ $# > 0 ]]; do
opt="$(echo "${1/#--/-}" | awk '{print tolower($0)}')" opt="$(echo "${1/#--/-}" | tr "[:upper:]" "[:lower:]")"
case "$opt" in case "$opt" in
-help|-h) -help|-h)
usage usage
@ -159,14 +156,10 @@ while [[ $# > 0 ]]; do
runtime_source_feed=$2 runtime_source_feed=$2
shift shift
;; ;;
-runtimesourcefeedkey) -runtimesourcefeedkey)
runtime_source_feed_key=$2 runtime_source_feed_key=$2
shift shift
;; ;;
-usedefaultdotnetinstall)
use_default_dotnet_install=$2
shift
;;
*) *)
properties="$properties $1" properties="$properties $1"
;; ;;

2
eng/common/cross/arm64/tizen-fetch.sh

@ -157,7 +157,7 @@ fetch_tizen_pkgs()
Inform "Initialize arm base" Inform "Initialize arm base"
fetch_tizen_pkgs_init standard base fetch_tizen_pkgs_init standard base
Inform "fetch common packages" Inform "fetch common packages"
fetch_tizen_pkgs aarch64 gcc glibc glibc-devel libicu libicu-devel libatomic linux-glibc-devel fetch_tizen_pkgs aarch64 gcc glibc glibc-devel libicu libicu-devel libatomic linux-glibc-devel keyutils keyutils-devel libkeyutils
Inform "fetch coreclr packages" Inform "fetch coreclr packages"
fetch_tizen_pkgs aarch64 lldb lldb-devel libgcc libstdc++ libstdc++-devel libunwind libunwind-devel lttng-ust-devel lttng-ust userspace-rcu-devel userspace-rcu fetch_tizen_pkgs aarch64 lldb lldb-devel libgcc libstdc++ libstdc++-devel libunwind libunwind-devel lttng-ust-devel lttng-ust userspace-rcu-devel userspace-rcu
Inform "fetch corefx packages" Inform "fetch corefx packages"

43
eng/common/cross/armel/armel.jessie.patch

@ -0,0 +1,43 @@
diff -u -r a/usr/include/urcu/uatomic/generic.h b/usr/include/urcu/uatomic/generic.h
--- a/usr/include/urcu/uatomic/generic.h 2014-10-22 15:00:58.000000000 -0700
+++ b/usr/include/urcu/uatomic/generic.h 2020-10-30 21:38:28.550000000 -0700
@@ -69,10 +69,10 @@
#endif
#ifdef UATOMIC_HAS_ATOMIC_SHORT
case 2:
- return __sync_val_compare_and_swap_2(addr, old, _new);
+ return __sync_val_compare_and_swap_2((uint16_t*) addr, old, _new);
#endif
case 4:
- return __sync_val_compare_and_swap_4(addr, old, _new);
+ return __sync_val_compare_and_swap_4((uint32_t*) addr, old, _new);
#if (CAA_BITS_PER_LONG == 64)
case 8:
return __sync_val_compare_and_swap_8(addr, old, _new);
@@ -109,7 +109,7 @@
return;
#endif
case 4:
- __sync_and_and_fetch_4(addr, val);
+ __sync_and_and_fetch_4((uint32_t*) addr, val);
return;
#if (CAA_BITS_PER_LONG == 64)
case 8:
@@ -148,7 +148,7 @@
return;
#endif
case 4:
- __sync_or_and_fetch_4(addr, val);
+ __sync_or_and_fetch_4((uint32_t*) addr, val);
return;
#if (CAA_BITS_PER_LONG == 64)
case 8:
@@ -187,7 +187,7 @@
return __sync_add_and_fetch_2(addr, val);
#endif
case 4:
- return __sync_add_and_fetch_4(addr, val);
+ return __sync_add_and_fetch_4((uint32_t*) addr, val);
#if (CAA_BITS_PER_LONG == 64)
case 8:
return __sync_add_and_fetch_8(addr, val);

2
eng/common/cross/armel/tizen-fetch.sh

@ -157,7 +157,7 @@ fetch_tizen_pkgs()
Inform "Initialize arm base" Inform "Initialize arm base"
fetch_tizen_pkgs_init standard base fetch_tizen_pkgs_init standard base
Inform "fetch common packages" Inform "fetch common packages"
fetch_tizen_pkgs armv7l gcc glibc glibc-devel libicu libicu-devel libatomic linux-glibc-devel fetch_tizen_pkgs armv7l gcc gcc-devel-static glibc glibc-devel libicu libicu-devel libatomic linux-glibc-devel keyutils keyutils-devel libkeyutils
Inform "fetch coreclr packages" Inform "fetch coreclr packages"
fetch_tizen_pkgs armv7l lldb lldb-devel libgcc libstdc++ libstdc++-devel libunwind libunwind-devel lttng-ust-devel lttng-ust userspace-rcu-devel userspace-rcu fetch_tizen_pkgs armv7l lldb lldb-devel libgcc libstdc++ libstdc++-devel libunwind libunwind-devel lttng-ust-devel lttng-ust userspace-rcu-devel userspace-rcu
Inform "fetch corefx packages" Inform "fetch corefx packages"

2
eng/common/cross/build-android-rootfs.sh

@ -27,7 +27,7 @@ __AndroidToolchain=aarch64-linux-android
for i in "$@" for i in "$@"
do do
lowerI="$(echo $i | awk '{print tolower($0)}')" lowerI="$(echo $i | tr "[:upper:]" "[:lower:]")"
case $lowerI in case $lowerI in
-?|-h|--help) -?|-h|--help)
usage usage

56
eng/common/cross/build-rootfs.sh

@ -6,7 +6,7 @@ usage()
{ {
echo "Usage: $0 [BuildArch] [CodeName] [lldbx.y] [--skipunmount] --rootfsdir <directory>]" echo "Usage: $0 [BuildArch] [CodeName] [lldbx.y] [--skipunmount] --rootfsdir <directory>]"
echo "BuildArch can be: arm(default), armel, arm64, x86" echo "BuildArch can be: arm(default), armel, arm64, x86"
echo "CodeName - optional, Code name for Linux, can be: trusty, xenial(default), zesty, bionic, alpine. If BuildArch is armel, LinuxCodeName is jessie(default) or tizen." echo "CodeName - optional, Code name for Linux, can be: trusty, xenial(default), zesty, bionic, alpine, alpine3.9 or alpine3.13. If BuildArch is armel, LinuxCodeName is jessie(default) or tizen."
echo " for FreeBSD can be: freebsd11 or freebsd12." echo " for FreeBSD can be: freebsd11 or freebsd12."
echo " for illumos can be: illumos." echo " for illumos can be: illumos."
echo "lldbx.y - optional, LLDB version, can be: lldb3.9(default), lldb4.0, lldb5.0, lldb6.0 no-lldb. Ignored for alpine and FReeBSD" echo "lldbx.y - optional, LLDB version, can be: lldb3.9(default), lldb4.0, lldb5.0, lldb6.0 no-lldb. Ignored for alpine and FReeBSD"
@ -74,6 +74,10 @@ __IllumosPackages+=" mit-krb5-1.16.2nb4"
__IllumosPackages+=" openssl-1.1.1e" __IllumosPackages+=" openssl-1.1.1e"
__IllumosPackages+=" zlib-1.2.11" __IllumosPackages+=" zlib-1.2.11"
# ML.NET dependencies
__UbuntuPackages+=" libomp5"
__UbuntuPackages+=" libomp-dev"
__UseMirror=0 __UseMirror=0
__UnprocessedBuildArgs= __UnprocessedBuildArgs=
@ -82,7 +86,7 @@ while :; do
break break
fi fi
lowerI="$(echo $1 | awk '{print tolower($0)}')" lowerI="$(echo $1 | tr "[:upper:]" "[:lower:]")"
case $lowerI in case $lowerI in
-?|-h|--help) -?|-h|--help)
usage usage
@ -106,6 +110,13 @@ while :; do
__UbuntuRepo="http://ftp.debian.org/debian/" __UbuntuRepo="http://ftp.debian.org/debian/"
__CodeName=jessie __CodeName=jessie
;; ;;
s390x)
__BuildArch=s390x
__UbuntuArch=s390x
__UbuntuRepo="http://ports.ubuntu.com/ubuntu-ports/"
__UbuntuPackages=$(echo ${__UbuntuPackages} | sed 's/ libunwind8-dev//')
unset __LLDB_Package
;;
x86) x86)
__BuildArch=x86 __BuildArch=x86
__UbuntuArch=i386 __UbuntuArch=i386
@ -176,9 +187,20 @@ while :; do
__UbuntuRepo= __UbuntuRepo=
__Tizen=tizen __Tizen=tizen
;; ;;
alpine) alpine|alpine3.9)
__CodeName=alpine
__UbuntuRepo=
__AlpineVersion=3.9
;;
alpine3.13)
__CodeName=alpine __CodeName=alpine
__UbuntuRepo= __UbuntuRepo=
__AlpineVersion=3.13
# Alpine 3.13 has all the packages we need in the 3.13 repository
__AlpinePackages+=$__AlpinePackagesEdgeCommunity
__AlpinePackagesEdgeCommunity=
__AlpinePackages+=$__AlpinePackagesEdgeMain
__AlpinePackagesEdgeMain=
;; ;;
freebsd11) freebsd11)
__FreeBSDBase="11.3-RELEASE" __FreeBSDBase="11.3-RELEASE"
@ -236,7 +258,6 @@ __RootfsDir="$( cd "$__RootfsDir" && pwd )"
if [[ "$__CodeName" == "alpine" ]]; then if [[ "$__CodeName" == "alpine" ]]; then
__ApkToolsVersion=2.9.1 __ApkToolsVersion=2.9.1
__AlpineVersion=3.9
__ApkToolsDir=$(mktemp -d) __ApkToolsDir=$(mktemp -d)
wget https://github.com/alpinelinux/apk-tools/releases/download/v$__ApkToolsVersion/apk-tools-$__ApkToolsVersion-x86_64-linux.tar.gz -P $__ApkToolsDir wget https://github.com/alpinelinux/apk-tools/releases/download/v$__ApkToolsVersion/apk-tools-$__ApkToolsVersion-x86_64-linux.tar.gz -P $__ApkToolsDir
tar -xf $__ApkToolsDir/apk-tools-$__ApkToolsVersion-x86_64-linux.tar.gz -C $__ApkToolsDir tar -xf $__ApkToolsDir/apk-tools-$__ApkToolsVersion-x86_64-linux.tar.gz -C $__ApkToolsDir
@ -249,15 +270,19 @@ if [[ "$__CodeName" == "alpine" ]]; then
-U --allow-untrusted --root $__RootfsDir --arch $__AlpineArch --initdb \ -U --allow-untrusted --root $__RootfsDir --arch $__AlpineArch --initdb \
add $__AlpinePackages add $__AlpinePackages
$__ApkToolsDir/apk-tools-$__ApkToolsVersion/apk \ if [[ -n "$__AlpinePackagesEdgeMain" ]]; then
-X http://dl-cdn.alpinelinux.org/alpine/edge/main \ $__ApkToolsDir/apk-tools-$__ApkToolsVersion/apk \
-U --allow-untrusted --root $__RootfsDir --arch $__AlpineArch --initdb \ -X http://dl-cdn.alpinelinux.org/alpine/edge/main \
add $__AlpinePackagesEdgeMain -U --allow-untrusted --root $__RootfsDir --arch $__AlpineArch --initdb \
add $__AlpinePackagesEdgeMain
fi
$__ApkToolsDir/apk-tools-$__ApkToolsVersion/apk \ if [[ -n "$__AlpinePackagesEdgeCommunity" ]]; then
-X http://dl-cdn.alpinelinux.org/alpine/edge/community \ $__ApkToolsDir/apk-tools-$__ApkToolsVersion/apk \
-U --allow-untrusted --root $__RootfsDir --arch $__AlpineArch --initdb \ -X http://dl-cdn.alpinelinux.org/alpine/edge/community \
add $__AlpinePackagesEdgeCommunity -U --allow-untrusted --root $__RootfsDir --arch $__AlpineArch --initdb \
add $__AlpinePackagesEdgeCommunity
fi
rm -r $__ApkToolsDir rm -r $__ApkToolsDir
elif [[ "$__CodeName" == "freebsd" ]]; then elif [[ "$__CodeName" == "freebsd" ]]; then
@ -329,6 +354,7 @@ elif [[ -n $__CodeName ]]; then
chroot $__RootfsDir apt-get -f -y install chroot $__RootfsDir apt-get -f -y install
chroot $__RootfsDir apt-get -y install $__UbuntuPackages chroot $__RootfsDir apt-get -y install $__UbuntuPackages
chroot $__RootfsDir symlinks -cr /usr chroot $__RootfsDir symlinks -cr /usr
chroot $__RootfsDir apt-get clean
if [ $__SkipUnmount == 0 ]; then if [ $__SkipUnmount == 0 ]; then
umount $__RootfsDir/* || true umount $__RootfsDir/* || true
@ -340,6 +366,12 @@ elif [[ -n $__CodeName ]]; then
patch -p1 < $__CrossDir/$__BuildArch/trusty-lttng-2.4.patch patch -p1 < $__CrossDir/$__BuildArch/trusty-lttng-2.4.patch
popd popd
fi fi
if [[ "$__BuildArch" == "armel" && "$__CodeName" == "jessie" ]]; then
pushd $__RootfsDir
patch -p1 < $__CrossDir/$__BuildArch/armel.jessie.patch
popd
fi
elif [[ "$__Tizen" == "tizen" ]]; then elif [[ "$__Tizen" == "tizen" ]]; then
ROOTFS_DIR=$__RootfsDir $__CrossDir/$__BuildArch/tizen-build-rootfs.sh ROOTFS_DIR=$__RootfsDir $__CrossDir/$__BuildArch/tizen-build-rootfs.sh
else else

11
eng/common/cross/s390x/sources.list.bionic

@ -0,0 +1,11 @@
deb http://ports.ubuntu.com/ubuntu-ports/ bionic main restricted universe
deb-src http://ports.ubuntu.com/ubuntu-ports/ bionic main restricted universe
deb http://ports.ubuntu.com/ubuntu-ports/ bionic-updates main restricted universe
deb-src http://ports.ubuntu.com/ubuntu-ports/ bionic-updates main restricted universe
deb http://ports.ubuntu.com/ubuntu-ports/ bionic-backports main restricted
deb-src http://ports.ubuntu.com/ubuntu-ports/ bionic-backports main restricted
deb http://ports.ubuntu.com/ubuntu-ports/ bionic-security main restricted universe multiverse
deb-src http://ports.ubuntu.com/ubuntu-ports/ bionic-security main restricted universe multiverse

11
eng/common/cross/toolchain.cmake

@ -36,6 +36,9 @@ elseif(TARGET_ARCH_NAME STREQUAL "arm64")
if("$ENV{__DistroRid}" MATCHES "tizen.*") if("$ENV{__DistroRid}" MATCHES "tizen.*")
set(TIZEN_TOOLCHAIN "aarch64-tizen-linux-gnu/9.2.0") set(TIZEN_TOOLCHAIN "aarch64-tizen-linux-gnu/9.2.0")
endif() endif()
elseif(TARGET_ARCH_NAME STREQUAL "s390x")
set(CMAKE_SYSTEM_PROCESSOR s390x)
set(TOOLCHAIN "s390x-linux-gnu")
elseif(TARGET_ARCH_NAME STREQUAL "x86") elseif(TARGET_ARCH_NAME STREQUAL "x86")
set(CMAKE_SYSTEM_PROCESSOR i686) set(CMAKE_SYSTEM_PROCESSOR i686)
set(TOOLCHAIN "i686-linux-gnu") set(TOOLCHAIN "i686-linux-gnu")
@ -46,7 +49,7 @@ elseif (ILLUMOS)
set(CMAKE_SYSTEM_PROCESSOR "x86_64") set(CMAKE_SYSTEM_PROCESSOR "x86_64")
set(TOOLCHAIN "x86_64-illumos") set(TOOLCHAIN "x86_64-illumos")
else() else()
message(FATAL_ERROR "Arch is ${TARGET_ARCH_NAME}. Only armel, arm, arm64 and x86 are supported!") message(FATAL_ERROR "Arch is ${TARGET_ARCH_NAME}. Only armel, arm, arm64, s390x and x86 are supported!")
endif() endif()
if(DEFINED ENV{TOOLCHAIN}) if(DEFINED ENV{TOOLCHAIN})
@ -139,6 +142,10 @@ function(add_toolchain_linker_flag Flag)
set("CMAKE_SHARED_LINKER_FLAGS${CONFIG_SUFFIX}" "${CMAKE_SHARED_LINKER_FLAGS${CONFIG_SUFFIX}} ${Flag}" PARENT_SCOPE) set("CMAKE_SHARED_LINKER_FLAGS${CONFIG_SUFFIX}" "${CMAKE_SHARED_LINKER_FLAGS${CONFIG_SUFFIX}} ${Flag}" PARENT_SCOPE)
endfunction() endfunction()
if(CMAKE_SYSTEM_NAME STREQUAL "Linux")
add_toolchain_linker_flag("-Wl,--rpath-link=${CROSS_ROOTFS}/lib/${TOOLCHAIN}")
add_toolchain_linker_flag("-Wl,--rpath-link=${CROSS_ROOTFS}/usr/lib/${TOOLCHAIN}")
endif()
if(TARGET_ARCH_NAME STREQUAL "armel") if(TARGET_ARCH_NAME STREQUAL "armel")
if(DEFINED TIZEN_TOOLCHAIN) # For Tizen only if(DEFINED TIZEN_TOOLCHAIN) # For Tizen only
@ -167,7 +174,7 @@ endif()
# Specify compile options # Specify compile options
if((TARGET_ARCH_NAME MATCHES "^(arm|armel|arm64)$" AND NOT "$ENV{__DistroRid}" MATCHES "android.*") OR ILLUMOS) if((TARGET_ARCH_NAME MATCHES "^(arm|armel|arm64|s390x)$" AND NOT "$ENV{__DistroRid}" MATCHES "android.*") OR ILLUMOS)
set(CMAKE_C_COMPILER_TARGET ${TOOLCHAIN}) set(CMAKE_C_COMPILER_TARGET ${TOOLCHAIN})
set(CMAKE_CXX_COMPILER_TARGET ${TOOLCHAIN}) set(CMAKE_CXX_COMPILER_TARGET ${TOOLCHAIN})
set(CMAKE_ASM_COMPILER_TARGET ${TOOLCHAIN}) set(CMAKE_ASM_COMPILER_TARGET ${TOOLCHAIN})

2
eng/common/darc-init.sh

@ -6,7 +6,7 @@ versionEndpoint='https://maestro-prod.westus2.cloudapp.azure.com/api/assets/darc
verbosity='minimal' verbosity='minimal'
while [[ $# > 0 ]]; do while [[ $# > 0 ]]; do
opt="$(echo "$1" | awk '{print tolower($0)}')" opt="$(echo "$1" | tr "[:upper:]" "[:lower:]")"
case "$opt" in case "$opt" in
--darcversion) --darcversion)
darcVersion=$2 darcVersion=$2

13
eng/common/dotnet-install.sh

@ -19,7 +19,7 @@ runtime='dotnet'
runtimeSourceFeed='' runtimeSourceFeed=''
runtimeSourceFeedKey='' runtimeSourceFeedKey=''
while [[ $# > 0 ]]; do while [[ $# > 0 ]]; do
opt="$(echo "$1" | awk '{print tolower($0)}')" opt="$(echo "$1" | tr "[:upper:]" "[:lower:]")"
case "$opt" in case "$opt" in
-version|-v) -version|-v)
shift shift
@ -49,13 +49,8 @@ while [[ $# > 0 ]]; do
shift shift
done done
# Use uname to determine what the CPU is. # Use uname to determine what the CPU is, see https://en.wikipedia.org/wiki/Uname#Examples
cpuname=$(uname -p) cpuname=$(uname -m)
# Some Linux platforms report unknown for platform, but the arch for machine.
if [[ "$cpuname" == "unknown" ]]; then
cpuname=$(uname -m)
fi
case $cpuname in case $cpuname in
aarch64) aarch64)
buildarch=arm64 buildarch=arm64
@ -75,7 +70,7 @@ case $cpuname in
;; ;;
esac esac
dotnetRoot="$repo_root/.dotnet" dotnetRoot="${repo_root}.dotnet"
if [[ $architecture != "" ]] && [[ $architecture != $buildarch ]]; then if [[ $architecture != "" ]] && [[ $architecture != $buildarch ]]; then
dotnetRoot="$dotnetRoot/$architecture" dotnetRoot="$dotnetRoot/$architecture"
fi fi

117
eng/common/generate-locproject.ps1

@ -0,0 +1,117 @@
Param(
[Parameter(Mandatory=$true)][string] $SourcesDirectory, # Directory where source files live; if using a Localize directory it should live in here
[string] $LanguageSet = 'VS_Main_Languages', # Language set to be used in the LocProject.json
[switch] $UseCheckedInLocProjectJson, # When set, generates a LocProject.json and compares it to one that already exists in the repo; otherwise just generates one
[switch] $CreateNeutralXlfs # Creates neutral xlf files. Only set to false when running locally
)
# Generates LocProject.json files for the OneLocBuild task. OneLocBuildTask is described here:
# https://ceapex.visualstudio.com/CEINTL/_wiki/wikis/CEINTL.wiki/107/Localization-with-OneLocBuild-Task
Set-StrictMode -Version 2.0
$ErrorActionPreference = "Stop"
. $PSScriptRoot\tools.ps1
Import-Module -Name (Join-Path $PSScriptRoot 'native\CommonLibrary.psm1')
$exclusionsFilePath = "$SourcesDirectory\eng\Localize\LocExclusions.json"
$exclusions = @{ Exclusions = @() }
if (Test-Path -Path $exclusionsFilePath)
{
$exclusions = Get-Content "$exclusionsFilePath" | ConvertFrom-Json
}
Push-Location "$SourcesDirectory" # push location for Resolve-Path -Relative to work
# Template files
$jsonFiles = @()
$jsonTemplateFiles = Get-ChildItem -Recurse -Path "$SourcesDirectory" | Where-Object { $_.FullName -Match "\.template\.config\\localize\\.+\.en\.json" } # .NET templating pattern
$jsonTemplateFiles | ForEach-Object {
$null = $_.Name -Match "(.+)\.[\w-]+\.json" # matches '[filename].[langcode].json
$destinationFile = "$($_.Directory.FullName)\$($Matches.1).json"
$jsonFiles += Copy-Item "$($_.FullName)" -Destination $destinationFile -PassThru
}
$jsonWinformsTemplateFiles = Get-ChildItem -Recurse -Path "$SourcesDirectory" | Where-Object { $_.FullName -Match "en\\strings\.json" } # current winforms pattern
$xlfFiles = @()
$allXlfFiles = Get-ChildItem -Recurse -Path "$SourcesDirectory\*\*.xlf"
$langXlfFiles = @()
if ($allXlfFiles) {
$null = $allXlfFiles[0].FullName -Match "\.([\w-]+)\.xlf" # matches '[langcode].xlf'
$firstLangCode = $Matches.1
$langXlfFiles = Get-ChildItem -Recurse -Path "$SourcesDirectory\*\*.$firstLangCode.xlf"
}
$langXlfFiles | ForEach-Object {
$null = $_.Name -Match "(.+)\.[\w-]+\.xlf" # matches '[filename].[langcode].xlf
$destinationFile = "$($_.Directory.FullName)\$($Matches.1).xlf"
$xlfFiles += Copy-Item "$($_.FullName)" -Destination $destinationFile -PassThru
}
$locFiles = $jsonFiles + $jsonWinformsTemplateFiles + $xlfFiles
$locJson = @{
Projects = @(
@{
LanguageSet = $LanguageSet
LocItems = @(
$locFiles | ForEach-Object {
$outputPath = "$(($_.DirectoryName | Resolve-Path -Relative) + "\")"
$continue = $true
foreach ($exclusion in $exclusions.Exclusions) {
if ($outputPath.Contains($exclusion))
{
$continue = $false
}
}
$sourceFile = ($_.FullName | Resolve-Path -Relative)
if (!$CreateNeutralXlfs -and $_.Extension -eq '.xlf') {
Remove-Item -Path $sourceFile
}
if ($continue)
{
if ($_.Directory.Name -eq 'en' -and $_.Extension -eq '.json') {
return @{
SourceFile = $sourceFile
CopyOption = "LangIDOnPath"
OutputPath = "$($_.Directory.Parent.FullName | Resolve-Path -Relative)\"
}
}
else {
return @{
SourceFile = $sourceFile
CopyOption = "LangIDOnName"
OutputPath = $outputPath
}
}
}
}
)
}
)
}
$json = ConvertTo-Json $locJson -Depth 5
Write-Host "LocProject.json generated:`n`n$json`n`n"
Pop-Location
if (!$UseCheckedInLocProjectJson) {
New-Item "$SourcesDirectory\eng\Localize\LocProject.json" -Force # Need this to make sure the Localize directory is created
Set-Content "$SourcesDirectory\eng\Localize\LocProject.json" $json
}
else {
New-Item "$SourcesDirectory\eng\Localize\LocProject-generated.json" -Force # Need this to make sure the Localize directory is created
Set-Content "$SourcesDirectory\eng\Localize\LocProject-generated.json" $json
if ((Get-FileHash "$SourcesDirectory\eng\Localize\LocProject-generated.json").Hash -ne (Get-FileHash "$SourcesDirectory\eng\Localize\LocProject.json").Hash) {
Write-PipelineTelemetryError -Category "OneLocBuild" -Message "Existing LocProject.json differs from generated LocProject.json. Download LocProject-generated.json and compare them."
exit 1
}
else {
Write-Host "Generated LocProject.json and current LocProject.json are identical."
}
}

113
eng/common/init-tools-native.sh

@ -16,7 +16,7 @@ declare -A native_assets
. $scriptroot/native/common-library.sh . $scriptroot/native/common-library.sh
while (($# > 0)); do while (($# > 0)); do
lowerI="$(echo $1 | awk '{print tolower($0)}')" lowerI="$(echo $1 | tr "[:upper:]" "[:lower:]")"
case $lowerI in case $lowerI in
--baseuri) --baseuri)
base_uri=$2 base_uri=$2
@ -76,24 +76,89 @@ while (($# > 0)); do
done done
function ReadGlobalJsonNativeTools { function ReadGlobalJsonNativeTools {
# Get the native-tools section from the global.json. # happy path: we have a proper JSON parsing tool `jq(1)` in PATH!
local native_tools_section=$(cat $global_json_file | awk '/"native-tools"/,/}/') if command -v jq &> /dev/null; then
# Only extract the contents of the object.
local native_tools_list=$(echo $native_tools_section | awk -F"[{}]" '{print $2}') # jq: read each key/value pair under "native-tools" entry and emit:
native_tools_list=${native_tools_list//[\" ]/} # KEY="<entry-key>" VALUE="<entry-value>"
native_tools_list=$( echo "$native_tools_list" | sed 's/\s//g' | sed 's/,/\n/g' ) # followed by a null byte.
#
local old_IFS=$IFS # bash: read line with null byte delimeter and push to array (for later `eval`uation).
while read -r line; do
# Lines are of the form: 'tool:version' while IFS= read -rd '' line; do
IFS=: native_assets+=("$line")
while read -r key value; do done < <(jq -r '. |
native_assets[$key]=$value select(has("native-tools")) |
done <<< "$line" ."native-tools" |
done <<< "$native_tools_list" keys[] as $k |
IFS=$old_IFS @sh "KEY=\($k) VALUE=\(.[$k])\u0000"' "$global_json_file")
return 0; return
fi
# Warning: falling back to manually parsing JSON, which is not recommended.
# Following routine matches the output and escaping logic of jq(1)'s @sh formatter used above.
# It has been tested with several weird strings with escaped characters in entries (key and value)
# and results were compared with the output of jq(1) in binary representation using xxd(1);
# just before the assignment to 'native_assets' array (above and below).
# try to capture the section under "native-tools".
if [[ ! "$(cat "$global_json_file")" =~ \"native-tools\"[[:space:]\:\{]*([^\}]+) ]]; then
return
fi
section="${BASH_REMATCH[1]}"
parseStarted=0
possibleEnd=0
escaping=0
escaped=0
isKey=1
for (( i=0; i<${#section}; i++ )); do
char="${section:$i:1}"
if ! ((parseStarted)) && [[ "$char" =~ [[:space:],:] ]]; then continue; fi
if ! ((escaping)) && [[ "$char" == "\\" ]]; then
escaping=1
elif ((escaping)) && ! ((escaped)); then
escaped=1
fi
if ! ((parseStarted)) && [[ "$char" == "\"" ]]; then
parseStarted=1
possibleEnd=0
elif [[ "$char" == "'" ]]; then
token="$token'\\\''"
possibleEnd=0
elif ((escaping)) || [[ "$char" != "\"" ]]; then
token="$token$char"
possibleEnd=1
fi
if ((possibleEnd)) && ! ((escaping)) && [[ "$char" == "\"" ]]; then
# Use printf to unescape token to match jq(1)'s @sh formatting rules.
# do not use 'token="$(printf "$token")"' syntax, as $() eats the trailing linefeed.
printf -v token "'$token'"
if ((isKey)); then
KEY="$token"
isKey=0
else
line="KEY=$KEY VALUE=$token"
native_assets+=("$line")
isKey=1
fi
# reset for next token
parseStarted=0
token=
elif ((escaping)) && ((escaped)); then
escaping=0
escaped=0
fi
done
} }
native_base_dir=$install_directory native_base_dir=$install_directory
@ -111,14 +176,14 @@ if [[ ${#native_assets[@]} -eq 0 ]]; then
exit 0; exit 0;
else else
native_installer_dir="$scriptroot/native" native_installer_dir="$scriptroot/native"
for tool in "${!native_assets[@]}" for index in "${!native_assets[@]}"; do
do eval "${native_assets["$index"]}"
tool_version=${native_assets[$tool]}
installer_path="$native_installer_dir/install-$tool.sh" installer_path="$native_installer_dir/install-$KEY.sh"
installer_command="$installer_path" installer_command="$installer_path"
installer_command+=" --baseuri $base_uri" installer_command+=" --baseuri $base_uri"
installer_command+=" --installpath $install_bin" installer_command+=" --installpath $install_bin"
installer_command+=" --version $tool_version" installer_command+=" --version $VALUE"
echo $installer_command echo $installer_command
if [[ $force = true ]]; then if [[ $force = true ]]; then

8
eng/common/internal-feed-operations.ps1

@ -45,11 +45,11 @@ function SetupCredProvider {
# Then, we set the 'VSS_NUGET_EXTERNAL_FEED_ENDPOINTS' environment variable to restore from the stable # Then, we set the 'VSS_NUGET_EXTERNAL_FEED_ENDPOINTS' environment variable to restore from the stable
# feeds successfully # feeds successfully
$nugetConfigPath = "$RepoRoot\NuGet.config" $nugetConfigPath = Join-Path $RepoRoot "NuGet.config"
if (-Not (Test-Path -Path $nugetConfigPath)) { if (-Not (Test-Path -Path $nugetConfigPath)) {
Write-PipelineTelemetryError -Category 'Build' -Message 'NuGet.config file not found in repo root!' Write-PipelineTelemetryError -Category 'Build' -Message 'NuGet.config file not found in repo root!'
ExitWithExitCode 1 ExitWithExitCode 1
} }
$endpoints = New-Object System.Collections.ArrayList $endpoints = New-Object System.Collections.ArrayList
@ -63,8 +63,6 @@ function SetupCredProvider {
} }
if (($endpoints | Measure-Object).Count -gt 0) { if (($endpoints | Measure-Object).Count -gt 0) {
# [SuppressMessage("Microsoft.Security", "CS002:SecretInNextLine", Justification="Endpoint code example with no real credentials.")]
# Create the JSON object. It should look like '{"endpointCredentials": [{"endpoint":"http://example.index.json", "username":"optional", "password":"accesstoken"}]}'
$endpointCredentials = @{endpointCredentials=$endpoints} | ConvertTo-Json -Compress $endpointCredentials = @{endpointCredentials=$endpoints} | ConvertTo-Json -Compress
# Create the environment variables the AzDo way # Create the environment variables the AzDo way
@ -87,7 +85,7 @@ function SetupCredProvider {
#Workaround for https://github.com/microsoft/msbuild/issues/4430 #Workaround for https://github.com/microsoft/msbuild/issues/4430
function InstallDotNetSdkAndRestoreArcade { function InstallDotNetSdkAndRestoreArcade {
$dotnetTempDir = "$RepoRoot\dotnet" $dotnetTempDir = Join-Path $RepoRoot "dotnet"
$dotnetSdkVersion="2.1.507" # After experimentation we know this version works when restoring the SDK (compared to 3.0.*) $dotnetSdkVersion="2.1.507" # After experimentation we know this version works when restoring the SDK (compared to 3.0.*)
$dotnet = "$dotnetTempDir\dotnet.exe" $dotnet = "$dotnetTempDir\dotnet.exe"
$restoreProjPath = "$PSScriptRoot\restore.proj" $restoreProjPath = "$PSScriptRoot\restore.proj"

6
eng/common/internal-feed-operations.sh

@ -39,7 +39,7 @@ function SetupCredProvider {
# Then, we set the 'VSS_NUGET_EXTERNAL_FEED_ENDPOINTS' environment variable to restore from the stable # Then, we set the 'VSS_NUGET_EXTERNAL_FEED_ENDPOINTS' environment variable to restore from the stable
# feeds successfully # feeds successfully
local nugetConfigPath="$repo_root/NuGet.config" local nugetConfigPath="{$repo_root}NuGet.config"
if [ ! "$nugetConfigPath" ]; then if [ ! "$nugetConfigPath" ]; then
Write-PipelineTelemetryError -category 'Build' "NuGet.config file not found in repo's root!" Write-PipelineTelemetryError -category 'Build' "NuGet.config file not found in repo's root!"
@ -62,8 +62,6 @@ function SetupCredProvider {
endpoints+=']' endpoints+=']'
if [ ${#endpoints} -gt 2 ]; then if [ ${#endpoints} -gt 2 ]; then
# [SuppressMessage("Microsoft.Security", "CS002:SecretInNextLine", Justification="Endpoint code example with no real credentials.")]
# Create the JSON object. It should look like '{"endpointCredentials": [{"endpoint":"http://example.index.json", "username":"optional", "password":"accesstoken"}]}'
local endpointCredentials="{\"endpointCredentials\": "$endpoints"}" local endpointCredentials="{\"endpointCredentials\": "$endpoints"}"
echo "##vso[task.setvariable variable=VSS_NUGET_EXTERNAL_FEED_ENDPOINTS]$endpointCredentials" echo "##vso[task.setvariable variable=VSS_NUGET_EXTERNAL_FEED_ENDPOINTS]$endpointCredentials"
@ -103,7 +101,7 @@ authToken=''
repoName='' repoName=''
while [[ $# > 0 ]]; do while [[ $# > 0 ]]; do
opt="$(echo "$1" | awk '{print tolower($0)}')" opt="$(echo "$1" | tr "[:upper:]" "[:lower:]")"
case "$opt" in case "$opt" in
--operation) --operation)
operation=$2 operation=$2

1
eng/common/msbuild.ps1

@ -5,6 +5,7 @@ Param(
[bool] $nodeReuse = $true, [bool] $nodeReuse = $true,
[switch] $ci, [switch] $ci,
[switch] $prepareMachine, [switch] $prepareMachine,
[switch] $excludePrereleaseVS,
[Parameter(ValueFromRemainingArguments=$true)][String[]]$extraArgs [Parameter(ValueFromRemainingArguments=$true)][String[]]$extraArgs
) )

2
eng/common/msbuild.sh

@ -19,7 +19,7 @@ prepare_machine=false
extra_args='' extra_args=''
while (($# > 0)); do while (($# > 0)); do
lowerI="$(echo $1 | awk '{print tolower($0)}')" lowerI="$(echo $1 | tr "[:upper:]" "[:lower:]")"
case $lowerI in case $lowerI in
--verbosity) --verbosity)
verbosity=$2 verbosity=$2

2
eng/common/native/CommonLibrary.psm1

@ -48,7 +48,7 @@ function DownloadAndExtract {
-Verbose:$Verbose -Verbose:$Verbose
if ($DownloadStatus -Eq $False) { if ($DownloadStatus -Eq $False) {
Write-Error "Download failed" Write-Error "Download failed from $Uri"
return $False return $False
} }

6
eng/common/native/install-cmake-test.sh

@ -14,7 +14,7 @@ download_retries=5
retry_wait_time_seconds=30 retry_wait_time_seconds=30
while (($# > 0)); do while (($# > 0)); do
lowerI="$(echo $1 | awk '{print tolower($0)}')" lowerI="$(echo $1 | tr "[:upper:]" "[:lower:]")"
case $lowerI in case $lowerI in
--baseuri) --baseuri)
base_uri=$2 base_uri=$2
@ -63,7 +63,7 @@ done
tool_name="cmake-test" tool_name="cmake-test"
tool_os=$(GetCurrentOS) tool_os=$(GetCurrentOS)
tool_folder=$(echo $tool_os | awk '{print tolower($0)}') tool_folder="$(echo $tool_os | tr "[:upper:]" "[:lower:]")"
tool_arch="x86_64" tool_arch="x86_64"
tool_name_moniker="$tool_name-$version-$tool_os-$tool_arch" tool_name_moniker="$tool_name-$version-$tool_os-$tool_arch"
tool_install_directory="$install_path/$tool_name/$version" tool_install_directory="$install_path/$tool_name/$version"
@ -114,4 +114,4 @@ if [[ $? != 0 ]]; then
exit 1 exit 1
fi fi
exit 0 exit 0

6
eng/common/native/install-cmake.sh

@ -14,7 +14,7 @@ download_retries=5
retry_wait_time_seconds=30 retry_wait_time_seconds=30
while (($# > 0)); do while (($# > 0)); do
lowerI="$(echo $1 | awk '{print tolower($0)}')" lowerI="$(echo $1 | tr "[:upper:]" "[:lower:]")"
case $lowerI in case $lowerI in
--baseuri) --baseuri)
base_uri=$2 base_uri=$2
@ -63,7 +63,7 @@ done
tool_name="cmake" tool_name="cmake"
tool_os=$(GetCurrentOS) tool_os=$(GetCurrentOS)
tool_folder=$(echo $tool_os | awk '{print tolower($0)}') tool_folder="$(echo $tool_os | tr "[:upper:]" "[:lower:]")"
tool_arch="x86_64" tool_arch="x86_64"
tool_name_moniker="$tool_name-$version-$tool_os-$tool_arch" tool_name_moniker="$tool_name-$version-$tool_os-$tool_arch"
tool_install_directory="$install_path/$tool_name/$version" tool_install_directory="$install_path/$tool_name/$version"
@ -114,4 +114,4 @@ if [[ $? != 0 ]]; then
exit 1 exit 1
fi fi
exit 0 exit 0

2
eng/common/native/install-tool.ps1

@ -105,7 +105,7 @@ try {
Write-Error "There are multiple copies of $ToolName in $($ToolInstallDirectory): `n$(@($ToolFilePath | out-string))" Write-Error "There are multiple copies of $ToolName in $($ToolInstallDirectory): `n$(@($ToolFilePath | out-string))"
exit 1 exit 1
} elseif (@($ToolFilePath).Length -Lt 1) { } elseif (@($ToolFilePath).Length -Lt 1) {
Write-Host "$ToolName was not found in $ToolFilePath." Write-Host "$ToolName was not found in $ToolInstallDirectory."
exit 1 exit 1
} }

104
eng/common/pipeline-logging-functions.ps1

@ -29,14 +29,14 @@ function Write-PipelineTelemetryError {
[switch]$AsOutput, [switch]$AsOutput,
[switch]$Force) [switch]$Force)
$PSBoundParameters.Remove('Category') | Out-Null $PSBoundParameters.Remove('Category') | Out-Null
if($Force -Or ((Test-Path variable:ci) -And $ci)) { if ($Force -Or ((Test-Path variable:ci) -And $ci)) {
$Message = "(NETCORE_ENGINEERING_TELEMETRY=$Category) $Message" $Message = "(NETCORE_ENGINEERING_TELEMETRY=$Category) $Message"
} }
$PSBoundParameters.Remove('Message') | Out-Null $PSBoundParameters.Remove('Message') | Out-Null
$PSBoundParameters.Add('Message', $Message) $PSBoundParameters.Add('Message', $Message)
Write-PipelineTaskError @PSBoundParameters Write-PipelineTaskError @PSBoundParameters
} }
# Specify "-Force" to force pipeline formatted output even if "$ci" is false or not set # Specify "-Force" to force pipeline formatted output even if "$ci" is false or not set
@ -55,8 +55,8 @@ function Write-PipelineTaskError {
[switch]$Force [switch]$Force
) )
if(!$Force -And (-Not (Test-Path variable:ci) -Or !$ci)) { if (!$Force -And (-Not (Test-Path variable:ci) -Or !$ci)) {
if($Type -eq 'error') { if ($Type -eq 'error') {
Write-Host $Message -ForegroundColor Red Write-Host $Message -ForegroundColor Red
return return
} }
@ -66,47 +66,61 @@ function Write-PipelineTaskError {
} }
} }
if(($Type -ne 'error') -and ($Type -ne 'warning')) { if (($Type -ne 'error') -and ($Type -ne 'warning')) {
Write-Host $Message Write-Host $Message
return return
} }
$PSBoundParameters.Remove('Force') | Out-Null $PSBoundParameters.Remove('Force') | Out-Null
if(-not $PSBoundParameters.ContainsKey('Type')) { if (-not $PSBoundParameters.ContainsKey('Type')) {
$PSBoundParameters.Add('Type', 'error') $PSBoundParameters.Add('Type', 'error')
} }
Write-LogIssue @PSBoundParameters Write-LogIssue @PSBoundParameters
} }
function Write-PipelineSetVariable { function Write-PipelineSetVariable {
[CmdletBinding()] [CmdletBinding()]
param( param(
[Parameter(Mandatory = $true)] [Parameter(Mandatory = $true)]
[string]$Name, [string]$Name,
[string]$Value, [string]$Value,
[switch]$Secret, [switch]$Secret,
[switch]$AsOutput, [switch]$AsOutput,
[bool]$IsMultiJobVariable=$true) [bool]$IsMultiJobVariable = $true)
if((Test-Path variable:ci) -And $ci) { if ((Test-Path variable:ci) -And $ci) {
Write-LoggingCommand -Area 'task' -Event 'setvariable' -Data $Value -Properties @{ Write-LoggingCommand -Area 'task' -Event 'setvariable' -Data $Value -Properties @{
'variable' = $Name 'variable' = $Name
'isSecret' = $Secret 'isSecret' = $Secret
'isOutput' = $IsMultiJobVariable 'isOutput' = $IsMultiJobVariable
} -AsOutput:$AsOutput } -AsOutput:$AsOutput
} }
} }
function Write-PipelinePrependPath { function Write-PipelinePrependPath {
[CmdletBinding()] [CmdletBinding()]
param( param(
[Parameter(Mandatory=$true)] [Parameter(Mandatory = $true)]
[string]$Path, [string]$Path,
[switch]$AsOutput) [switch]$AsOutput)
if((Test-Path variable:ci) -And $ci) { if ((Test-Path variable:ci) -And $ci) {
Write-LoggingCommand -Area 'task' -Event 'prependpath' -Data $Path -AsOutput:$AsOutput Write-LoggingCommand -Area 'task' -Event 'prependpath' -Data $Path -AsOutput:$AsOutput
} }
} }
function Write-PipelineSetResult {
[CmdletBinding()]
param(
[ValidateSet("Succeeded", "SucceededWithIssues", "Failed", "Cancelled", "Skipped")]
[Parameter(Mandatory = $true)]
[string]$Result,
[string]$Message)
if ((Test-Path variable:ci) -And $ci) {
Write-LoggingCommand -Area 'task' -Event 'complete' -Data $Message -Properties @{
'result' = $Result
}
}
}
<######################################## <########################################
# Private functions. # Private functions.
@ -123,7 +137,8 @@ function Format-LoggingCommandData {
foreach ($mapping in $script:loggingCommandEscapeMappings) { foreach ($mapping in $script:loggingCommandEscapeMappings) {
$Value = $Value.Replace($mapping.Token, $mapping.Replacement) $Value = $Value.Replace($mapping.Token, $mapping.Replacement)
} }
} else { }
else {
for ($i = $script:loggingCommandEscapeMappings.Length - 1 ; $i -ge 0 ; $i--) { for ($i = $script:loggingCommandEscapeMappings.Length - 1 ; $i -ge 0 ; $i--) {
$mapping = $script:loggingCommandEscapeMappings[$i] $mapping = $script:loggingCommandEscapeMappings[$i]
$Value = $Value.Replace($mapping.Replacement, $mapping.Token) $Value = $Value.Replace($mapping.Replacement, $mapping.Token)
@ -156,7 +171,8 @@ function Format-LoggingCommand {
if ($first) { if ($first) {
$null = $sb.Append(' ') $null = $sb.Append(' ')
$first = $false $first = $false
} else { }
else {
$null = $sb.Append(';') $null = $sb.Append(';')
} }
@ -193,7 +209,8 @@ function Write-LoggingCommand {
$command = Format-LoggingCommand -Area $Area -Event $Event -Data $Data -Properties $Properties $command = Format-LoggingCommand -Area $Area -Event $Event -Data $Data -Properties $Properties
if ($AsOutput) { if ($AsOutput) {
$command $command
} else { }
else {
Write-Host $command Write-Host $command
} }
} }
@ -212,12 +229,12 @@ function Write-LogIssue {
[switch]$AsOutput) [switch]$AsOutput)
$command = Format-LoggingCommand -Area 'task' -Event 'logissue' -Data $Message -Properties @{ $command = Format-LoggingCommand -Area 'task' -Event 'logissue' -Data $Message -Properties @{
'type' = $Type 'type' = $Type
'code' = $ErrCode 'code' = $ErrCode
'sourcepath' = $SourcePath 'sourcepath' = $SourcePath
'linenumber' = $LineNumber 'linenumber' = $LineNumber
'columnnumber' = $ColumnNumber 'columnnumber' = $ColumnNumber
} }
if ($AsOutput) { if ($AsOutput) {
return $command return $command
} }
@ -229,7 +246,8 @@ function Write-LogIssue {
$foregroundColor = [System.ConsoleColor]::Red $foregroundColor = [System.ConsoleColor]::Red
$backgroundColor = [System.ConsoleColor]::Black $backgroundColor = [System.ConsoleColor]::Black
} }
} else { }
else {
$foregroundColor = $host.PrivateData.WarningForegroundColor $foregroundColor = $host.PrivateData.WarningForegroundColor
$backgroundColor = $host.PrivateData.WarningBackgroundColor $backgroundColor = $host.PrivateData.WarningBackgroundColor
if ($foregroundColor -isnot [System.ConsoleColor] -or $backgroundColor -isnot [System.ConsoleColor]) { if ($foregroundColor -isnot [System.ConsoleColor] -or $backgroundColor -isnot [System.ConsoleColor]) {

34
eng/common/pipeline-logging-functions.sh

@ -6,7 +6,7 @@ function Write-PipelineTelemetryError {
local function_args=() local function_args=()
local message='' local message=''
while [[ $# -gt 0 ]]; do while [[ $# -gt 0 ]]; do
opt="$(echo "${1/#--/-}" | awk '{print tolower($0)}')" opt="$(echo "${1/#--/-}" | tr "[:upper:]" "[:lower:]")"
case "$opt" in case "$opt" in
-category|-c) -category|-c)
telemetry_category=$2 telemetry_category=$2
@ -48,7 +48,7 @@ function Write-PipelineTaskError {
local force=false local force=false
while [[ $# -gt 0 ]]; do while [[ $# -gt 0 ]]; do
opt="$(echo "${1/#--/-}" | awk '{print tolower($0)}')" opt="$(echo "${1/#--/-}" | tr "[:upper:]" "[:lower:]")"
case "$opt" in case "$opt" in
-type|-t) -type|-t)
message_type=$2 message_type=$2
@ -122,7 +122,7 @@ function Write-PipelineSetVariable {
local is_multi_job_variable=true local is_multi_job_variable=true
while [[ $# -gt 0 ]]; do while [[ $# -gt 0 ]]; do
opt="$(echo "${1/#--/-}" | awk '{print tolower($0)}')" opt="$(echo "${1/#--/-}" | tr "[:upper:]" "[:lower:]")"
case "$opt" in case "$opt" in
-name|-n) -name|-n)
name=$2 name=$2
@ -164,7 +164,7 @@ function Write-PipelinePrependPath {
local prepend_path='' local prepend_path=''
while [[ $# -gt 0 ]]; do while [[ $# -gt 0 ]]; do
opt="$(echo "${1/#--/-}" | awk '{print tolower($0)}')" opt="$(echo "${1/#--/-}" | tr "[:upper:]" "[:lower:]")"
case "$opt" in case "$opt" in
-path|-p) -path|-p)
prepend_path=$2 prepend_path=$2
@ -179,4 +179,28 @@ function Write-PipelinePrependPath {
if [[ "$ci" == true ]]; then if [[ "$ci" == true ]]; then
echo "##vso[task.prependpath]$prepend_path" echo "##vso[task.prependpath]$prepend_path"
fi fi
} }
function Write-PipelineSetResult {
local result=''
local message=''
while [[ $# -gt 0 ]]; do
opt="$(echo "${1/#--/-}" | tr "[:upper:]" "[:lower:]")"
case "$opt" in
-result|-r)
result=$2
shift
;;
-message|-m)
message=$2
shift
;;
esac
shift
done
if [[ "$ci" == true ]]; then
echo "##vso[task.complete result=$result;]$message"
fi
}

6
eng/common/post-build/post-build-utils.ps1

@ -69,9 +69,9 @@ function Trigger-Subscription([string]$SubscriptionId) {
function Validate-MaestroVars { function Validate-MaestroVars {
try { try {
Get-Variable MaestroApiEndPoint -Scope Global | Out-Null Get-Variable MaestroApiEndPoint | Out-Null
Get-Variable MaestroApiVersion -Scope Global | Out-Null Get-Variable MaestroApiVersion | Out-Null
Get-Variable MaestroApiAccessToken -Scope Global | Out-Null Get-Variable MaestroApiAccessToken | Out-Null
if (!($MaestroApiEndPoint -Match '^http[s]?://maestro-(int|prod).westus2.cloudapp.azure.com$')) { if (!($MaestroApiEndPoint -Match '^http[s]?://maestro-(int|prod).westus2.cloudapp.azure.com$')) {
Write-PipelineTelemetryError -Category 'MaestroVars' -Message "MaestroApiEndPoint is not a valid Maestro URL. '$MaestroApiEndPoint'" Write-PipelineTelemetryError -Category 'MaestroVars' -Message "MaestroApiEndPoint is not a valid Maestro URL. '$MaestroApiEndPoint'"

14
eng/common/post-build/publish-using-darc.ps1

@ -10,21 +10,27 @@ param(
[Parameter(Mandatory=$false)][string] $EnableNugetValidation, [Parameter(Mandatory=$false)][string] $EnableNugetValidation,
[Parameter(Mandatory=$false)][string] $PublishInstallersAndChecksums, [Parameter(Mandatory=$false)][string] $PublishInstallersAndChecksums,
[Parameter(Mandatory=$false)][string] $ArtifactsPublishingAdditionalParameters, [Parameter(Mandatory=$false)][string] $ArtifactsPublishingAdditionalParameters,
[Parameter(Mandatory=$false)][string] $SymbolPublishingAdditionalParameters,
[Parameter(Mandatory=$false)][string] $SigningValidationAdditionalParameters [Parameter(Mandatory=$false)][string] $SigningValidationAdditionalParameters
) )
try { try {
. $PSScriptRoot\post-build-utils.ps1 . $PSScriptRoot\post-build-utils.ps1
# Hard coding darc version till the next arcade-services roll out, cos this version has required API changes for darc add-build-to-channel
$darc = Get-Darc "1.1.0-beta.20418.1" $darc = Get-Darc
$optionalParams = [System.Collections.ArrayList]::new() $optionalParams = [System.Collections.ArrayList]::new()
if ("" -ne $ArtifactsPublishingAdditionalParameters) { if ("" -ne $ArtifactsPublishingAdditionalParameters) {
$optionalParams.Add("artifact-publishing-parameters") | Out-Null $optionalParams.Add("--artifact-publishing-parameters") | Out-Null
$optionalParams.Add($ArtifactsPublishingAdditionalParameters) | Out-Null $optionalParams.Add($ArtifactsPublishingAdditionalParameters) | Out-Null
} }
if ("" -ne $SymbolPublishingAdditionalParameters) {
$optionalParams.Add("--symbol-publishing-parameters") | Out-Null
$optionalParams.Add($SymbolPublishingAdditionalParameters) | Out-Null
}
if ("false" -eq $WaitPublishingFinish) { if ("false" -eq $WaitPublishingFinish) {
$optionalParams.Add("--no-wait") | Out-Null $optionalParams.Add("--no-wait") | Out-Null
} }
@ -54,7 +60,7 @@ try {
--id $buildId ` --id $buildId `
--publishing-infra-version $PublishingInfraVersion ` --publishing-infra-version $PublishingInfraVersion `
--default-channels ` --default-channels `
--source-branch master ` --source-branch main `
--azdev-pat $AzdoToken ` --azdev-pat $AzdoToken `
--bar-uri $MaestroApiEndPoint ` --bar-uri $MaestroApiEndPoint `
--password $MaestroToken ` --password $MaestroToken `

95
eng/common/post-build/sourcelink-validation.ps1

@ -14,7 +14,9 @@ param(
$global:RepoFiles = @{} $global:RepoFiles = @{}
# Maximum number of jobs to run in parallel # Maximum number of jobs to run in parallel
$MaxParallelJobs = 6 $MaxParallelJobs = 16
$MaxRetries = 5
# Wait time between check for system load # Wait time between check for system load
$SecondsBetweenLoadChecks = 10 $SecondsBetweenLoadChecks = 10
@ -29,7 +31,10 @@ $ValidatePackage = {
# Ensure input file exist # Ensure input file exist
if (!(Test-Path $PackagePath)) { if (!(Test-Path $PackagePath)) {
Write-Host "Input file does not exist: $PackagePath" Write-Host "Input file does not exist: $PackagePath"
return 1 return [pscustomobject]@{
result = 1
packagePath = $PackagePath
}
} }
# Extensions for which we'll look for SourceLink information # Extensions for which we'll look for SourceLink information
@ -59,7 +64,10 @@ $ValidatePackage = {
# We ignore resource DLLs # We ignore resource DLLs
if ($FileName.EndsWith('.resources.dll')) { if ($FileName.EndsWith('.resources.dll')) {
return return [pscustomobject]@{
result = 0
packagePath = $PackagePath
}
} }
[System.IO.Compression.ZipFileExtensions]::ExtractToFile($_, $TargetFile, $true) [System.IO.Compression.ZipFileExtensions]::ExtractToFile($_, $TargetFile, $true)
@ -91,36 +99,49 @@ $ValidatePackage = {
$Status = 200 $Status = 200
$Cache = $using:RepoFiles $Cache = $using:RepoFiles
if ( !($Cache.ContainsKey($FilePath)) ) { $totalRetries = 0
try {
$Uri = $Link -as [System.URI] while ($totalRetries -lt $using:MaxRetries) {
if ( !($Cache.ContainsKey($FilePath)) ) {
# Only GitHub links are valid try {
if ($Uri.AbsoluteURI -ne $null -and ($Uri.Host -match 'github' -or $Uri.Host -match 'githubusercontent')) { $Uri = $Link -as [System.URI]
$Status = (Invoke-WebRequest -Uri $Link -UseBasicParsing -Method HEAD -TimeoutSec 5).StatusCode
# Only GitHub links are valid
if ($Uri.AbsoluteURI -ne $null -and ($Uri.Host -match 'github' -or $Uri.Host -match 'githubusercontent')) {
$Status = (Invoke-WebRequest -Uri $Link -UseBasicParsing -Method HEAD -TimeoutSec 5).StatusCode
}
else {
# If it's not a github link, we want to break out of the loop and not retry.
$Status = 0
$totalRetries = $using:MaxRetries
}
} }
else { catch {
Write-Host $_
$Status = 0 $Status = 0
} }
} }
catch {
write-host $_
$Status = 0
}
}
if ($Status -ne 200) { if ($Status -ne 200) {
if ($NumFailedLinks -eq 0) { $totalRetries++
if ($FailedFiles.Value -eq 0) {
Write-Host if ($totalRetries -ge $using:MaxRetries) {
if ($NumFailedLinks -eq 0) {
if ($FailedFiles.Value -eq 0) {
Write-Host
}
Write-Host "`tFile $RealPath has broken links:"
}
Write-Host "`t`tFailed to retrieve $Link"
$NumFailedLinks++
} }
Write-Host "`tFile $RealPath has broken links:"
} }
else {
Write-Host "`t`tFailed to retrieve $Link" break
}
$NumFailedLinks++
} }
} }
} }
@ -136,7 +157,7 @@ $ValidatePackage = {
} }
} }
catch { catch {
Write-Host $_
} }
finally { finally {
$zip.Dispose() $zip.Dispose()
@ -161,9 +182,12 @@ $ValidatePackage = {
function CheckJobResult( function CheckJobResult(
$result, $result,
$packagePath, $packagePath,
[ref]$ValidationFailures) { [ref]$ValidationFailures,
if ($jobResult.result -ne '0') { [switch]$logErrors) {
Write-PipelineTelemetryError -Category 'SourceLink' -Message "$packagePath has broken SourceLink links." if ($result -ne '0') {
if ($logErrors) {
Write-PipelineTelemetryError -Category 'SourceLink' -Message "$packagePath has broken SourceLink links."
}
$ValidationFailures.Value++ $ValidationFailures.Value++
} }
} }
@ -217,6 +241,7 @@ function ValidateSourceLinkLinks {
# Process each NuGet package in parallel # Process each NuGet package in parallel
Get-ChildItem "$InputPath\*.symbols.nupkg" | Get-ChildItem "$InputPath\*.symbols.nupkg" |
ForEach-Object { ForEach-Object {
Write-Host "Starting $($_.FullName)"
Start-Job -ScriptBlock $ValidatePackage -ArgumentList $_.FullName | Out-Null Start-Job -ScriptBlock $ValidatePackage -ArgumentList $_.FullName | Out-Null
$NumJobs = @(Get-Job -State 'Running').Count $NumJobs = @(Get-Job -State 'Running').Count
@ -228,16 +253,14 @@ function ValidateSourceLinkLinks {
foreach ($Job in @(Get-Job -State 'Completed')) { foreach ($Job in @(Get-Job -State 'Completed')) {
$jobResult = Wait-Job -Id $Job.Id | Receive-Job $jobResult = Wait-Job -Id $Job.Id | Receive-Job
CheckJobResult $jobResult.result $jobResult.packagePath ([ref]$ValidationFailures) CheckJobResult $jobResult.result $jobResult.packagePath ([ref]$ValidationFailures) -LogErrors
Remove-Job -Id $Job.Id Remove-Job -Id $Job.Id
} }
} }
foreach ($Job in @(Get-Job)) { foreach ($Job in @(Get-Job)) {
$jobResult = Wait-Job -Id $Job.Id | Receive-Job $jobResult = Wait-Job -Id $Job.Id | Receive-Job
if ($jobResult -ne '0') { CheckJobResult $jobResult.result $jobResult.packagePath ([ref]$ValidationFailures)
$ValidationFailures++
}
Remove-Job -Id $Job.Id Remove-Job -Id $Job.Id
} }
if ($ValidationFailures -gt 0) { if ($ValidationFailures -gt 0) {
@ -266,6 +289,10 @@ function InstallSourcelinkCli {
try { try {
InstallSourcelinkCli InstallSourcelinkCli
foreach ($Job in @(Get-Job)) {
Remove-Job -Id $Job.Id
}
ValidateSourceLinkLinks ValidateSourceLinkLinks
} }
catch { catch {

182
eng/common/post-build/symbols-validation.ps1

@ -1,30 +1,49 @@
param( param(
[Parameter(Mandatory=$true)][string] $InputPath, # Full path to directory where NuGet packages to be checked are stored [Parameter(Mandatory = $true)][string] $InputPath, # Full path to directory where NuGet packages to be checked are stored
[Parameter(Mandatory=$true)][string] $ExtractPath, # Full path to directory where the packages will be extracted during validation [Parameter(Mandatory = $true)][string] $ExtractPath, # Full path to directory where the packages will be extracted during validation
[Parameter(Mandatory=$true)][string] $DotnetSymbolVersion, # Version of dotnet symbol to use [Parameter(Mandatory = $true)][string] $DotnetSymbolVersion, # Version of dotnet symbol to use
[Parameter(Mandatory=$false)][switch] $ContinueOnError, # If we should keep checking symbols after an error [Parameter(Mandatory = $false)][switch] $CheckForWindowsPdbs, # If we should check for the existence of windows pdbs in addition to portable PDBs
[Parameter(Mandatory=$false)][switch] $Clean # Clean extracted symbols directory after checking symbols [Parameter(Mandatory = $false)][switch] $ContinueOnError, # If we should keep checking symbols after an error
[Parameter(Mandatory = $false)][switch] $Clean # Clean extracted symbols directory after checking symbols
) )
# Maximum number of jobs to run in parallel # Maximum number of jobs to run in parallel
$MaxParallelJobs = 6 $MaxParallelJobs = 16
# Max number of retries
$MaxRetry = 5
# Wait time between check for system load # Wait time between check for system load
$SecondsBetweenLoadChecks = 10 $SecondsBetweenLoadChecks = 10
# Set error codes
Set-Variable -Name "ERROR_BADEXTRACT" -Option Constant -Value -1
Set-Variable -Name "ERROR_FILEDOESNOTEXIST" -Option Constant -Value -2
$WindowsPdbVerificationParam = ""
if ($CheckForWindowsPdbs) {
$WindowsPdbVerificationParam = "--windows-pdbs"
}
$CountMissingSymbols = { $CountMissingSymbols = {
param( param(
[string] $PackagePath # Path to a NuGet package [string] $PackagePath, # Path to a NuGet package
[string] $WindowsPdbVerificationParam # If we should check for the existence of windows pdbs in addition to portable PDBs
) )
. $using:PSScriptRoot\..\tools.ps1 . $using:PSScriptRoot\..\tools.ps1
Add-Type -AssemblyName System.IO.Compression.FileSystem Add-Type -AssemblyName System.IO.Compression.FileSystem
Write-Host "Validating $PackagePath "
# Ensure input file exist # Ensure input file exist
if (!(Test-Path $PackagePath)) { if (!(Test-Path $PackagePath)) {
Write-PipelineTaskError "Input file does not exist: $PackagePath" Write-PipelineTaskError "Input file does not exist: $PackagePath"
return -2 return [pscustomobject]@{
result = $using:ERROR_FILEDOESNOTEXIST
packagePath = $PackagePath
}
} }
# Extensions for which we'll look for symbols # Extensions for which we'll look for symbols
@ -45,24 +64,25 @@ $CountMissingSymbols = {
Write-Host "Something went wrong extracting $PackagePath" Write-Host "Something went wrong extracting $PackagePath"
Write-Host $_ Write-Host $_
return [pscustomobject]@{ return [pscustomobject]@{
result = -1 result = $using:ERROR_BADEXTRACT
packagePath = $PackagePath packagePath = $PackagePath
} }
} }
Get-ChildItem -Recurse $ExtractPath | Get-ChildItem -Recurse $ExtractPath |
Where-Object {$RelevantExtensions -contains $_.Extension} | Where-Object { $RelevantExtensions -contains $_.Extension } |
ForEach-Object { ForEach-Object {
$FileName = $_.FullName $FileName = $_.FullName
if ($FileName -Match '\\ref\\') { if ($FileName -Match '\\ref\\') {
Write-Host "`t Ignoring reference assembly file " $FileName Write-Host "`t Ignoring reference assembly file " $FileName
return return
} }
$FirstMatchingSymbolDescriptionOrDefault = { $FirstMatchingSymbolDescriptionOrDefault = {
param( param(
[string] $FullPath, # Full path to the module that has to be checked [string] $FullPath, # Full path to the module that has to be checked
[string] $TargetServerParam, # Parameter to pass to `Symbol Tool` indicating the server to lookup for symbols [string] $TargetServerParam, # Parameter to pass to `Symbol Tool` indicating the server to lookup for symbols
[string] $WindowsPdbVerificationParam, # Parameter to pass to potential check for windows-pdbs.
[string] $SymbolsPath [string] $SymbolsPath
) )
@ -87,56 +107,76 @@ $CountMissingSymbols = {
# DWARF file for a .dylib # DWARF file for a .dylib
$DylibDwarf = $SymbolPath.Replace($Extension, '.dylib.dwarf') $DylibDwarf = $SymbolPath.Replace($Extension, '.dylib.dwarf')
$dotnetSymbolExe = "$env:USERPROFILE\.dotnet\tools" $dotnetSymbolExe = "$env:USERPROFILE\.dotnet\tools"
$dotnetSymbolExe = Resolve-Path "$dotnetSymbolExe\dotnet-symbol.exe" $dotnetSymbolExe = Resolve-Path "$dotnetSymbolExe\dotnet-symbol.exe"
& $dotnetSymbolExe --symbols --modules --windows-pdbs $TargetServerParam $FullPath -o $SymbolsPath | Out-Null $totalRetries = 0
if (Test-Path $PdbPath) { while ($totalRetries -lt $using:MaxRetry) {
return 'PDB'
} # Save the output and get diagnostic output
elseif (Test-Path $NGenPdb) { $output = & $dotnetSymbolExe --symbols --modules $WindowsPdbVerificationParam $TargetServerParam $FullPath -o $SymbolsPath --diagnostics | Out-String
return 'NGen PDB'
} if (Test-Path $PdbPath) {
elseif (Test-Path $SODbg) { return 'PDB'
return 'DBG for SO' }
} elseif (Test-Path $NGenPdb) {
elseif (Test-Path $DylibDwarf) { return 'NGen PDB'
return 'Dwarf for Dylib' }
} elseif (Test-Path $SODbg) {
elseif (Test-Path $SymbolPath) { return 'DBG for SO'
return 'Module' }
} elseif (Test-Path $DylibDwarf) {
else { return 'Dwarf for Dylib'
return $null }
elseif (Test-Path $SymbolPath) {
return 'Module'
}
else
{
$totalRetries++
}
} }
return $null
} }
$SymbolsOnMSDL = & $FirstMatchingSymbolDescriptionOrDefault $FileName '--microsoft-symbol-server' $SymbolsPath $FileGuid = New-Guid
$SymbolsOnSymWeb = & $FirstMatchingSymbolDescriptionOrDefault $FileName '--internal-server' $SymbolsPath $ExpandedSymbolsPath = Join-Path -Path $SymbolsPath -ChildPath $FileGuid
Write-Host -NoNewLine "`t Checking file " $FileName "... " $SymbolsOnMSDL = & $FirstMatchingSymbolDescriptionOrDefault `
-FullPath $FileName `
-TargetServerParam '--microsoft-symbol-server' `
-SymbolsPath "$ExpandedSymbolsPath-msdl" `
-WindowsPdbVerificationParam $WindowsPdbVerificationParam
$SymbolsOnSymWeb = & $FirstMatchingSymbolDescriptionOrDefault `
-FullPath $FileName `
-TargetServerParam '--internal-server' `
-SymbolsPath "$ExpandedSymbolsPath-symweb" `
-WindowsPdbVerificationParam $WindowsPdbVerificationParam
Write-Host -NoNewLine "`t Checking file " $FileName "... "
if ($SymbolsOnMSDL -ne $null -and $SymbolsOnSymWeb -ne $null) { if ($SymbolsOnMSDL -ne $null -and $SymbolsOnSymWeb -ne $null) {
Write-Host "Symbols found on MSDL ($SymbolsOnMSDL) and SymWeb ($SymbolsOnSymWeb)" Write-Host "Symbols found on MSDL ($SymbolsOnMSDL) and SymWeb ($SymbolsOnSymWeb)"
}
else {
$MissingSymbols++
if ($SymbolsOnMSDL -eq $null -and $SymbolsOnSymWeb -eq $null) {
Write-Host 'No symbols found on MSDL or SymWeb!'
} }
else { else {
$MissingSymbols++ if ($SymbolsOnMSDL -eq $null) {
Write-Host 'No symbols found on MSDL!'
if ($SymbolsOnMSDL -eq $null -and $SymbolsOnSymWeb -eq $null) {
Write-Host 'No symbols found on MSDL or SymWeb!'
} }
else { else {
if ($SymbolsOnMSDL -eq $null) { Write-Host 'No symbols found on SymWeb!'
Write-Host 'No symbols found on MSDL!'
}
else {
Write-Host 'No symbols found on SymWeb!'
}
} }
} }
} }
}
if ($using:Clean) { if ($using:Clean) {
Remove-Item $ExtractPath -Recurse -Force Remove-Item $ExtractPath -Recurse -Force
@ -145,24 +185,31 @@ $CountMissingSymbols = {
Pop-Location Pop-Location
return [pscustomobject]@{ return [pscustomobject]@{
result = $MissingSymbols result = $MissingSymbols
packagePath = $PackagePath packagePath = $PackagePath
} }
} }
function CheckJobResult( function CheckJobResult(
$result, $result,
$packagePath, $packagePath,
[ref]$DupedSymbols, [ref]$DupedSymbols,
[ref]$TotalFailures) { [ref]$TotalFailures) {
if ($result -eq '-1') { if ($result -eq $ERROR_BADEXTRACT) {
Write-PipelineTelemetryError -Category 'CheckSymbols' -Message "$packagePath has duplicated symbol files" Write-PipelineTelemetryError -Category 'CheckSymbols' -Message "$packagePath has duplicated symbol files"
$DupedSymbols.Value++ $DupedSymbols.Value++
} }
elseif ($jobResult.result -ne '0') { elseif ($result -eq $ERROR_FILEDOESNOTEXIST) {
Write-PipelineTelemetryError -Category 'CheckSymbols' -Message "$packagePath does not exist"
$TotalFailures.Value++
}
elseif ($result -gt '0') {
Write-PipelineTelemetryError -Category 'CheckSymbols' -Message "Missing symbols for $result modules in the package $packagePath" Write-PipelineTelemetryError -Category 'CheckSymbols' -Message "Missing symbols for $result modules in the package $packagePath"
$TotalFailures.Value++ $TotalFailures.Value++
} }
else {
Write-Host "All symbols verified for package $packagePath"
}
} }
function CheckSymbolsAvailable { function CheckSymbolsAvailable {
@ -170,6 +217,7 @@ function CheckSymbolsAvailable {
Remove-Item $ExtractPath -Force -Recurse -ErrorAction SilentlyContinue Remove-Item $ExtractPath -Force -Recurse -ErrorAction SilentlyContinue
} }
$TotalPackages = 0
$TotalFailures = 0 $TotalFailures = 0
$DupedSymbols = 0 $DupedSymbols = 0
@ -192,9 +240,9 @@ function CheckSymbolsAvailable {
return return
} }
Write-Host "Validating $FileName " $TotalPackages++
Start-Job -ScriptBlock $CountMissingSymbols -ArgumentList $FullName | Out-Null Start-Job -ScriptBlock $CountMissingSymbols -ArgumentList @($FullName,$WindowsPdbVerificationParam) | Out-Null
$NumJobs = @(Get-Job -State 'Running').Count $NumJobs = @(Get-Job -State 'Running').Count
@ -219,11 +267,11 @@ function CheckSymbolsAvailable {
if ($TotalFailures -gt 0 -or $DupedSymbols -gt 0) { if ($TotalFailures -gt 0 -or $DupedSymbols -gt 0) {
if ($TotalFailures -gt 0) { if ($TotalFailures -gt 0) {
Write-PipelineTelemetryError -Category 'CheckSymbols' -Message "Symbols missing for $TotalFailures packages" Write-PipelineTelemetryError -Category 'CheckSymbols' -Message "Symbols missing for $TotalFailures/$TotalPackages packages"
} }
if ($DupedSymbols -gt 0) { if ($DupedSymbols -gt 0) {
Write-PipelineTelemetryError -Category 'CheckSymbols' -Message "$DupedSymbols packages had duplicated symbol files" Write-PipelineTelemetryError -Category 'CheckSymbols' -Message "$DupedSymbols/$TotalPackages packages had duplicated symbol files and could not be extracted"
} }
ExitWithExitCode 1 ExitWithExitCode 1

8
eng/common/sdk-task.ps1

@ -34,7 +34,7 @@ function Print-Usage() {
function Build([string]$target) { function Build([string]$target) {
$logSuffix = if ($target -eq 'Execute') { '' } else { ".$target" } $logSuffix = if ($target -eq 'Execute') { '' } else { ".$target" }
$log = Join-Path $LogDir "$task$logSuffix.binlog" $log = Join-Path $LogDir "$task$logSuffix.binlog"
$outputPath = Join-Path $ToolsetDir "$task\\" $outputPath = Join-Path $ToolsetDir "$task\"
MSBuild $taskProject ` MSBuild $taskProject `
/bl:$log ` /bl:$log `
@ -53,7 +53,7 @@ try {
} }
if ($task -eq "") { if ($task -eq "") {
Write-PipelineTelemetryError -Category 'Build' -Message "Missing required parameter '-task <value>'" -ForegroundColor Red Write-PipelineTelemetryError -Category 'Build' -Message "Missing required parameter '-task <value>'"
Print-Usage Print-Usage
ExitWithExitCode 1 ExitWithExitCode 1
} }
@ -64,7 +64,7 @@ try {
$GlobalJson.tools | Add-Member -Name "vs" -Value (ConvertFrom-Json "{ `"version`": `"16.5`" }") -MemberType NoteProperty $GlobalJson.tools | Add-Member -Name "vs" -Value (ConvertFrom-Json "{ `"version`": `"16.5`" }") -MemberType NoteProperty
} }
if( -not ($GlobalJson.tools.PSObject.Properties.Name -match "xcopy-msbuild" )) { if( -not ($GlobalJson.tools.PSObject.Properties.Name -match "xcopy-msbuild" )) {
$GlobalJson.tools | Add-Member -Name "xcopy-msbuild" -Value "16.8.0-preview3" -MemberType NoteProperty $GlobalJson.tools | Add-Member -Name "xcopy-msbuild" -Value "16.10.0-preview2" -MemberType NoteProperty
} }
if ($GlobalJson.tools."xcopy-msbuild".Trim() -ine "none") { if ($GlobalJson.tools."xcopy-msbuild".Trim() -ine "none") {
$xcopyMSBuildToolsFolder = InitializeXCopyMSBuild $GlobalJson.tools."xcopy-msbuild" -install $true $xcopyMSBuildToolsFolder = InitializeXCopyMSBuild $GlobalJson.tools."xcopy-msbuild" -install $true
@ -78,7 +78,7 @@ try {
$taskProject = GetSdkTaskProject $task $taskProject = GetSdkTaskProject $task
if (!(Test-Path $taskProject)) { if (!(Test-Path $taskProject)) {
Write-PipelineTelemetryError -Category 'Build' -Message "Unknown task: $task" -ForegroundColor Red Write-PipelineTelemetryError -Category 'Build' -Message "Unknown task: $task"
ExitWithExitCode 1 ExitWithExitCode 1
} }

6
eng/common/sdl/execute-all-sdl-tools.ps1

@ -32,7 +32,7 @@ try {
$ErrorActionPreference = 'Stop' $ErrorActionPreference = 'Stop'
Set-StrictMode -Version 2.0 Set-StrictMode -Version 2.0
$disableConfigureToolsetImport = $true $disableConfigureToolsetImport = $true
$LASTEXITCODE = 0 $global:LASTEXITCODE = 0
# `tools.ps1` checks $ci to perform some actions. Since the SDL # `tools.ps1` checks $ci to perform some actions. Since the SDL
# scripts don't necessarily execute in the same agent that run the # scripts don't necessarily execute in the same agent that run the
@ -87,10 +87,6 @@ try {
& $(Join-Path $PSScriptRoot 'run-sdl.ps1') -GuardianCliLocation $guardianCliLocation -WorkingDirectory $workingDirectory -TargetDirectory $SourceDirectory -GdnFolder $gdnFolder -ToolsList $SourceToolsList -AzureDevOpsAccessToken $AzureDevOpsAccessToken -UpdateBaseline $UpdateBaseline -GuardianLoggerLevel $GuardianLoggerLevel -CrScanAdditionalRunConfigParams $CrScanAdditionalRunConfigParams -PoliCheckAdditionalRunConfigParams $PoliCheckAdditionalRunConfigParams & $(Join-Path $PSScriptRoot 'run-sdl.ps1') -GuardianCliLocation $guardianCliLocation -WorkingDirectory $workingDirectory -TargetDirectory $SourceDirectory -GdnFolder $gdnFolder -ToolsList $SourceToolsList -AzureDevOpsAccessToken $AzureDevOpsAccessToken -UpdateBaseline $UpdateBaseline -GuardianLoggerLevel $GuardianLoggerLevel -CrScanAdditionalRunConfigParams $CrScanAdditionalRunConfigParams -PoliCheckAdditionalRunConfigParams $PoliCheckAdditionalRunConfigParams
} }
if ($UpdateBaseline) {
& (Join-Path $PSScriptRoot 'push-gdn.ps1') -Repository $RepoName -BranchName $BranchName -GdnFolder $GdnFolder -AzureDevOpsAccessToken $AzureDevOpsAccessToken -PushReason 'Update baseline'
}
if ($TsaPublish) { if ($TsaPublish) {
if ($TsaBranchName -and $BuildNumber) { if ($TsaBranchName -and $BuildNumber) {
if (-not $TsaRepositoryName) { if (-not $TsaRepositoryName) {

16
eng/common/sdl/init-sdl.ps1

@ -10,7 +10,7 @@ Param(
$ErrorActionPreference = 'Stop' $ErrorActionPreference = 'Stop'
Set-StrictMode -Version 2.0 Set-StrictMode -Version 2.0
$disableConfigureToolsetImport = $true $disableConfigureToolsetImport = $true
$LASTEXITCODE = 0 $global:LASTEXITCODE = 0
# `tools.ps1` checks $ci to perform some actions. Since the SDL # `tools.ps1` checks $ci to perform some actions. Since the SDL
# scripts don't necessarily execute in the same agent that run the # scripts don't necessarily execute in the same agent that run the
@ -29,18 +29,7 @@ $zipFile = "$WorkingDirectory/gdn.zip"
Add-Type -AssemblyName System.IO.Compression.FileSystem Add-Type -AssemblyName System.IO.Compression.FileSystem
$gdnFolder = (Join-Path $WorkingDirectory '.gdn') $gdnFolder = (Join-Path $WorkingDirectory '.gdn')
try {
# We try to download the zip; if the request fails (e.g. the file doesn't exist), we catch it and init guardian instead
Write-Host 'Downloading gdn folder from internal config repostiory...'
Invoke-WebRequest -Headers @{ "Accept"="application/zip"; "Authorization"="Basic $encodedPat" } -Uri $uri -OutFile $zipFile
if (Test-Path $gdnFolder) {
# Remove the gdn folder if it exists (it shouldn't unless there's too much caching; this is just in case)
Remove-Item -Force -Recurse $gdnFolder
}
[System.IO.Compression.ZipFile]::ExtractToDirectory($zipFile, $WorkingDirectory)
Write-Host $gdnFolder
ExitWithExitCode 0
} catch [System.Net.WebException] { } # Catch and ignore webexception
try { try {
# if the folder does not exist, we'll do a guardian init and push it to the remote repository # if the folder does not exist, we'll do a guardian init and push it to the remote repository
Write-Host 'Initializing Guardian...' Write-Host 'Initializing Guardian...'
@ -57,7 +46,6 @@ try {
Write-PipelineTelemetryError -Force -Category 'Build' -Message "Guardian baseline failed with exit code $LASTEXITCODE." Write-PipelineTelemetryError -Force -Category 'Build' -Message "Guardian baseline failed with exit code $LASTEXITCODE."
ExitWithExitCode $LASTEXITCODE ExitWithExitCode $LASTEXITCODE
} }
& $(Join-Path $PSScriptRoot 'push-gdn.ps1') -Repository $Repository -BranchName $BranchName -GdnFolder $gdnFolder -AzureDevOpsAccessToken $AzureDevOpsAccessToken -PushReason 'Initialize gdn folder'
ExitWithExitCode 0 ExitWithExitCode 0
} }
catch { catch {

2
eng/common/sdl/packages.config

@ -1,4 +1,4 @@
<?xml version="1.0" encoding="utf-8"?> <?xml version="1.0" encoding="utf-8"?>
<packages> <packages>
<package id="Microsoft.Guardian.Cli.win10-x64" version="0.20.1"/> <package id="Microsoft.Guardian.Cli" version="0.53.3"/>
</packages> </packages>

2
eng/common/sdl/run-sdl.ps1

@ -13,7 +13,7 @@ Param(
$ErrorActionPreference = 'Stop' $ErrorActionPreference = 'Stop'
Set-StrictMode -Version 2.0 Set-StrictMode -Version 2.0
$disableConfigureToolsetImport = $true $disableConfigureToolsetImport = $true
$LASTEXITCODE = 0 $global:LASTEXITCODE = 0
try { try {
# `tools.ps1` checks $ci to perform some actions. Since the SDL # `tools.ps1` checks $ci to perform some actions. Since the SDL

4
eng/common/templates/job/execute-sdl.yml

@ -45,6 +45,7 @@ jobs:
buildId: $(AzDOBuildId) buildId: $(AzDOBuildId)
artifactName: ${{ artifactName }} artifactName: ${{ artifactName }}
downloadPath: $(Build.ArtifactStagingDirectory)\artifacts downloadPath: $(Build.ArtifactStagingDirectory)\artifacts
checkDownloadedFiles: true
- ${{ if eq(parameters.artifactNames, '') }}: - ${{ if eq(parameters.artifactNames, '') }}:
- task: DownloadBuildArtifacts@0 - task: DownloadBuildArtifacts@0
displayName: Download Build Artifacts displayName: Download Build Artifacts
@ -57,6 +58,7 @@ jobs:
downloadType: specific files downloadType: specific files
itemPattern: "**" itemPattern: "**"
downloadPath: $(Build.ArtifactStagingDirectory)\artifacts downloadPath: $(Build.ArtifactStagingDirectory)\artifacts
checkDownloadedFiles: true
- powershell: eng/common/sdl/extract-artifact-packages.ps1 - powershell: eng/common/sdl/extract-artifact-packages.ps1
-InputPath $(Build.ArtifactStagingDirectory)\artifacts\BlobArtifacts -InputPath $(Build.ArtifactStagingDirectory)\artifacts\BlobArtifacts
-ExtractPath $(Build.ArtifactStagingDirectory)\artifacts\BlobArtifacts -ExtractPath $(Build.ArtifactStagingDirectory)\artifacts\BlobArtifacts
@ -83,7 +85,7 @@ jobs:
continueOnError: ${{ parameters.sdlContinueOnError }} continueOnError: ${{ parameters.sdlContinueOnError }}
- ${{ if eq(parameters.overrideParameters, '') }}: - ${{ if eq(parameters.overrideParameters, '') }}:
- powershell: eng/common/sdl/execute-all-sdl-tools.ps1 - powershell: eng/common/sdl/execute-all-sdl-tools.ps1
-GuardianPackageName Microsoft.Guardian.Cli.win10-x64.0.20.1 -GuardianPackageName Microsoft.Guardian.Cli.0.53.3
-NugetPackageDirectory $(Build.SourcesDirectory)\.packages -NugetPackageDirectory $(Build.SourcesDirectory)\.packages
-AzureDevOpsAccessToken $(dn-bot-dotnet-build-rw-code-rw) -AzureDevOpsAccessToken $(dn-bot-dotnet-build-rw-code-rw)
${{ parameters.additionalParameters }} ${{ parameters.additionalParameters }}

18
eng/common/templates/job/job.yml

@ -24,9 +24,9 @@ parameters:
enablePublishBuildAssets: false enablePublishBuildAssets: false
enablePublishTestResults: false enablePublishTestResults: false
enablePublishUsingPipelines: false enablePublishUsingPipelines: false
useBuildManifest: false
mergeTestResults: false mergeTestResults: false
testRunTitle: '' testRunTitle: ''
testResultsFormat: ''
name: '' name: ''
preSteps: [] preSteps: []
runAsPublic: false runAsPublic: false
@ -131,8 +131,8 @@ jobs:
- task: RichCodeNavIndexer@0 - task: RichCodeNavIndexer@0
displayName: RichCodeNav Upload displayName: RichCodeNav Upload
inputs: inputs:
languages: 'csharp' languages: ${{ coalesce(parameters.richCodeNavigationLanguage, 'csharp') }}
environment: ${{ coalesce(parameters.richCodeNavigationEnvironment, 'prod') }} environment: ${{ coalesce(parameters.richCodeNavigationEnvironment, 'production') }}
richNavLogOutputDirectory: $(Build.SourcesDirectory)/artifacts/bin richNavLogOutputDirectory: $(Build.SourcesDirectory)/artifacts/bin
continueOnError: true continueOnError: true
@ -202,7 +202,7 @@ jobs:
continueOnError: true continueOnError: true
condition: always() condition: always()
- ${{ if eq(parameters.enablePublishTestResults, 'true') }}: - ${{ if or(and(eq(parameters.enablePublishTestResults, 'true'), eq(parameters.testResultsFormat, '')), eq(parameters.testResultsFormat, 'xunit')) }}:
- task: PublishTestResults@2 - task: PublishTestResults@2
displayName: Publish XUnit Test Results displayName: Publish XUnit Test Results
inputs: inputs:
@ -213,6 +213,7 @@ jobs:
mergeTestResults: ${{ parameters.mergeTestResults }} mergeTestResults: ${{ parameters.mergeTestResults }}
continueOnError: true continueOnError: true
condition: always() condition: always()
- ${{ if or(and(eq(parameters.enablePublishTestResults, 'true'), eq(parameters.testResultsFormat, '')), eq(parameters.testResultsFormat, 'vstest')) }}:
- task: PublishTestResults@2 - task: PublishTestResults@2
displayName: Publish TRX Test Results displayName: Publish TRX Test Results
inputs: inputs:
@ -241,12 +242,3 @@ jobs:
ArtifactName: AssetManifests ArtifactName: AssetManifests
continueOnError: ${{ parameters.continueOnError }} continueOnError: ${{ parameters.continueOnError }}
condition: and(succeeded(), eq(variables['_DotNetPublishToBlobFeed'], 'true')) condition: and(succeeded(), eq(variables['_DotNetPublishToBlobFeed'], 'true'))
- ${{ if eq(parameters.useBuildManifest, true) }}:
- task: PublishBuildArtifacts@1
displayName: Publish Build Manifest
inputs:
PathToPublish: '$(Build.SourcesDirectory)/artifacts/log/$(_BuildConfig)/manifest.props'
PublishLocation: Container
ArtifactName: BuildManifests
continueOnError: ${{ parameters.continueOnError }}

93
eng/common/templates/job/onelocbuild.yml

@ -0,0 +1,93 @@
parameters:
# Optional: dependencies of the job
dependsOn: ''
# Optional: A defined YAML pool - https://docs.microsoft.com/en-us/azure/devops/pipelines/yaml-schema?view=vsts&tabs=schema#pool
pool:
vmImage: vs2017-win2016
CeapexPat: $(dn-bot-ceapex-package-r) # PAT for the loc AzDO instance https://dev.azure.com/ceapex
GithubPat: $(BotAccount-dotnet-bot-repo-PAT)
SourcesDirectory: $(Build.SourcesDirectory)
CreatePr: true
AutoCompletePr: false
UseLfLineEndings: true
UseCheckedInLocProjectJson: false
LanguageSet: VS_Main_Languages
LclSource: lclFilesInRepo
LclPackageId: ''
RepoType: gitHub
GitHubOrg: dotnet
MirrorRepo: ''
MirrorBranch: main
condition: ''
jobs:
- job: OneLocBuild
dependsOn: ${{ parameters.dependsOn }}
displayName: OneLocBuild
pool: ${{ parameters.pool }}
variables:
- group: OneLocBuildVariables # Contains the CeapexPat and GithubPat
- name: _GenerateLocProjectArguments
value: -SourcesDirectory ${{ parameters.SourcesDirectory }}
-LanguageSet "${{ parameters.LanguageSet }}"
-CreateNeutralXlfs
- ${{ if eq(parameters.UseCheckedInLocProjectJson, 'true') }}:
- name: _GenerateLocProjectArguments
value: ${{ variables._GenerateLocProjectArguments }} -UseCheckedInLocProjectJson
steps:
- task: Powershell@2
inputs:
filePath: $(Build.SourcesDirectory)/eng/common/generate-locproject.ps1
arguments: $(_GenerateLocProjectArguments)
displayName: Generate LocProject.json
condition: ${{ parameters.condition }}
- task: OneLocBuild@2
displayName: OneLocBuild
env:
SYSTEM_ACCESSTOKEN: $(System.AccessToken)
inputs:
locProj: eng/Localize/LocProject.json
outDir: $(Build.ArtifactStagingDirectory)
lclSource: ${{ parameters.LclSource }}
lclPackageId: ${{ parameters.LclPackageId }}
isCreatePrSelected: ${{ parameters.CreatePr }}
${{ if eq(parameters.CreatePr, true) }}:
isAutoCompletePrSelected: ${{ parameters.AutoCompletePr }}
isUseLfLineEndingsSelected: ${{ parameters.UseLfLineEndings }}
packageSourceAuth: patAuth
patVariable: ${{ parameters.CeapexPat }}
${{ if eq(parameters.RepoType, 'gitHub') }}:
repoType: ${{ parameters.RepoType }}
gitHubPatVariable: "${{ parameters.GithubPat }}"
${{ if ne(parameters.MirrorRepo, '') }}:
isMirrorRepoSelected: true
gitHubOrganization: ${{ parameters.GitHubOrg }}
mirrorRepo: ${{ parameters.MirrorRepo }}
mirrorBranch: ${{ parameters.MirrorBranch }}
condition: ${{ parameters.condition }}
- task: PublishBuildArtifacts@1
displayName: Publish Localization Files
inputs:
PathtoPublish: '$(Build.ArtifactStagingDirectory)/loc'
PublishLocation: Container
ArtifactName: Loc
condition: ${{ parameters.condition }}
- task: PublishBuildArtifacts@1
displayName: Publish LocProject.json
inputs:
PathtoPublish: '$(Build.SourcesDirectory)/eng/Localize/'
PublishLocation: Container
ArtifactName: Loc
condition: ${{ parameters.condition }}

8
eng/common/templates/job/publish-build-assets.yml

@ -37,6 +37,7 @@ jobs:
- name: _BuildConfig - name: _BuildConfig
value: ${{ parameters.configuration }} value: ${{ parameters.configuration }}
- group: Publish-Build-Assets - group: Publish-Build-Assets
- group: AzureDevOps-Artifact-Feeds-Pats
# Skip component governance and codesign validation for SDL. These jobs # Skip component governance and codesign validation for SDL. These jobs
# create no content. # create no content.
- name: skipComponentGovernanceDetection - name: skipComponentGovernanceDetection
@ -51,12 +52,19 @@ jobs:
inputs: inputs:
artifactName: AssetManifests artifactName: AssetManifests
downloadPath: '$(Build.StagingDirectory)/Download' downloadPath: '$(Build.StagingDirectory)/Download'
checkDownloadedFiles: true
condition: ${{ parameters.condition }} condition: ${{ parameters.condition }}
continueOnError: ${{ parameters.continueOnError }} continueOnError: ${{ parameters.continueOnError }}
- ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}:
- task: NuGetAuthenticate@0 - task: NuGetAuthenticate@0
- task: PowerShell@2
displayName: Enable cross-org NuGet feed authentication
inputs:
filePath: $(Build.SourcesDirectory)/eng/common/enable-cross-org-publishing.ps1
arguments: -token $(dn-bot-all-orgs-artifact-feeds-rw)
- task: PowerShell@2 - task: PowerShell@2
displayName: Publish Build Assets displayName: Publish Build Assets
inputs: inputs:

11
eng/common/templates/job/source-build.yml

@ -15,6 +15,9 @@ parameters:
# nonPortable: false # nonPortable: false
# Enables non-portable mode. This means a more specific RID (e.g. fedora.32-x64 rather than # Enables non-portable mode. This means a more specific RID (e.g. fedora.32-x64 rather than
# linux-x64), and compiling against distro-provided packages rather than portable ones. # linux-x64), and compiling against distro-provided packages rather than portable ones.
# skipPublishValidation: false
# Disables publishing validation. By default, a check is performed to ensure no packages are
# published by source-build.
# container: '' # container: ''
# A container to use. Runs in docker. # A container to use. Runs in docker.
# pool: {} # pool: {}
@ -28,6 +31,11 @@ parameters:
# container and pool. # container and pool.
platform: {} platform: {}
# The default VM host AzDO pool. This should be capable of running Docker containers: almost all
# source-build builds run in Docker, including the default managed platform.
defaultContainerHostPool:
vmImage: ubuntu-20.04
jobs: jobs:
- job: ${{ parameters.jobNamePrefix }}_${{ parameters.platform.name }} - job: ${{ parameters.jobNamePrefix }}_${{ parameters.platform.name }}
displayName: Source-Build (${{ parameters.platform.name }}) displayName: Source-Build (${{ parameters.platform.name }})
@ -37,6 +45,9 @@ jobs:
${{ if ne(parameters.platform.container, '') }}: ${{ if ne(parameters.platform.container, '') }}:
container: ${{ parameters.platform.container }} container: ${{ parameters.platform.container }}
${{ if eq(parameters.platform.pool, '') }}:
pool: ${{ parameters.defaultContainerHostPool }}
${{ if ne(parameters.platform.pool, '') }}: ${{ if ne(parameters.platform.pool, '') }}:
pool: ${{ parameters.platform.pool }} pool: ${{ parameters.platform.pool }}

62
eng/common/templates/job/source-index-stage1.yml

@ -0,0 +1,62 @@
parameters:
runAsPublic: false
sourceIndexPackageVersion: 1.0.1-20210614.1
sourceIndexPackageSource: https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools/nuget/v3/index.json
sourceIndexBuildCommand: powershell -NoLogo -NoProfile -ExecutionPolicy Bypass -Command "eng/common/build.ps1 -restore -build -binarylog -ci"
preSteps: []
binlogPath: artifacts/log/Debug/Build.binlog
pool:
vmImage: vs2017-win2016
condition: ''
dependsOn: ''
jobs:
- job: SourceIndexStage1
dependsOn: ${{ parameters.dependsOn }}
condition: ${{ parameters.condition }}
variables:
- name: SourceIndexPackageVersion
value: ${{ parameters.sourceIndexPackageVersion }}
- name: SourceIndexPackageSource
value: ${{ parameters.sourceIndexPackageSource }}
- name: BinlogPath
value: ${{ parameters.binlogPath }}
- ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}:
- group: source-dot-net stage1 variables
pool: ${{ parameters.pool }}
steps:
- ${{ each preStep in parameters.preSteps }}:
- ${{ preStep }}
- task: UseDotNet@2
displayName: Use .NET Core sdk 3.1
inputs:
packageType: sdk
version: 3.1.x
- task: UseDotNet@2
displayName: Use .NET Core sdk
inputs:
useGlobalJson: true
- script: |
dotnet tool install BinLogToSln --version $(SourceIndexPackageVersion) --add-source $(SourceIndexPackageSource) --tool-path .source-index/tools
dotnet tool install UploadIndexStage1 --version $(SourceIndexPackageVersion) --add-source $(SourceIndexPackageSource) --tool-path .source-index/tools
echo ##vso[task.prependpath]$(Build.SourcesDirectory)/.source-index/tools
displayName: Download Tools
- script: ${{ parameters.sourceIndexBuildCommand }}
displayName: Build Repository
- script: BinLogToSln -i $(BinlogPath) -r $(Build.SourcesDirectory) -n $(Build.Repository.Name) -o .source-index/stage1output
displayName: Process Binlog into indexable sln
env:
DOTNET_ROLL_FORWARD_ON_NO_CANDIDATE_FX: 2
- ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}:
- script: UploadIndexStage1 -i .source-index/stage1output -n $(Build.Repository.Name)
displayName: Upload stage1 artifacts to source index
env:
BLOB_CONTAINER_URL: $(source-dot-net-stage1-blob-container-url)
DOTNET_ROLL_FORWARD_ON_NO_CANDIDATE_FX: 2

29
eng/common/templates/jobs/jobs.yml

@ -7,7 +7,14 @@ parameters:
# Optional: Enable publishing using release pipelines # Optional: Enable publishing using release pipelines
enablePublishUsingPipelines: false enablePublishUsingPipelines: false
# Optional: Enable running the source-build jobs to build repo from source
enableSourceBuild: false
# Optional: Parameters for source-build template.
# See /eng/common/templates/jobs/source-build.yml for options
sourceBuildParameters: []
graphFileGeneration: graphFileGeneration:
# Optional: Enable generating the graph files at the end of the build # Optional: Enable generating the graph files at the end of the build
enabled: false enabled: false
@ -24,12 +31,8 @@ parameters:
# if 'true', the build won't run any of the internal only steps, even if it is running in non-public projects. # if 'true', the build won't run any of the internal only steps, even if it is running in non-public projects.
runAsPublic: false runAsPublic: false
# Optional: Enable running the source-build jobs to build repo from source enableSourceIndex: false
runSourceBuild: false sourceIndexParams: {}
# Optional: Parameters for source-build template.
# See /eng/common/templates/jobs/source-build.yml for options
sourceBuildParameters: []
# Internal resources (telemetry, microbuild) can only be accessed from non-public projects, # Internal resources (telemetry, microbuild) can only be accessed from non-public projects,
# and some (Microbuild) should only be applied to non-PR cases for internal builds. # and some (Microbuild) should only be applied to non-PR cases for internal builds.
@ -50,14 +53,22 @@ jobs:
name: ${{ job.job }} name: ${{ job.job }}
- ${{ if eq(parameters.runSourceBuild, true) }}: - ${{ if eq(parameters.enableSourceBuild, true) }}:
- template: /eng/common/templates/jobs/source-build.yml - template: /eng/common/templates/jobs/source-build.yml
parameters: parameters:
allCompletedJobId: Source_Build_Complete allCompletedJobId: Source_Build_Complete
${{ each parameter in parameters.sourceBuildParameters }}: ${{ each parameter in parameters.sourceBuildParameters }}:
${{ parameter.key }}: ${{ parameter.value }} ${{ parameter.key }}: ${{ parameter.value }}
- ${{ if eq(parameters.enableSourceIndex, 'true') }}:
- template: ../job/source-index-stage1.yml
parameters:
runAsPublic: ${{ parameters.runAsPublic }}
${{ each parameter in parameters.sourceIndexParams }}:
${{ parameter.key }}: ${{ parameter.value }}
- ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}:
- ${{ if or(eq(parameters.enablePublishBuildAssets, true), eq(parameters.artifacts.publish.manifests, 'true'), ne(parameters.artifacts.publish.manifests, '')) }}: - ${{ if or(eq(parameters.enablePublishBuildAssets, true), eq(parameters.artifacts.publish.manifests, 'true'), ne(parameters.artifacts.publish.manifests, '')) }}:
- template: ../job/publish-build-assets.yml - template: ../job/publish-build-assets.yml
parameters: parameters:
@ -69,7 +80,7 @@ jobs:
- ${{ if eq(parameters.publishBuildAssetsDependsOn, '') }}: - ${{ if eq(parameters.publishBuildAssetsDependsOn, '') }}:
- ${{ each job in parameters.jobs }}: - ${{ each job in parameters.jobs }}:
- ${{ job.job }} - ${{ job.job }}
- ${{ if eq(parameters.runSourceBuild, true) }}: - ${{ if eq(parameters.enableSourceBuild, true) }}:
- Source_Build_Complete - Source_Build_Complete
pool: pool:
vmImage: vs2017-win2016 vmImage: vs2017-win2016

12
eng/common/templates/jobs/source-build.yml

@ -11,16 +11,14 @@ parameters:
# See /eng/common/templates/job/source-build.yml # See /eng/common/templates/job/source-build.yml
jobNamePrefix: 'Source_Build' jobNamePrefix: 'Source_Build'
# If changed to true, causes this template to include the default platform for a managed-only # This is the default platform provided by Arcade, intended for use by a managed-only repo.
# repo. The exact Docker image used for this build will be provided by Arcade. This has some risk,
# but since the repo is supposed to be managed-only, the risk should be very low.
includeDefaultManagedPlatform: false
defaultManagedPlatform: defaultManagedPlatform:
name: 'Managed' name: 'Managed'
container: 'mcr.microsoft.com/dotnet-buildtools/prereqs:centos-7-3e800f1-20190501005343' container: 'mcr.microsoft.com/dotnet-buildtools/prereqs:centos-7-3e800f1-20190501005343'
# Defines the platforms on which to run build jobs. One job is created for each platform, and the # Defines the platforms on which to run build jobs. One job is created for each platform, and the
# object in this array is sent to the job template as 'platform'. # object in this array is sent to the job template as 'platform'. If no platforms are specified,
# one job runs on 'defaultManagedPlatform'.
platforms: [] platforms: []
jobs: jobs:
@ -32,7 +30,7 @@ jobs:
dependsOn: dependsOn:
- ${{ each platform in parameters.platforms }}: - ${{ each platform in parameters.platforms }}:
- ${{ parameters.jobNamePrefix }}_${{ platform.name }} - ${{ parameters.jobNamePrefix }}_${{ platform.name }}
- ${{ if eq(parameters.includeDefaultManagedPlatform, true) }}: - ${{ if eq(length(parameters.platforms), 0) }}:
- ${{ parameters.jobNamePrefix }}_${{ parameters.defaultManagedPlatform.name }} - ${{ parameters.jobNamePrefix }}_${{ parameters.defaultManagedPlatform.name }}
- ${{ each platform in parameters.platforms }}: - ${{ each platform in parameters.platforms }}:
@ -41,7 +39,7 @@ jobs:
jobNamePrefix: ${{ parameters.jobNamePrefix }} jobNamePrefix: ${{ parameters.jobNamePrefix }}
platform: ${{ platform }} platform: ${{ platform }}
- ${{ if eq(parameters.includeDefaultManagedPlatform, true) }}: - ${{ if eq(length(parameters.platforms), 0) }}:
- template: /eng/common/templates/job/source-build.yml - template: /eng/common/templates/job/source-build.yml
parameters: parameters:
jobNamePrefix: ${{ parameters.jobNamePrefix }} jobNamePrefix: ${{ parameters.jobNamePrefix }}

1
eng/common/templates/phases/publish-build-assets.yml

@ -20,6 +20,7 @@ phases:
inputs: inputs:
artifactName: AssetManifests artifactName: AssetManifests
downloadPath: '$(Build.StagingDirectory)/Download' downloadPath: '$(Build.StagingDirectory)/Download'
checkDownloadedFiles: true
condition: ${{ parameters.condition }} condition: ${{ parameters.condition }}
continueOnError: ${{ parameters.continueOnError }} continueOnError: ${{ parameters.continueOnError }}
- task: AzureKeyVault@1 - task: AzureKeyVault@1

8
eng/common/templates/post-build/channels/generic-internal-channel.yml

@ -40,6 +40,9 @@ stages:
pool: pool:
vmImage: 'windows-2019' vmImage: 'windows-2019'
steps: steps:
- script: echo "##vso[task.logissue type=warning]Going forward, v2 Arcade publishing is no longer supported. Please read https://github.com/dotnet/arcade/blob/main/Documentation/CorePackages/Publishing.md for details, then contact dnceng if you have further questions."
displayName: Warn about v2 Arcade Publishing Usage
# This is necessary whenever we want to publish/restore to an AzDO private feed # This is necessary whenever we want to publish/restore to an AzDO private feed
- task: NuGetAuthenticate@0 - task: NuGetAuthenticate@0
displayName: 'Authenticate to AzDO Feeds' displayName: 'Authenticate to AzDO Feeds'
@ -58,6 +61,7 @@ stages:
PdbArtifacts/** PdbArtifacts/**
BlobArtifacts/** BlobArtifacts/**
downloadPath: '$(Build.ArtifactStagingDirectory)' downloadPath: '$(Build.ArtifactStagingDirectory)'
checkDownloadedFiles: true
# This is necessary whenever we want to publish/restore to an AzDO private feed # This is necessary whenever we want to publish/restore to an AzDO private feed
# Since sdk-task.ps1 tries to restore packages we need to do this authentication here # Since sdk-task.ps1 tries to restore packages we need to do this authentication here
@ -109,6 +113,9 @@ stages:
pool: pool:
vmImage: 'windows-2019' vmImage: 'windows-2019'
steps: steps:
- script: echo "##vso[task.logissue type=warning]Going forward, v2 Arcade publishing is no longer supported. Please read https://github.com/dotnet/arcade/blob/main/Documentation/CorePackages/Publishing.md for details, then contact dnceng if you have further questions."
displayName: Warn about v2 Arcade Publishing Usage
- task: DownloadBuildArtifacts@0 - task: DownloadBuildArtifacts@0
displayName: Download Build Assets displayName: Download Build Assets
continueOnError: true continueOnError: true
@ -124,6 +131,7 @@ stages:
BlobArtifacts/** BlobArtifacts/**
AssetManifests/** AssetManifests/**
downloadPath: '$(Build.ArtifactStagingDirectory)' downloadPath: '$(Build.ArtifactStagingDirectory)'
checkDownloadedFiles: true
- task: NuGetToolInstaller@1 - task: NuGetToolInstaller@1
displayName: 'Install NuGet.exe' displayName: 'Install NuGet.exe'

8
eng/common/templates/post-build/channels/generic-public-channel.yml

@ -42,6 +42,9 @@ stages:
pool: pool:
vmImage: 'windows-2019' vmImage: 'windows-2019'
steps: steps:
- script: echo "##vso[task.logissue type=warning]Going forward, v2 Arcade publishing is no longer supported. Please read https://github.com/dotnet/arcade/blob/main/Documentation/CorePackages/Publishing.md for details, then contact dnceng if you have further questions."
displayName: Warn about v2 Arcade Publishing Usage
- task: DownloadBuildArtifacts@0 - task: DownloadBuildArtifacts@0
displayName: Download Build Assets displayName: Download Build Assets
continueOnError: true continueOnError: true
@ -56,6 +59,7 @@ stages:
PdbArtifacts/** PdbArtifacts/**
BlobArtifacts/** BlobArtifacts/**
downloadPath: '$(Build.ArtifactStagingDirectory)' downloadPath: '$(Build.ArtifactStagingDirectory)'
checkDownloadedFiles: true
# This is necessary whenever we want to publish/restore to an AzDO private feed # This is necessary whenever we want to publish/restore to an AzDO private feed
# Since sdk-task.ps1 tries to restore packages we need to do this authentication here # Since sdk-task.ps1 tries to restore packages we need to do this authentication here
@ -108,6 +112,9 @@ stages:
pool: pool:
vmImage: 'windows-2019' vmImage: 'windows-2019'
steps: steps:
- script: echo "##vso[task.logissue type=warning]Going forward, v2 Arcade publishing is no longer supported. Please read https://github.com/dotnet/arcade/blob/main/Documentation/CorePackages/Publishing.md for details, then contact dnceng if you have further questions."
displayName: Warn about v2 Arcade Publishing Usage
- task: DownloadBuildArtifacts@0 - task: DownloadBuildArtifacts@0
displayName: Download Build Assets displayName: Download Build Assets
continueOnError: true continueOnError: true
@ -123,6 +130,7 @@ stages:
BlobArtifacts/** BlobArtifacts/**
AssetManifests/** AssetManifests/**
downloadPath: '$(Build.ArtifactStagingDirectory)' downloadPath: '$(Build.ArtifactStagingDirectory)'
checkDownloadedFiles: true
- task: NuGetToolInstaller@1 - task: NuGetToolInstaller@1
displayName: 'Install NuGet.exe' displayName: 'Install NuGet.exe'

391
eng/common/templates/post-build/post-build.yml

@ -32,7 +32,6 @@ parameters:
symbolPublishingAdditionalParameters: '' symbolPublishingAdditionalParameters: ''
artifactsPublishingAdditionalParameters: '' artifactsPublishingAdditionalParameters: ''
signingValidationAdditionalParameters: '' signingValidationAdditionalParameters: ''
useBuildManifest: false
# Which stages should finish execution before post-build stages start # Which stages should finish execution before post-build stages start
validateDependsOn: validateDependsOn:
@ -54,9 +53,6 @@ parameters:
NETCoreExperimentalChannelId: 562 NETCoreExperimentalChannelId: 562
NetEngServicesIntChannelId: 678 NetEngServicesIntChannelId: 678
NetEngServicesProdChannelId: 679 NetEngServicesProdChannelId: 679
Net5Preview8ChannelId: 1155
Net5RC1ChannelId: 1157
Net5RC2ChannelId: 1329
NetCoreSDK313xxChannelId: 759 NetCoreSDK313xxChannelId: 759
NetCoreSDK313xxInternalChannelId: 760 NetCoreSDK313xxInternalChannelId: 760
NetCoreSDK314xxChannelId: 921 NetCoreSDK314xxChannelId: 921
@ -65,178 +61,180 @@ parameters:
VS167ChannelId: 1011 VS167ChannelId: 1011
VS168ChannelId: 1154 VS168ChannelId: 1154
VSMasterChannelId: 1012 VSMasterChannelId: 1012
VS169ChannelId: 1473
VS1610ChannelId: 1692
stages: stages:
- stage: Validate - ${{ if or(and(le(parameters.publishingInfraVersion, 2), eq(parameters.inline, 'true')), eq( parameters.enableNugetValidation, 'true'), eq(parameters.enableSigningValidation, 'true'), eq(parameters.enableSourceLinkValidation, 'true'), eq(parameters.SDLValidationParameters.enable, 'true')) }}:
dependsOn: ${{ parameters.validateDependsOn }} - stage: Validate
displayName: Validate Build Assets dependsOn: ${{ parameters.validateDependsOn }}
variables: displayName: Validate Build Assets
- template: common-variables.yml variables:
jobs: - template: common-variables.yml
- template: setup-maestro-vars.yml jobs:
parameters: - template: setup-maestro-vars.yml
BARBuildId: ${{ parameters.BARBuildId }} parameters:
PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }} BARBuildId: ${{ parameters.BARBuildId }}
PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }}
- ${{ if and(le(parameters.publishingInfraVersion, 2), eq(parameters.inline, 'true')) }}:
- job:
displayName: Post-build Checks
dependsOn: setupMaestroVars
variables:
- name: TargetChannels
value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.TargetChannels'] ]
pool:
vmImage: 'windows-2019'
steps:
- task: PowerShell@2
displayName: Maestro Channels Consistency
inputs:
filePath: $(Build.SourcesDirectory)/eng/common/post-build/check-channel-consistency.ps1
arguments: -PromoteToChannels "$(TargetChannels)"
-AvailableChannelIds ${{parameters.NetEngLatestChannelId}},${{parameters.NetEngValidationChannelId}},${{parameters.NetDev5ChannelId}},${{parameters.NetDev6ChannelId}},${{parameters.GeneralTestingChannelId}},${{parameters.NETCoreToolingDevChannelId}},${{parameters.NETCoreToolingReleaseChannelId}},${{parameters.NETInternalToolingChannelId}},${{parameters.NETCoreExperimentalChannelId}},${{parameters.NetEngServicesIntChannelId}},${{parameters.NetEngServicesProdChannelId}},${{parameters.NetCoreSDK313xxChannelId}},${{parameters.NetCoreSDK313xxInternalChannelId}},${{parameters.NetCoreSDK314xxChannelId}},${{parameters.NetCoreSDK314xxInternalChannelId}},${{parameters.VS166ChannelId}},${{parameters.VS167ChannelId}},${{parameters.VS168ChannelId}},${{parameters.VSMasterChannelId}},${{parameters.VS169ChannelId}},${{parameters.VS1610ChannelId}}
- ${{ if and(le(parameters.publishingInfraVersion, 2), eq(parameters.inline, 'true')) }}:
- job: - job:
displayName: Post-build Checks displayName: NuGet Validation
dependsOn: setupMaestroVars dependsOn: setupMaestroVars
variables: condition: eq( ${{ parameters.enableNugetValidation }}, 'true')
- name: TargetChannels
value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.TargetChannels'] ]
pool: pool:
vmImage: 'windows-2019' vmImage: 'windows-2019'
variables:
- name: AzDOProjectName
value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOProjectName'] ]
- name: AzDOPipelineId
value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOPipelineId'] ]
- name: AzDOBuildId
value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOBuildId'] ]
steps: steps:
- task: DownloadBuildArtifacts@0
displayName: Download Package Artifacts
inputs:
buildType: specific
buildVersionToDownload: specific
project: $(AzDOProjectName)
pipeline: $(AzDOPipelineId)
buildId: $(AzDOBuildId)
artifactName: PackageArtifacts
checkDownloadedFiles: true
- task: PowerShell@2 - task: PowerShell@2
displayName: Maestro Channels Consistency displayName: Validate
inputs: inputs:
filePath: $(Build.SourcesDirectory)/eng/common/post-build/check-channel-consistency.ps1 filePath: $(Build.SourcesDirectory)/eng/common/post-build/nuget-validation.ps1
arguments: -PromoteToChannels "$(TargetChannels)" arguments: -PackagesPath $(Build.ArtifactStagingDirectory)/PackageArtifacts/
-AvailableChannelIds ${{parameters.NetEngLatestChannelId}},${{parameters.NetEngValidationChannelId}},${{parameters.NetDev5ChannelId}},${{parameters.NetDev6ChannelId}},${{parameters.GeneralTestingChannelId}},${{parameters.NETCoreToolingDevChannelId}},${{parameters.NETCoreToolingReleaseChannelId}},${{parameters.NETInternalToolingChannelId}},${{parameters.NETCoreExperimentalChannelId}},${{parameters.NetEngServicesIntChannelId}},${{parameters.NetEngServicesProdChannelId}},${{parameters.Net5Preview8ChannelId}},${{parameters.Net5RC1ChannelId}},${{parameters.Net5RC2ChannelId}},${{parameters.NetCoreSDK313xxChannelId}},${{parameters.NetCoreSDK313xxInternalChannelId}},${{parameters.NetCoreSDK314xxChannelId}},${{parameters.NetCoreSDK314xxInternalChannelId}},${{parameters.VS166ChannelId}},${{parameters.VS167ChannelId}},${{parameters.VS168ChannelId}},${{parameters.VSMasterChannelId}} -ToolDestinationPath $(Agent.BuildDirectory)/Extract/
- job: - job:
displayName: NuGet Validation displayName: Signing Validation
dependsOn: setupMaestroVars dependsOn: setupMaestroVars
condition: eq( ${{ parameters.enableNugetValidation }}, 'true') condition: and( eq( ${{ parameters.enableSigningValidation }}, 'true'), ne( variables['PostBuildSign'], 'true'))
pool: variables:
vmImage: 'windows-2019' - template: common-variables.yml
variables: - name: AzDOProjectName
- name: AzDOProjectName value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOProjectName'] ]
value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOProjectName'] ] - name: AzDOPipelineId
- name: AzDOPipelineId value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOPipelineId'] ]
value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOPipelineId'] ] - name: AzDOBuildId
- name: AzDOBuildId value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOBuildId'] ]
value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOBuildId'] ] pool:
steps: vmImage: 'windows-2019'
- task: DownloadBuildArtifacts@0 steps:
displayName: Download Package Artifacts
inputs:
buildType: specific
buildVersionToDownload: specific
project: $(AzDOProjectName)
pipeline: $(AzDOPipelineId)
buildId: $(AzDOBuildId)
artifactName: PackageArtifacts
- task: PowerShell@2
displayName: Validate
inputs:
filePath: $(Build.SourcesDirectory)/eng/common/post-build/nuget-validation.ps1
arguments: -PackagesPath $(Build.ArtifactStagingDirectory)/PackageArtifacts/
-ToolDestinationPath $(Agent.BuildDirectory)/Extract/
- job:
displayName: Signing Validation
dependsOn: setupMaestroVars
condition: eq( ${{ parameters.enableSigningValidation }}, 'true')
variables:
- template: common-variables.yml
- name: AzDOProjectName
value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOProjectName'] ]
- name: AzDOPipelineId
value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOPipelineId'] ]
- name: AzDOBuildId
value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOBuildId'] ]
pool:
vmImage: 'windows-2019'
steps:
- ${{ if eq(parameters.useBuildManifest, true) }}:
- task: DownloadBuildArtifacts@0 - task: DownloadBuildArtifacts@0
displayName: Download build manifest displayName: Download Package Artifacts
inputs: inputs:
buildType: specific buildType: specific
buildVersionToDownload: specific buildVersionToDownload: specific
project: $(AzDOProjectName) project: $(AzDOProjectName)
pipeline: $(AzDOPipelineId) pipeline: $(AzDOPipelineId)
buildId: $(AzDOBuildId) buildId: $(AzDOBuildId)
artifactName: BuildManifests artifactName: PackageArtifacts
- task: DownloadBuildArtifacts@0 checkDownloadedFiles: true
displayName: Download Package Artifacts itemPattern: |
inputs: **
buildType: specific !**/Microsoft.SourceBuild.Intermediate.*.nupkg
buildVersionToDownload: specific
project: $(AzDOProjectName) # This is necessary whenever we want to publish/restore to an AzDO private feed
pipeline: $(AzDOPipelineId) # Since sdk-task.ps1 tries to restore packages we need to do this authentication here
buildId: $(AzDOBuildId) # otherwise it'll complain about accessing a private feed.
artifactName: PackageArtifacts - task: NuGetAuthenticate@0
displayName: 'Authenticate to AzDO Feeds'
# This is necessary whenever we want to publish/restore to an AzDO private feed
# Since sdk-task.ps1 tries to restore packages we need to do this authentication here - task: PowerShell@2
# otherwise it'll complain about accessing a private feed. displayName: Enable cross-org publishing
- task: NuGetAuthenticate@0 inputs:
displayName: 'Authenticate to AzDO Feeds' filePath: eng\common\enable-cross-org-publishing.ps1
arguments: -token $(dn-bot-dnceng-artifact-feeds-rw)
- task: PowerShell@2
displayName: Enable cross-org publishing # Signing validation will optionally work with the buildmanifest file which is downloaded from
inputs: # Azure DevOps above.
filePath: eng\common\enable-cross-org-publishing.ps1 - task: PowerShell@2
arguments: -token $(dn-bot-dnceng-artifact-feeds-rw) displayName: Validate
inputs:
# Signing validation will optionally work with the buildmanifest file which is downloaded from filePath: eng\common\sdk-task.ps1
# Azure DevOps above. arguments: -task SigningValidation -restore -msbuildEngine vs
- task: PowerShell@2 /p:PackageBasePath='$(Build.ArtifactStagingDirectory)/PackageArtifacts'
displayName: Validate /p:SignCheckExclusionsFile='$(Build.SourcesDirectory)/eng/SignCheckExclusionsFile.txt'
inputs: ${{ parameters.signingValidationAdditionalParameters }}
filePath: eng\common\sdk-task.ps1
arguments: -task SigningValidation -restore -msbuildEngine vs - template: ../steps/publish-logs.yml
/p:PackageBasePath='$(Build.ArtifactStagingDirectory)/PackageArtifacts' parameters:
/p:SignCheckExclusionsFile='$(Build.SourcesDirectory)/eng/SignCheckExclusionsFile.txt' StageLabel: 'Validation'
${{ parameters.signingValidationAdditionalParameters }} JobLabel: 'Signing'
- template: ../steps/publish-logs.yml - job:
parameters: displayName: SourceLink Validation
StageLabel: 'Validation'
JobLabel: 'Signing'
- job:
displayName: SourceLink Validation
dependsOn: setupMaestroVars
condition: eq( ${{ parameters.enableSourceLinkValidation }}, 'true')
variables:
- template: common-variables.yml
- name: AzDOProjectName
value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOProjectName'] ]
- name: AzDOPipelineId
value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOPipelineId'] ]
- name: AzDOBuildId
value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOBuildId'] ]
pool:
vmImage: 'windows-2019'
steps:
- task: DownloadBuildArtifacts@0
displayName: Download Blob Artifacts
inputs:
buildType: specific
buildVersionToDownload: specific
project: $(AzDOProjectName)
pipeline: $(AzDOPipelineId)
buildId: $(AzDOBuildId)
artifactName: BlobArtifacts
- task: PowerShell@2
displayName: Validate
inputs:
filePath: $(Build.SourcesDirectory)/eng/common/post-build/sourcelink-validation.ps1
arguments: -InputPath $(Build.ArtifactStagingDirectory)/BlobArtifacts/
-ExtractPath $(Agent.BuildDirectory)/Extract/
-GHRepoName $(Build.Repository.Name)
-GHCommit $(Build.SourceVersion)
-SourcelinkCliVersion $(SourceLinkCLIVersion)
continueOnError: true
- template: /eng/common/templates/job/execute-sdl.yml
parameters:
enable: ${{ parameters.SDLValidationParameters.enable }}
dependsOn: setupMaestroVars dependsOn: setupMaestroVars
additionalParameters: ${{ parameters.SDLValidationParameters.params }} condition: eq( ${{ parameters.enableSourceLinkValidation }}, 'true')
continueOnError: ${{ parameters.SDLValidationParameters.continueOnError }} variables:
artifactNames: ${{ parameters.SDLValidationParameters.artifactNames }} - template: common-variables.yml
downloadArtifacts: ${{ parameters.SDLValidationParameters.downloadArtifacts }} - name: AzDOProjectName
value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOProjectName'] ]
- name: AzDOPipelineId
value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOPipelineId'] ]
- name: AzDOBuildId
value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOBuildId'] ]
pool:
vmImage: 'windows-2019'
steps:
- task: DownloadBuildArtifacts@0
displayName: Download Blob Artifacts
inputs:
buildType: specific
buildVersionToDownload: specific
project: $(AzDOProjectName)
pipeline: $(AzDOPipelineId)
buildId: $(AzDOBuildId)
artifactName: BlobArtifacts
checkDownloadedFiles: true
- task: PowerShell@2
displayName: Validate
inputs:
filePath: $(Build.SourcesDirectory)/eng/common/post-build/sourcelink-validation.ps1
arguments: -InputPath $(Build.ArtifactStagingDirectory)/BlobArtifacts/
-ExtractPath $(Agent.BuildDirectory)/Extract/
-GHRepoName $(Build.Repository.Name)
-GHCommit $(Build.SourceVersion)
-SourcelinkCliVersion $(SourceLinkCLIVersion)
continueOnError: true
- template: /eng/common/templates/job/execute-sdl.yml
parameters:
enable: ${{ parameters.SDLValidationParameters.enable }}
dependsOn: setupMaestroVars
additionalParameters: ${{ parameters.SDLValidationParameters.params }}
continueOnError: ${{ parameters.SDLValidationParameters.continueOnError }}
artifactNames: ${{ parameters.SDLValidationParameters.artifactNames }}
downloadArtifacts: ${{ parameters.SDLValidationParameters.downloadArtifacts }}
- ${{ if or(ge(parameters.publishingInfraVersion, 3), eq(parameters.inline, 'false')) }}: - ${{ if or(ge(parameters.publishingInfraVersion, 3), eq(parameters.inline, 'false')) }}:
- stage: publish_using_darc - stage: publish_using_darc
dependsOn: Validate ${{ if or(eq(parameters.enableNugetValidation, 'true'), eq(parameters.enableSigningValidation, 'true'), eq(parameters.enableSourceLinkValidation, 'true'), eq(parameters.SDLValidationParameters.enable, 'true')) }}:
dependsOn: ${{ parameters.publishDependsOn }}
${{ if and(ne(parameters.enableNugetValidation, 'true'), ne(parameters.enableSigningValidation, 'true'), ne(parameters.enableSourceLinkValidation, 'true'), ne(parameters.SDLValidationParameters.enable, 'true')) }}:
dependsOn: ${{ parameters.validateDependsOn }}
displayName: Publish using Darc displayName: Publish using Darc
variables: variables:
- template: common-variables.yml - template: common-variables.yml
@ -249,6 +247,7 @@ stages:
- job: - job:
displayName: Publish Using Darc displayName: Publish Using Darc
dependsOn: setupMaestroVars dependsOn: setupMaestroVars
timeoutInMinutes: 120
variables: variables:
- name: BARBuildId - name: BARBuildId
value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.BARBuildId'] ] value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.BARBuildId'] ]
@ -265,6 +264,8 @@ stages:
-MaestroToken '$(MaestroApiAccessToken)' -MaestroToken '$(MaestroApiAccessToken)'
-WaitPublishingFinish ${{ parameters.waitPublishingFinish }} -WaitPublishingFinish ${{ parameters.waitPublishingFinish }}
-PublishInstallersAndChecksums ${{ parameters.publishInstallersAndChecksums }} -PublishInstallersAndChecksums ${{ parameters.publishInstallersAndChecksums }}
-ArtifactsPublishingAdditionalParameters '${{ parameters.artifactsPublishingAdditionalParameters }}'
-SymbolPublishingAdditionalParameters '${{ parameters.symbolPublishingAdditionalParameters }}'
- ${{ if and(le(parameters.publishingInfraVersion, 2), eq(parameters.inline, 'true')) }}: - ${{ if and(le(parameters.publishingInfraVersion, 2), eq(parameters.inline, 'true')) }}:
- template: \eng\common\templates\post-build\channels\generic-public-channel.yml - template: \eng\common\templates\post-build\channels\generic-public-channel.yml
@ -299,54 +300,6 @@ stages:
shippingFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet6/nuget/v3/index.json' shippingFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet6/nuget/v3/index.json'
symbolsFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet6-symbols/nuget/v3/index.json' symbolsFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet6-symbols/nuget/v3/index.json'
- template: \eng\common\templates\post-build\channels\generic-internal-channel.yml
parameters:
BARBuildId: ${{ parameters.BARBuildId }}
PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }}
artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }}
dependsOn: ${{ parameters.publishDependsOn }}
publishInstallersAndChecksums: ${{ parameters.publishInstallersAndChecksums }}
symbolPublishingAdditionalParameters: ${{ parameters.symbolPublishingAdditionalParameters }}
stageName: 'Net5_Preview8_Publish'
channelName: '.NET 5 Preview 8'
akaMSChannelName: 'net5/preview8'
channelId: ${{ parameters.Net5Preview8ChannelId }}
transportFeed: 'https://pkgs.dev.azure.com/dnceng/internal/_packaging/dotnet5-internal-transport/nuget/v3/index.json'
shippingFeed: 'https://pkgs.dev.azure.com/dnceng/internal/_packaging/dotnet5-internal/nuget/v3/index.json'
symbolsFeed: 'https://pkgs.dev.azure.com/dnceng/internal/_packaging/dotnet5-internal-symbols/nuget/v3/index.json'
- template: \eng\common\templates\post-build\channels\generic-public-channel.yml
parameters:
BARBuildId: ${{ parameters.BARBuildId }}
PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }}
artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }}
dependsOn: ${{ parameters.publishDependsOn }}
publishInstallersAndChecksums: ${{ parameters.publishInstallersAndChecksums }}
symbolPublishingAdditionalParameters: ${{ parameters.symbolPublishingAdditionalParameters }}
stageName: 'Net5_RC1_Publish'
channelName: '.NET 5 RC 1'
akaMSChannelName: 'net5/rc1'
channelId: ${{ parameters.Net5RC1ChannelId }}
transportFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet5-transport/nuget/v3/index.json'
shippingFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet5/nuget/v3/index.json'
symbolsFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet5-symbols/nuget/v3/index.json'
- template: \eng\common\templates\post-build\channels\generic-public-channel.yml
parameters:
BARBuildId: ${{ parameters.BARBuildId }}
PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }}
artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }}
dependsOn: ${{ parameters.publishDependsOn }}
publishInstallersAndChecksums: ${{ parameters.publishInstallersAndChecksums }}
symbolPublishingAdditionalParameters: ${{ parameters.symbolPublishingAdditionalParameters }}
stageName: 'Net5_RC2_Publish'
channelName: '.NET 5 RC 2'
akaMSChannelName: 'net5/rc2'
channelId: ${{ parameters.Net5RC2ChannelId }}
transportFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet5-transport/nuget/v3/index.json'
shippingFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet5/nuget/v3/index.json'
symbolsFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet5-symbols/nuget/v3/index.json'
- template: \eng\common\templates\post-build\channels\generic-public-channel.yml - template: \eng\common\templates\post-build\channels\generic-public-channel.yml
parameters: parameters:
BARBuildId: ${{ parameters.BARBuildId }} BARBuildId: ${{ parameters.BARBuildId }}
@ -604,3 +557,33 @@ stages:
transportFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools-transport/nuget/v3/index.json' transportFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools-transport/nuget/v3/index.json'
shippingFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools/nuget/v3/index.json' shippingFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools/nuget/v3/index.json'
symbolsFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools-symbols/nuget/v3/index.json' symbolsFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools-symbols/nuget/v3/index.json'
- template: \eng\common\templates\post-build\channels\generic-public-channel.yml
parameters:
BARBuildId: ${{ parameters.BARBuildId }}
PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }}
artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }}
dependsOn: ${{ parameters.publishDependsOn }}
publishInstallersAndChecksums: ${{ parameters.publishInstallersAndChecksums }}
symbolPublishingAdditionalParameters: ${{ parameters.symbolPublishingAdditionalParameters }}
stageName: 'VS_16_9_Publishing'
channelName: 'VS 16.9'
channelId: ${{ parameters.VS169ChannelId }}
transportFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools-transport/nuget/v3/index.json'
shippingFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools/nuget/v3/index.json'
symbolsFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools-symbols/nuget/v3/index.json'
- template: \eng\common\templates\post-build\channels\generic-public-channel.yml
parameters:
BARBuildId: ${{ parameters.BARBuildId }}
PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }}
artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }}
dependsOn: ${{ parameters.publishDependsOn }}
publishInstallersAndChecksums: ${{ parameters.publishInstallersAndChecksums }}
symbolPublishingAdditionalParameters: ${{ parameters.symbolPublishingAdditionalParameters }}
stageName: 'VS_16_10_Publishing'
channelName: 'VS 16.10'
channelId: ${{ parameters.VS1610ChannelId }}
transportFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools-transport/nuget/v3/index.json'
shippingFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools/nuget/v3/index.json'
symbolsFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools-symbols/nuget/v3/index.json'

1
eng/common/templates/post-build/setup-maestro-vars.yml

@ -18,6 +18,7 @@ jobs:
inputs: inputs:
buildType: current buildType: current
artifactName: ReleaseConfigs artifactName: ReleaseConfigs
checkDownloadedFiles: true
- task: PowerShell@2 - task: PowerShell@2
name: setReleaseVars name: setReleaseVars

4
eng/common/templates/steps/send-to-helix.yml

@ -18,8 +18,8 @@ parameters:
XUnitRuntimeTargetFramework: '' # optional -- framework to use for the xUnit console runner XUnitRuntimeTargetFramework: '' # optional -- framework to use for the xUnit console runner
XUnitRunnerVersion: '' # optional -- version of the xUnit nuget package you wish to use on Helix; required for XUnitProjects XUnitRunnerVersion: '' # optional -- version of the xUnit nuget package you wish to use on Helix; required for XUnitProjects
IncludeDotNetCli: false # optional -- true will download a version of the .NET CLI onto the Helix machine as a correlation payload; requires DotNetCliPackageType and DotNetCliVersion IncludeDotNetCli: false # optional -- true will download a version of the .NET CLI onto the Helix machine as a correlation payload; requires DotNetCliPackageType and DotNetCliVersion
DotNetCliPackageType: '' # optional -- either 'sdk', 'runtime' or 'aspnetcore-runtime'; determines whether the sdk or runtime will be sent to Helix; see https://raw.githubusercontent.com/dotnet/core/master/release-notes/releases-index.json DotNetCliPackageType: '' # optional -- either 'sdk', 'runtime' or 'aspnetcore-runtime'; determines whether the sdk or runtime will be sent to Helix; see https://raw.githubusercontent.com/dotnet/core/main/release-notes/releases-index.json
DotNetCliVersion: '' # optional -- version of the CLI to send to Helix; based on this: https://raw.githubusercontent.com/dotnet/core/master/release-notes/releases-index.json DotNetCliVersion: '' # optional -- version of the CLI to send to Helix; based on this: https://raw.githubusercontent.com/dotnet/core/main/release-notes/releases-index.json
EnableXUnitReporter: false # optional -- true enables XUnit result reporting to Mission Control EnableXUnitReporter: false # optional -- true enables XUnit result reporting to Mission Control
WaitForWorkItemCompletion: true # optional -- true will make the task wait until work items have been completed and fail the build if work items fail. False is "fire and forget." WaitForWorkItemCompletion: true # optional -- true will make the task wait until work items have been completed and fail the build if work items fail. False is "fire and forget."
IsExternal: false # [DEPRECATED] -- doesn't do anything, jobs are external if HelixAccessToken is empty and Creator is set IsExternal: false # [DEPRECATED] -- doesn't do anything, jobs are external if HelixAccessToken is empty and Creator is set

7
eng/common/templates/steps/source-build.yml

@ -34,9 +34,14 @@ steps:
targetRidArgs='/p:TargetRid=${{ parameters.platform.targetRID }}' targetRidArgs='/p:TargetRid=${{ parameters.platform.targetRID }}'
fi fi
publishArgs=
if [ '${{ parameters.platform.skipPublishValidation }}' != 'true' ]; then
publishArgs='--publish'
fi
${{ coalesce(parameters.platform.buildScript, './build.sh') }} --ci \ ${{ coalesce(parameters.platform.buildScript, './build.sh') }} --ci \
--configuration $buildConfig \ --configuration $buildConfig \
--restore --build --pack --publish \ --restore --build --pack $publishArgs -bl \
$officialBuildArgs \ $officialBuildArgs \
$targetRidArgs \ $targetRidArgs \
/p:SourceBuildNonPortable=${{ parameters.platform.nonPortable }} \ /p:SourceBuildNonPortable=${{ parameters.platform.nonPortable }} \

164
eng/common/tools.ps1

@ -48,6 +48,9 @@
# True to use global NuGet cache instead of restoring packages to repository-local directory. # True to use global NuGet cache instead of restoring packages to repository-local directory.
[bool]$useGlobalNuGetCache = if (Test-Path variable:useGlobalNuGetCache) { $useGlobalNuGetCache } else { !$ci } [bool]$useGlobalNuGetCache = if (Test-Path variable:useGlobalNuGetCache) { $useGlobalNuGetCache } else { !$ci }
# True to exclude prerelease versions Visual Studio during build
[bool]$excludePrereleaseVS = if (Test-Path variable:excludePrereleaseVS) { $excludePrereleaseVS } else { $false }
# An array of names of processes to stop on script exit if prepareMachine is true. # An array of names of processes to stop on script exit if prepareMachine is true.
$processesToStopOnExit = if (Test-Path variable:processesToStopOnExit) { $processesToStopOnExit } else { @('msbuild', 'dotnet', 'vbcscompiler') } $processesToStopOnExit = if (Test-Path variable:processesToStopOnExit) { $processesToStopOnExit } else { @('msbuild', 'dotnet', 'vbcscompiler') }
@ -57,15 +60,11 @@ set-strictmode -version 2.0
$ErrorActionPreference = 'Stop' $ErrorActionPreference = 'Stop'
[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12 [Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12
# If specified, provides an alternate path for getting .NET Core SDKs and Runtimes. This script will still try public sources first. # If specifies, provides an alternate path for getting .NET Core SDKs and Runtimes. This script will still try public sources first.
[string]$runtimeSourceFeed = if (Test-Path variable:runtimeSourceFeed) { $runtimeSourceFeed } else { $null } [string]$runtimeSourceFeed = if (Test-Path variable:runtimeSourceFeed) { $runtimeSourceFeed } else { $null }
# Base-64 encoded SAS token that has permission to storage container described by $runtimeSourceFeed # Base-64 encoded SAS token that has permission to storage container described by $runtimeSourceFeed
[string]$runtimeSourceFeedKey = if (Test-Path variable:runtimeSourceFeedKey) { $runtimeSourceFeedKey } else { $null } [string]$runtimeSourceFeedKey = if (Test-Path variable:runtimeSourceFeedKey) { $runtimeSourceFeedKey } else { $null }
# If false, use copy of dotnet-install from /eng/common/dotnet-install-scripts (for custom behaviors).
# otherwise will fetch from public location.
[bool]$useDefaultDotnetInstall = if (Test-Path variable:useDefaultDotnetInstall) { $useDefaultDotnetInstall } else { $false }
function Create-Directory ([string[]] $path) { function Create-Directory ([string[]] $path) {
New-Item -Path $path -Force -ItemType 'Directory' | Out-Null New-Item -Path $path -Force -ItemType 'Directory' | Out-Null
} }
@ -145,7 +144,7 @@ function InitializeDotNetCli([bool]$install, [bool]$createSdkLocationFile) {
# Use dotnet installation specified in DOTNET_INSTALL_DIR if it contains the required SDK version, # Use dotnet installation specified in DOTNET_INSTALL_DIR if it contains the required SDK version,
# otherwise install the dotnet CLI and SDK to repo local .dotnet directory to avoid potential permission issues. # otherwise install the dotnet CLI and SDK to repo local .dotnet directory to avoid potential permission issues.
if ((-not $globalJsonHasRuntimes) -and ($env:DOTNET_INSTALL_DIR -ne $null) -and (Test-Path(Join-Path $env:DOTNET_INSTALL_DIR "sdk\$dotnetSdkVersion"))) { if ((-not $globalJsonHasRuntimes) -and (-not [string]::IsNullOrEmpty($env:DOTNET_INSTALL_DIR)) -and (Test-Path(Join-Path $env:DOTNET_INSTALL_DIR "sdk\$dotnetSdkVersion"))) {
$dotnetRoot = $env:DOTNET_INSTALL_DIR $dotnetRoot = $env:DOTNET_INSTALL_DIR
} else { } else {
$dotnetRoot = Join-Path $RepoRoot '.dotnet' $dotnetRoot = Join-Path $RepoRoot '.dotnet'
@ -173,7 +172,7 @@ function InitializeDotNetCli([bool]$install, [bool]$createSdkLocationFile) {
Set-Content -Path $sdkCacheFileTemp -Value $dotnetRoot Set-Content -Path $sdkCacheFileTemp -Value $dotnetRoot
try { try {
Rename-Item -Force -Path $sdkCacheFileTemp 'sdk.txt' Move-Item -Force $sdkCacheFileTemp (Join-Path $ToolsetDir 'sdk.txt')
} catch { } catch {
# Somebody beat us # Somebody beat us
Remove-Item -Path $sdkCacheFileTemp Remove-Item -Path $sdkCacheFileTemp
@ -194,49 +193,44 @@ function InitializeDotNetCli([bool]$install, [bool]$createSdkLocationFile) {
return $global:_DotNetInstallDir = $dotnetRoot return $global:_DotNetInstallDir = $dotnetRoot
} }
function GetDotNetInstallScript([string] $dotnetRoot) { function Retry($downloadBlock, $maxRetries = 5) {
$installScript = Join-Path $dotnetRoot 'dotnet-install.ps1' $retries = 1
if (!(Test-Path $installScript)) {
create-directory $dotnetroot
if ($useDefaultDotnetInstall) while($true) {
{ try {
$progresspreference = 'silentlycontinue' # don't display the console progress ui - it's a huge perf hit & $downloadBlock
break
}
catch {
Write-PipelineTelemetryError -Category 'InitializeToolset' -Message $_
}
$maxretries = 5 if (++$retries -le $maxRetries) {
$retries = 1 $delayInSeconds = [math]::Pow(2, $retries) - 1 # Exponential backoff
Write-Host "Retrying. Waiting for $delayInSeconds seconds before next attempt ($retries of $maxRetries)."
Start-Sleep -Seconds $delayInSeconds
}
else {
Write-PipelineTelemetryError -Category 'InitializeToolset' -Message "Unable to download file in $maxRetries attempts."
break
}
$uri = "https://dot.net/$dotnetinstallscriptversion/dotnet-install.ps1" }
}
while($true) { function GetDotNetInstallScript([string] $dotnetRoot) {
try { $installScript = Join-Path $dotnetRoot 'dotnet-install.ps1'
write-host "get $uri" if (!(Test-Path $installScript)) {
invoke-webrequest $uri -outfile $installscript Create-Directory $dotnetRoot
break $ProgressPreference = 'SilentlyContinue' # Don't display the console progress UI - it's a huge perf hit
} $uri = "https://dot.net/$dotnetInstallScriptVersion/dotnet-install.ps1"
catch {
write-host "failed to download '$uri'"
write-error $_.exception.message -erroraction continue
}
if (++$retries -le $maxretries) { Retry({
$delayinseconds = [math]::pow(2, $retries) - 1 # exponential backoff Write-Host "GET $uri"
write-host "retrying. waiting for $delayinseconds seconds before next attempt ($retries of $maxretries)." Invoke-WebRequest $uri -OutFile $installScript
start-sleep -seconds $delayinseconds })
}
else {
throw "unable to download file in $maxretries attempts."
}
}
}
else
{
# Use a special version of the script from eng/common that understands the existence of a "productVersion.txt" in a dotnet path.
# See https://github.com/dotnet/arcade/issues/6047 for details
$engCommonCopy = Resolve-Path (Join-Path $PSScriptRoot 'dotnet-install-scripts\dotnet-install.ps1')
Copy-Item $engCommonCopy -Destination $installScript -Force
}
} }
return $installScript return $installScript
} }
@ -318,8 +312,8 @@ function InitializeVisualStudioMSBuild([bool]$install, [object]$vsRequirements =
# If the version of msbuild is going to be xcopied, # If the version of msbuild is going to be xcopied,
# use this version. Version matches a package here: # use this version. Version matches a package here:
# https://dev.azure.com/dnceng/public/_packaging?_a=package&feed=dotnet-eng&package=RoslynTools.MSBuild&protocolType=NuGet&version=16.8.0-preview3&view=overview # https://dev.azure.com/dnceng/public/_packaging?_a=package&feed=dotnet-eng&package=RoslynTools.MSBuild&protocolType=NuGet&version=16.10.0-preview2&view=overview
$defaultXCopyMSBuildVersion = '16.8.0-preview3' $defaultXCopyMSBuildVersion = '16.10.0-preview2'
if (!$vsRequirements) { $vsRequirements = $GlobalJson.tools.vs } if (!$vsRequirements) { $vsRequirements = $GlobalJson.tools.vs }
$vsMinVersionStr = if ($vsRequirements.version) { $vsRequirements.version } else { $vsMinVersionReqdStr } $vsMinVersionStr = if ($vsRequirements.version) { $vsRequirements.version } else { $vsMinVersionReqdStr }
@ -413,9 +407,13 @@ function InitializeXCopyMSBuild([string]$packageVersion, [bool]$install) {
} }
Create-Directory $packageDir Create-Directory $packageDir
Write-Host "Downloading $packageName $packageVersion" Write-Host "Downloading $packageName $packageVersion"
$ProgressPreference = 'SilentlyContinue' # Don't display the console progress UI - it's a huge perf hit $ProgressPreference = 'SilentlyContinue' # Don't display the console progress UI - it's a huge perf hit
Invoke-WebRequest "https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-eng/nuget/v3/flat2/$packageName/$packageVersion/$packageName.$packageVersion.nupkg" -OutFile $packagePath Retry({
Invoke-WebRequest "https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-eng/nuget/v3/flat2/$packageName/$packageVersion/$packageName.$packageVersion.nupkg" -OutFile $packagePath
})
Unzip $packagePath $packageDir Unzip $packagePath $packageDir
} }
@ -452,16 +450,17 @@ function LocateVisualStudio([object]$vsRequirements = $null){
if (!(Test-Path $vsWhereExe)) { if (!(Test-Path $vsWhereExe)) {
Create-Directory $vsWhereDir Create-Directory $vsWhereDir
Write-Host 'Downloading vswhere' Write-Host 'Downloading vswhere'
try { Retry({
Invoke-WebRequest "https://netcorenativeassets.blob.core.windows.net/resource-packages/external/windows/vswhere/$vswhereVersion/vswhere.exe" -OutFile $vswhereExe Invoke-WebRequest "https://netcorenativeassets.blob.core.windows.net/resource-packages/external/windows/vswhere/$vswhereVersion/vswhere.exe" -OutFile $vswhereExe
} })
catch {
Write-PipelineTelemetryError -Category 'InitializeToolset' -Message $_
}
} }
if (!$vsRequirements) { $vsRequirements = $GlobalJson.tools.vs } if (!$vsRequirements) { $vsRequirements = $GlobalJson.tools.vs }
$args = @('-latest', '-prerelease', '-format', 'json', '-requires', 'Microsoft.Component.MSBuild', '-products', '*') $args = @('-latest', '-format', 'json', '-requires', 'Microsoft.Component.MSBuild', '-products', '*')
if (!$excludePrereleaseVS) {
$args += '-prerelease'
}
if (Get-Member -InputObject $vsRequirements -Name 'version') { if (Get-Member -InputObject $vsRequirements -Name 'version') {
$args += '-version' $args += '-version'
@ -487,7 +486,13 @@ function LocateVisualStudio([object]$vsRequirements = $null){
function InitializeBuildTool() { function InitializeBuildTool() {
if (Test-Path variable:global:_BuildTool) { if (Test-Path variable:global:_BuildTool) {
return $global:_BuildTool # If the requested msbuild parameters do not match, clear the cached variables.
if($global:_BuildTool.Contains('ExcludePrereleaseVS') -and $global:_BuildTool.ExcludePrereleaseVS -ne $excludePrereleaseVS) {
Remove-Item variable:global:_BuildTool
Remove-Item variable:global:_MSBuildExe
} else {
return $global:_BuildTool
}
} }
if (-not $msbuildEngine) { if (-not $msbuildEngine) {
@ -506,7 +511,7 @@ function InitializeBuildTool() {
ExitWithExitCode 1 ExitWithExitCode 1
} }
$dotnetPath = Join-Path $dotnetRoot (GetExecutableFileName 'dotnet') $dotnetPath = Join-Path $dotnetRoot (GetExecutableFileName 'dotnet')
$buildTool = @{ Path = $dotnetPath; Command = 'msbuild'; Tool = 'dotnet'; Framework = 'netcoreapp2.1' } $buildTool = @{ Path = $dotnetPath; Command = 'msbuild'; Tool = 'dotnet'; Framework = 'netcoreapp3.1' }
} elseif ($msbuildEngine -eq "vs") { } elseif ($msbuildEngine -eq "vs") {
try { try {
$msbuildPath = InitializeVisualStudioMSBuild -install:$restore $msbuildPath = InitializeVisualStudioMSBuild -install:$restore
@ -515,7 +520,7 @@ function InitializeBuildTool() {
ExitWithExitCode 1 ExitWithExitCode 1
} }
$buildTool = @{ Path = $msbuildPath; Command = ""; Tool = "vs"; Framework = "net472" } $buildTool = @{ Path = $msbuildPath; Command = ""; Tool = "vs"; Framework = "net472"; ExcludePrereleaseVS = $excludePrereleaseVS }
} else { } else {
Write-PipelineTelemetryError -Category 'InitializeToolset' -Message "Unexpected value of -msbuildEngine: '$msbuildEngine'." Write-PipelineTelemetryError -Category 'InitializeToolset' -Message "Unexpected value of -msbuildEngine: '$msbuildEngine'."
ExitWithExitCode 1 ExitWithExitCode 1
@ -540,7 +545,7 @@ function GetDefaultMSBuildEngine() {
function GetNuGetPackageCachePath() { function GetNuGetPackageCachePath() {
if ($env:NUGET_PACKAGES -eq $null) { if ($env:NUGET_PACKAGES -eq $null) {
# Use local cache on CI to ensure deterministic build. # Use local cache on CI to ensure deterministic build.
# Avoid using the http cache as workaround for https://github.com/NuGet/Home/issues/3116 # Avoid using the http cache as workaround for https://github.com/NuGet/Home/issues/3116
# use global cache in dev builds to avoid cost of downloading packages. # use global cache in dev builds to avoid cost of downloading packages.
# For directory normalization, see also: https://github.com/NuGet/Home/issues/7968 # For directory normalization, see also: https://github.com/NuGet/Home/issues/7968
@ -642,9 +647,26 @@ function MSBuild() {
} }
$toolsetBuildProject = InitializeToolset $toolsetBuildProject = InitializeToolset
$path = Split-Path -parent $toolsetBuildProject $basePath = Split-Path -parent $toolsetBuildProject
$path = Join-Path $path (Join-Path $buildTool.Framework 'Microsoft.DotNet.Arcade.Sdk.dll') $possiblePaths = @(
$args += "/logger:$path" # new scripts need to work with old packages, so we need to look for the old names/versions
(Join-Path $basePath (Join-Path $buildTool.Framework 'Microsoft.DotNet.ArcadeLogging.dll')),
(Join-Path $basePath (Join-Path $buildTool.Framework 'Microsoft.DotNet.Arcade.Sdk.dll')),
(Join-Path $basePath (Join-Path netcoreapp2.1 'Microsoft.DotNet.ArcadeLogging.dll')),
(Join-Path $basePath (Join-Path netcoreapp2.1 'Microsoft.DotNet.Arcade.Sdk.dll'))
)
$selectedPath = $null
foreach ($path in $possiblePaths) {
if (Test-Path $path -PathType Leaf) {
$selectedPath = $path
break
}
}
if (-not $selectedPath) {
Write-PipelineTelemetryError -Category 'Build' -Message 'Unable to find arcade sdk logger assembly.'
ExitWithExitCode 1
}
$args += "/logger:$selectedPath"
} }
MSBuild-Core @args MSBuild-Core @args
@ -680,7 +702,10 @@ function MSBuild-Core() {
} }
foreach ($arg in $args) { foreach ($arg in $args) {
if ($arg -ne $null -and $arg.Trim() -ne "") { if ($null -ne $arg -and $arg.Trim() -ne "") {
if ($arg.EndsWith('\')) {
$arg = $arg + "\"
}
$cmdArgs += " `"$arg`"" $cmdArgs += " `"$arg`""
} }
} }
@ -690,14 +715,23 @@ function MSBuild-Core() {
$exitCode = Exec-Process $buildTool.Path $cmdArgs $exitCode = Exec-Process $buildTool.Path $cmdArgs
if ($exitCode -ne 0) { if ($exitCode -ne 0) {
Write-PipelineTelemetryError -Category 'Build' -Message 'Build failed.' # We should not Write-PipelineTaskError here because that message shows up in the build summary
# The build already logged an error, that's the reason it failed. Producing an error here only adds noise.
Write-Host "Build failed with exit code $exitCode. Check errors above." -ForegroundColor Red
$buildLog = GetMSBuildBinaryLogCommandLineArgument $args $buildLog = GetMSBuildBinaryLogCommandLineArgument $args
if ($buildLog -ne $null) { if ($null -ne $buildLog) {
Write-Host "See log: $buildLog" -ForegroundColor DarkGray Write-Host "See log: $buildLog" -ForegroundColor DarkGray
} }
ExitWithExitCode $exitCode if ($ci) {
Write-PipelineSetResult -Result "Failed" -Message "msbuild execution failed."
# Exiting with an exit code causes the azure pipelines task to log yet another "noise" error
# The above Write-PipelineSetResult will cause the task to be marked as failure without adding yet another error
ExitWithExitCode 0
} else {
ExitWithExitCode $exitCode
}
} }
} }
@ -743,7 +777,7 @@ function Get-Darc($version) {
. $PSScriptRoot\pipeline-logging-functions.ps1 . $PSScriptRoot\pipeline-logging-functions.ps1
$RepoRoot = Resolve-Path (Join-Path $PSScriptRoot '..\..') $RepoRoot = Resolve-Path (Join-Path $PSScriptRoot '..\..\')
$EngRoot = Resolve-Path (Join-Path $PSScriptRoot '..') $EngRoot = Resolve-Path (Join-Path $PSScriptRoot '..')
$ArtifactsDir = Join-Path $RepoRoot 'artifacts' $ArtifactsDir = Join-Path $RepoRoot 'artifacts'
$ToolsetDir = Join-Path $ArtifactsDir 'toolset' $ToolsetDir = Join-Path $ArtifactsDir 'toolset'

104
eng/common/tools.sh

@ -68,10 +68,6 @@ fi
runtime_source_feed=${runtime_source_feed:-''} runtime_source_feed=${runtime_source_feed:-''}
runtime_source_feed_key=${runtime_source_feed_key:-''} runtime_source_feed_key=${runtime_source_feed_key:-''}
# Determines if dotnet-install.sh comes from the eng/common folder or the internet
# (default = public version)
use_default_dotnet_install=${use_default_dotnet_install:-false}
# Resolve any symlinks in the given path. # Resolve any symlinks in the given path.
function ResolvePath { function ResolvePath {
local path=$1 local path=$1
@ -93,16 +89,16 @@ function ResolvePath {
function ReadGlobalVersion { function ReadGlobalVersion {
local key=$1 local key=$1
local line=$(awk "/$key/ {print; exit}" "$global_json_file") if command -v jq &> /dev/null; then
local pattern="\"$key\" *: *\"(.*)\"" _ReadGlobalVersion="$(jq -r ".[] | select(has(\"$key\")) | .\"$key\"" "$global_json_file")"
elif [[ "$(cat "$global_json_file")" =~ \"$key\"[[:space:]\:]*\"([^\"]+) ]]; then
_ReadGlobalVersion=${BASH_REMATCH[1]}
fi
if [[ ! $line =~ $pattern ]]; then if [[ -z "$_ReadGlobalVersion" ]]; then
Write-PipelineTelemetryError -category 'Build' "Error: Cannot find \"$key\" in $global_json_file" Write-PipelineTelemetryError -category 'Build' "Error: Cannot find \"$key\" in $global_json_file"
ExitWithExitCode 1 ExitWithExitCode 1
fi fi
# return value
_ReadGlobalVersion=${BASH_REMATCH[1]}
} }
function InitializeDotNetCli { function InitializeDotNetCli {
@ -253,7 +249,7 @@ function with_retries {
return 0 return 0
fi fi
timeout=$((2**$retries-1)) timeout=$((3**$retries-1))
echo "Failed to execute '$@'. Waiting $timeout seconds before next attempt ($retries out of $maxRetries)." 1>&2 echo "Failed to execute '$@'. Waiting $timeout seconds before next attempt ($retries out of $maxRetries)." 1>&2
sleep $timeout sleep $timeout
done done
@ -271,30 +267,31 @@ function GetDotNetInstallScript {
if [[ ! -a "$install_script" ]]; then if [[ ! -a "$install_script" ]]; then
mkdir -p "$root" mkdir -p "$root"
if [[ "$use_default_dotnet_install" == true ]]; then echo "Downloading '$install_script_url'"
echo "Downloading '$install_script_url'"
# Use curl if available, otherwise use wget
# Use curl if available, otherwise use wget if command -v curl > /dev/null; then
if command -v curl > /dev/null; then # first, try directly, if this fails we will retry with verbose logging
with_retries curl "$install_script_url" -sSL --retry 10 --create-dirs -o "$install_script" || { curl "$install_script_url" -sSL --retry 10 --create-dirs -o "$install_script" || {
local exit_code=$? if command -v openssl &> /dev/null; then
Write-PipelineTelemetryError -category 'InitializeToolset' "Failed to acquire dotnet install script (exit code '$exit_code')." echo "Curl failed; dumping some information about dotnet.microsoft.com for later investigation"
ExitWithExitCode $exit_code echo | openssl s_client -showcerts -servername dotnet.microsoft.com -connect dotnet.microsoft.com:443
} fi
else echo "Will now retry the same URL with verbose logging."
with_retries wget -v -O "$install_script" "$install_script_url" || { with_retries curl "$install_script_url" -sSL --verbose --retry 10 --create-dirs -o "$install_script" || {
local exit_code=$? local exit_code=$?
Write-PipelineTelemetryError -category 'InitializeToolset' "Failed to acquire dotnet install script (exit code '$exit_code')." Write-PipelineTelemetryError -category 'InitializeToolset' "Failed to acquire dotnet install script (exit code '$exit_code')."
ExitWithExitCode $exit_code ExitWithExitCode $exit_code
} }
fi }
else else
# Use a special version of the script from eng/common that understands the existence of a "productVersion.txt" in a dotnet path. with_retries wget -v -O "$install_script" "$install_script_url" || {
# See https://github.com/dotnet/arcade/issues/6047 for details local exit_code=$?
cp $repo_root/eng/common/dotnet-install-scripts/dotnet-install.sh $install_script Write-PipelineTelemetryError -category 'InitializeToolset' "Failed to acquire dotnet install script (exit code '$exit_code')."
ExitWithExitCode $exit_code
}
fi fi
fi fi
# return value # return value
_GetDotNetInstallScript="$install_script" _GetDotNetInstallScript="$install_script"
} }
@ -309,7 +306,7 @@ function InitializeBuildTool {
# return values # return values
_InitializeBuildTool="$_InitializeDotNetCli/dotnet" _InitializeBuildTool="$_InitializeDotNetCli/dotnet"
_InitializeBuildToolCommand="msbuild" _InitializeBuildToolCommand="msbuild"
_InitializeBuildToolFramework="netcoreapp2.1" _InitializeBuildToolFramework="netcoreapp3.1"
} }
# Set RestoreNoCache as a workaround for https://github.com/NuGet/Home/issues/3116 # Set RestoreNoCache as a workaround for https://github.com/NuGet/Home/issues/3116
@ -416,8 +413,24 @@ function MSBuild {
fi fi
local toolset_dir="${_InitializeToolset%/*}" local toolset_dir="${_InitializeToolset%/*}"
local logger_path="$toolset_dir/$_InitializeBuildToolFramework/Microsoft.DotNet.Arcade.Sdk.dll" # new scripts need to work with old packages, so we need to look for the old names/versions
args=( "${args[@]}" "-logger:$logger_path" ) local selectedPath=
local possiblePaths=()
possiblePaths+=( "$toolset_dir/$_InitializeBuildToolFramework/Microsoft.DotNet.ArcadeLogging.dll" )
possiblePaths+=( "$toolset_dir/$_InitializeBuildToolFramework/Microsoft.DotNet.Arcade.Sdk.dll" )
possiblePaths+=( "$toolset_dir/netcoreapp2.1/Microsoft.DotNet.ArcadeLogging.dll" )
possiblePaths+=( "$toolset_dir/netcoreapp2.1/Microsoft.DotNet.Arcade.Sdk.dll" )
for path in "${possiblePaths[@]}"; do
if [[ -f $path ]]; then
selectedPath=$path
break
fi
done
if [[ -z "$selectedPath" ]]; then
Write-PipelineTelemetryError -category 'Build' "Unable to find arcade sdk logger assembly."
ExitWithExitCode 1
fi
args+=( "-logger:$selectedPath" )
fi fi
MSBuild-Core ${args[@]} MSBuild-Core ${args[@]}
@ -448,8 +461,17 @@ function MSBuild-Core {
"$_InitializeBuildTool" "$@" || { "$_InitializeBuildTool" "$@" || {
local exit_code=$? local exit_code=$?
Write-PipelineTaskError "Build failed (exit code '$exit_code')." # We should not Write-PipelineTaskError here because that message shows up in the build summary
ExitWithExitCode $exit_code # The build already logged an error, that's the reason it failed. Producing an error here only adds noise.
echo "Build failed with exit code $exit_code. Check errors above."
if [[ "$ci" == "true" ]]; then
Write-PipelineSetResult -result "Failed" -message "msbuild execution failed."
# Exiting with an exit code causes the azure pipelines task to log yet another "noise" error
# The above Write-PipelineSetResult will cause the task to be marked as failure without adding yet another error
ExitWithExitCode 0
else
ExitWithExitCode $exit_code
fi
} }
} }
@ -463,23 +485,27 @@ _script_dir=`dirname "$_ResolvePath"`
eng_root=`cd -P "$_script_dir/.." && pwd` eng_root=`cd -P "$_script_dir/.." && pwd`
repo_root=`cd -P "$_script_dir/../.." && pwd` repo_root=`cd -P "$_script_dir/../.." && pwd`
artifacts_dir="$repo_root/artifacts" repo_root="${repo_root}/"
artifacts_dir="${repo_root}artifacts"
toolset_dir="$artifacts_dir/toolset" toolset_dir="$artifacts_dir/toolset"
tools_dir="$repo_root/.tools" tools_dir="${repo_root}.tools"
log_dir="$artifacts_dir/log/$configuration" log_dir="$artifacts_dir/log/$configuration"
temp_dir="$artifacts_dir/tmp/$configuration" temp_dir="$artifacts_dir/tmp/$configuration"
global_json_file="$repo_root/global.json" global_json_file="${repo_root}global.json"
# determine if global.json contains a "runtimes" entry # determine if global.json contains a "runtimes" entry
global_json_has_runtimes=false global_json_has_runtimes=false
dotnetlocal_key=$(awk "/runtimes/ {print; exit}" "$global_json_file") || true if command -v jq &> /dev/null; then
if [[ -n "$dotnetlocal_key" ]]; then if jq -er '. | select(has("runtimes"))' "$global_json_file" &> /dev/null; then
global_json_has_runtimes=true
fi
elif [[ "$(cat "$global_json_file")" =~ \"runtimes\"[[:space:]\:]*\{ ]]; then
global_json_has_runtimes=true global_json_has_runtimes=true
fi fi
# HOME may not be defined in some scenarios, but it is required by NuGet # HOME may not be defined in some scenarios, but it is required by NuGet
if [[ -z $HOME ]]; then if [[ -z $HOME ]]; then
export HOME="$repo_root/artifacts/.home/" export HOME="${repo_root}artifacts/.home/"
mkdir -p "$HOME" mkdir -p "$HOME"
fi fi

9
global.json

@ -1,17 +1,18 @@
{ {
"tools": { "tools": {
"dotnet": "5.0.301", "dotnet": "6.0.100-rc.1.21458.32",
"runtimes": { "runtimes": {
"aspnetcore": [ "aspnetcore": [
"2.1.28", "2.1.28",
"3.1.16" "3.1.16",
"5.0.7"
] ]
} }
}, },
"msbuild-sdks": { "msbuild-sdks": {
"Microsoft.DotNet.Arcade.Sdk": "5.0.0-beta.20510.1", "Microsoft.DotNet.Arcade.Sdk": "6.0.0-beta.21316.1",
"Microsoft.DotNet.Helix.Sdk": "5.0.0-beta.20510.1" "Microsoft.DotNet.Helix.Sdk": "6.0.0-beta.21316.1"
} }
} }

Loading…
Cancel
Save