Browse Source

Update Arcade and bump the .NET SDK to 6.0.100-rc.1.21458.32

pull/1335/head
Kévin Chalet 5 years ago
parent
commit
aaba2d0008
  1. 1
      Directory.Build.props
  2. 8
      eng/Version.Details.xml
  3. 5
      eng/common/SetupNugetSources.ps1
  4. 4
      eng/common/SetupNugetSources.sh
  5. 4
      eng/common/build.ps1
  6. 11
      eng/common/build.sh
  7. 2
      eng/common/cross/arm64/tizen-fetch.sh
  8. 43
      eng/common/cross/armel/armel.jessie.patch
  9. 2
      eng/common/cross/armel/tizen-fetch.sh
  10. 2
      eng/common/cross/build-android-rootfs.sh
  11. 56
      eng/common/cross/build-rootfs.sh
  12. 11
      eng/common/cross/s390x/sources.list.bionic
  13. 11
      eng/common/cross/toolchain.cmake
  14. 2
      eng/common/darc-init.sh
  15. 13
      eng/common/dotnet-install.sh
  16. 117
      eng/common/generate-locproject.ps1
  17. 113
      eng/common/init-tools-native.sh
  18. 8
      eng/common/internal-feed-operations.ps1
  19. 6
      eng/common/internal-feed-operations.sh
  20. 1
      eng/common/msbuild.ps1
  21. 2
      eng/common/msbuild.sh
  22. 2
      eng/common/native/CommonLibrary.psm1
  23. 6
      eng/common/native/install-cmake-test.sh
  24. 6
      eng/common/native/install-cmake.sh
  25. 2
      eng/common/native/install-tool.ps1
  26. 104
      eng/common/pipeline-logging-functions.ps1
  27. 34
      eng/common/pipeline-logging-functions.sh
  28. 6
      eng/common/post-build/post-build-utils.ps1
  29. 14
      eng/common/post-build/publish-using-darc.ps1
  30. 95
      eng/common/post-build/sourcelink-validation.ps1
  31. 182
      eng/common/post-build/symbols-validation.ps1
  32. 8
      eng/common/sdk-task.ps1
  33. 6
      eng/common/sdl/execute-all-sdl-tools.ps1
  34. 16
      eng/common/sdl/init-sdl.ps1
  35. 2
      eng/common/sdl/packages.config
  36. 2
      eng/common/sdl/run-sdl.ps1
  37. 4
      eng/common/templates/job/execute-sdl.yml
  38. 18
      eng/common/templates/job/job.yml
  39. 93
      eng/common/templates/job/onelocbuild.yml
  40. 8
      eng/common/templates/job/publish-build-assets.yml
  41. 11
      eng/common/templates/job/source-build.yml
  42. 62
      eng/common/templates/job/source-index-stage1.yml
  43. 29
      eng/common/templates/jobs/jobs.yml
  44. 12
      eng/common/templates/jobs/source-build.yml
  45. 1
      eng/common/templates/phases/publish-build-assets.yml
  46. 8
      eng/common/templates/post-build/channels/generic-internal-channel.yml
  47. 8
      eng/common/templates/post-build/channels/generic-public-channel.yml
  48. 391
      eng/common/templates/post-build/post-build.yml
  49. 1
      eng/common/templates/post-build/setup-maestro-vars.yml
  50. 4
      eng/common/templates/steps/send-to-helix.yml
  51. 7
      eng/common/templates/steps/source-build.yml
  52. 164
      eng/common/tools.ps1
  53. 104
      eng/common/tools.sh
  54. 9
      global.json

1
Directory.Build.props

@ -42,6 +42,7 @@
<PropertyGroup>
<RepoRelativeProjectDir>$([MSBuild]::MakeRelative($(RepoRoot), $(MSBuildProjectDirectory)))</RepoRelativeProjectDir>
<WarnOnPackingNonPackableProject>false</WarnOnPackingNonPackableProject>
<CheckEolTargetFramework>false</CheckEolTargetFramework>
</PropertyGroup>
<!--

8
eng/Version.Details.xml

@ -5,14 +5,14 @@
</ProductDependencies>
<ToolsetDependencies>
<Dependency Name="Microsoft.DotNet.Arcade.Sdk" Version="5.0.0-beta.20510.1">
<Dependency Name="Microsoft.DotNet.Arcade.Sdk" Version="6.0.0-beta.21316.1">
<Uri>https://github.com/dotnet/arcade</Uri>
<Sha>6813f5aa511a7a4498fa217a54219b5704a01f83</Sha>
<Sha>bdcc258d0d4a547be51703795a140fa6b1f68c57</Sha>
</Dependency>
<Dependency Name="Microsoft.DotNet.Helix.Sdk" Version="5.0.0-beta.20510.1">
<Dependency Name="Microsoft.DotNet.Helix.Sdk" Version="6.0.0-beta.21316.1">
<Uri>https://github.com/dotnet/arcade</Uri>
<Sha>6813f5aa511a7a4498fa217a54219b5704a01f83</Sha>
<Sha>bdcc258d0d4a547be51703795a140fa6b1f68c57</Sha>
</Dependency>
</ToolsetDependencies>

5
eng/common/SetupNugetSources.ps1

@ -99,8 +99,9 @@ function InsertMaestroPrivateFeedCredentials($Sources, $Creds, $Username, $Passw
function EnablePrivatePackageSources($DisabledPackageSources) {
$maestroPrivateSources = $DisabledPackageSources.SelectNodes("add[contains(@key,'darc-int')]")
ForEach ($DisabledPackageSource in $maestroPrivateSources) {
Write-Host "`tEnsuring private source '$($DisabledPackageSource.key)' is enabled"
$DisabledPackageSource.SetAttribute("value", "false")
Write-Host "`tEnsuring private source '$($DisabledPackageSource.key)' is enabled by deleting it from disabledPackageSource"
# Due to https://github.com/NuGet/Home/issues/10291, we must actually remove the disabled entries
$DisabledPackageSources.RemoveChild($DisabledPackageSource)
}
}

4
eng/common/SetupNugetSources.sh

@ -158,8 +158,8 @@ if [ "$?" == "0" ]; then
for DisabledSourceName in ${DisabledDarcIntSources[@]} ; do
if [[ $DisabledSourceName == darc-int* ]]
then
OldDisableValue="add key=\"$DisabledSourceName\" value=\"true\""
NewDisableValue="add key=\"$DisabledSourceName\" value=\"false\""
OldDisableValue="<add key=\"$DisabledSourceName\" value=\"true\" />"
NewDisableValue="<!-- Reenabled for build : $DisabledSourceName -->"
sed -i.bak "s|$OldDisableValue|$NewDisableValue|" $ConfigFile
echo "Neutralized disablePackageSources entry for '$DisabledSourceName'"
fi

4
eng/common/build.ps1

@ -7,7 +7,6 @@ Param(
[string] $msbuildEngine = $null,
[bool] $warnAsError = $true,
[bool] $nodeReuse = $true,
[bool] $useDefaultDotnetInstall = $false,
[switch][Alias('r')]$restore,
[switch] $deployDeps,
[switch][Alias('b')]$build,
@ -26,6 +25,7 @@ Param(
[switch] $prepareMachine,
[string] $runtimeSourceFeed = '',
[string] $runtimeSourceFeedKey = '',
[switch] $excludePrereleaseVS,
[switch] $help,
[Parameter(ValueFromRemainingArguments=$true)][String[]]$properties
)
@ -66,7 +66,7 @@ function Print-Usage() {
Write-Host " -prepareMachine Prepare machine for CI run, clean up processes after build"
Write-Host " -warnAsError <value> Sets warnaserror msbuild parameter ('true' or 'false')"
Write-Host " -msbuildEngine <value> Msbuild engine to use to run build ('dotnet', 'vs', or unspecified)."
Write-Host " -useDefaultDotnetInstall <value> Use dotnet-install.* scripts from public location as opposed to from eng common folder"
Write-Host " -excludePrereleaseVS Set to exclude build engines in prerelease versions of Visual Studio"
Write-Host ""
Write-Host "Command line arguments not listed above are passed thru to msbuild."

11
eng/common/build.sh

@ -36,8 +36,6 @@ usage()
echo " --prepareMachine Prepare machine for CI run, clean up processes after build"
echo " --nodeReuse <value> Sets nodereuse msbuild parameter ('true' or 'false')"
echo " --warnAsError <value> Sets warnaserror msbuild parameter ('true' or 'false')"
echo " --useDefaultDotnetInstall <value> Use dotnet-install.* scripts from public location as opposed to from eng common folder"
echo ""
echo "Command line arguments not listed above are passed thru to msbuild."
echo "Arguments can also be passed in with a single hyphen."
@ -80,11 +78,10 @@ prepare_machine=false
verbosity='minimal'
runtime_source_feed=''
runtime_source_feed_key=''
use_default_dotnet_install=false
properties=''
while [[ $# > 0 ]]; do
opt="$(echo "${1/#--/-}" | awk '{print tolower($0)}')"
opt="$(echo "${1/#--/-}" | tr "[:upper:]" "[:lower:]")"
case "$opt" in
-help|-h)
usage
@ -159,14 +156,10 @@ while [[ $# > 0 ]]; do
runtime_source_feed=$2
shift
;;
-runtimesourcefeedkey)
-runtimesourcefeedkey)
runtime_source_feed_key=$2
shift
;;
-usedefaultdotnetinstall)
use_default_dotnet_install=$2
shift
;;
*)
properties="$properties $1"
;;

2
eng/common/cross/arm64/tizen-fetch.sh

@ -157,7 +157,7 @@ fetch_tizen_pkgs()
Inform "Initialize arm base"
fetch_tizen_pkgs_init standard base
Inform "fetch common packages"
fetch_tizen_pkgs aarch64 gcc glibc glibc-devel libicu libicu-devel libatomic linux-glibc-devel
fetch_tizen_pkgs aarch64 gcc glibc glibc-devel libicu libicu-devel libatomic linux-glibc-devel keyutils keyutils-devel libkeyutils
Inform "fetch coreclr packages"
fetch_tizen_pkgs aarch64 lldb lldb-devel libgcc libstdc++ libstdc++-devel libunwind libunwind-devel lttng-ust-devel lttng-ust userspace-rcu-devel userspace-rcu
Inform "fetch corefx packages"

43
eng/common/cross/armel/armel.jessie.patch

@ -0,0 +1,43 @@
diff -u -r a/usr/include/urcu/uatomic/generic.h b/usr/include/urcu/uatomic/generic.h
--- a/usr/include/urcu/uatomic/generic.h 2014-10-22 15:00:58.000000000 -0700
+++ b/usr/include/urcu/uatomic/generic.h 2020-10-30 21:38:28.550000000 -0700
@@ -69,10 +69,10 @@
#endif
#ifdef UATOMIC_HAS_ATOMIC_SHORT
case 2:
- return __sync_val_compare_and_swap_2(addr, old, _new);
+ return __sync_val_compare_and_swap_2((uint16_t*) addr, old, _new);
#endif
case 4:
- return __sync_val_compare_and_swap_4(addr, old, _new);
+ return __sync_val_compare_and_swap_4((uint32_t*) addr, old, _new);
#if (CAA_BITS_PER_LONG == 64)
case 8:
return __sync_val_compare_and_swap_8(addr, old, _new);
@@ -109,7 +109,7 @@
return;
#endif
case 4:
- __sync_and_and_fetch_4(addr, val);
+ __sync_and_and_fetch_4((uint32_t*) addr, val);
return;
#if (CAA_BITS_PER_LONG == 64)
case 8:
@@ -148,7 +148,7 @@
return;
#endif
case 4:
- __sync_or_and_fetch_4(addr, val);
+ __sync_or_and_fetch_4((uint32_t*) addr, val);
return;
#if (CAA_BITS_PER_LONG == 64)
case 8:
@@ -187,7 +187,7 @@
return __sync_add_and_fetch_2(addr, val);
#endif
case 4:
- return __sync_add_and_fetch_4(addr, val);
+ return __sync_add_and_fetch_4((uint32_t*) addr, val);
#if (CAA_BITS_PER_LONG == 64)
case 8:
return __sync_add_and_fetch_8(addr, val);

2
eng/common/cross/armel/tizen-fetch.sh

@ -157,7 +157,7 @@ fetch_tizen_pkgs()
Inform "Initialize arm base"
fetch_tizen_pkgs_init standard base
Inform "fetch common packages"
fetch_tizen_pkgs armv7l gcc glibc glibc-devel libicu libicu-devel libatomic linux-glibc-devel
fetch_tizen_pkgs armv7l gcc gcc-devel-static glibc glibc-devel libicu libicu-devel libatomic linux-glibc-devel keyutils keyutils-devel libkeyutils
Inform "fetch coreclr packages"
fetch_tizen_pkgs armv7l lldb lldb-devel libgcc libstdc++ libstdc++-devel libunwind libunwind-devel lttng-ust-devel lttng-ust userspace-rcu-devel userspace-rcu
Inform "fetch corefx packages"

2
eng/common/cross/build-android-rootfs.sh

@ -27,7 +27,7 @@ __AndroidToolchain=aarch64-linux-android
for i in "$@"
do
lowerI="$(echo $i | awk '{print tolower($0)}')"
lowerI="$(echo $i | tr "[:upper:]" "[:lower:]")"
case $lowerI in
-?|-h|--help)
usage

56
eng/common/cross/build-rootfs.sh

@ -6,7 +6,7 @@ usage()
{
echo "Usage: $0 [BuildArch] [CodeName] [lldbx.y] [--skipunmount] --rootfsdir <directory>]"
echo "BuildArch can be: arm(default), armel, arm64, x86"
echo "CodeName - optional, Code name for Linux, can be: trusty, xenial(default), zesty, bionic, alpine. If BuildArch is armel, LinuxCodeName is jessie(default) or tizen."
echo "CodeName - optional, Code name for Linux, can be: trusty, xenial(default), zesty, bionic, alpine, alpine3.9 or alpine3.13. If BuildArch is armel, LinuxCodeName is jessie(default) or tizen."
echo " for FreeBSD can be: freebsd11 or freebsd12."
echo " for illumos can be: illumos."
echo "lldbx.y - optional, LLDB version, can be: lldb3.9(default), lldb4.0, lldb5.0, lldb6.0 no-lldb. Ignored for alpine and FReeBSD"
@ -74,6 +74,10 @@ __IllumosPackages+=" mit-krb5-1.16.2nb4"
__IllumosPackages+=" openssl-1.1.1e"
__IllumosPackages+=" zlib-1.2.11"
# ML.NET dependencies
__UbuntuPackages+=" libomp5"
__UbuntuPackages+=" libomp-dev"
__UseMirror=0
__UnprocessedBuildArgs=
@ -82,7 +86,7 @@ while :; do
break
fi
lowerI="$(echo $1 | awk '{print tolower($0)}')"
lowerI="$(echo $1 | tr "[:upper:]" "[:lower:]")"
case $lowerI in
-?|-h|--help)
usage
@ -106,6 +110,13 @@ while :; do
__UbuntuRepo="http://ftp.debian.org/debian/"
__CodeName=jessie
;;
s390x)
__BuildArch=s390x
__UbuntuArch=s390x
__UbuntuRepo="http://ports.ubuntu.com/ubuntu-ports/"
__UbuntuPackages=$(echo ${__UbuntuPackages} | sed 's/ libunwind8-dev//')
unset __LLDB_Package
;;
x86)
__BuildArch=x86
__UbuntuArch=i386
@ -176,9 +187,20 @@ while :; do
__UbuntuRepo=
__Tizen=tizen
;;
alpine)
alpine|alpine3.9)
__CodeName=alpine
__UbuntuRepo=
__AlpineVersion=3.9
;;
alpine3.13)
__CodeName=alpine
__UbuntuRepo=
__AlpineVersion=3.13
# Alpine 3.13 has all the packages we need in the 3.13 repository
__AlpinePackages+=$__AlpinePackagesEdgeCommunity
__AlpinePackagesEdgeCommunity=
__AlpinePackages+=$__AlpinePackagesEdgeMain
__AlpinePackagesEdgeMain=
;;
freebsd11)
__FreeBSDBase="11.3-RELEASE"
@ -236,7 +258,6 @@ __RootfsDir="$( cd "$__RootfsDir" && pwd )"
if [[ "$__CodeName" == "alpine" ]]; then
__ApkToolsVersion=2.9.1
__AlpineVersion=3.9
__ApkToolsDir=$(mktemp -d)
wget https://github.com/alpinelinux/apk-tools/releases/download/v$__ApkToolsVersion/apk-tools-$__ApkToolsVersion-x86_64-linux.tar.gz -P $__ApkToolsDir
tar -xf $__ApkToolsDir/apk-tools-$__ApkToolsVersion-x86_64-linux.tar.gz -C $__ApkToolsDir
@ -249,15 +270,19 @@ if [[ "$__CodeName" == "alpine" ]]; then
-U --allow-untrusted --root $__RootfsDir --arch $__AlpineArch --initdb \
add $__AlpinePackages
$__ApkToolsDir/apk-tools-$__ApkToolsVersion/apk \
-X http://dl-cdn.alpinelinux.org/alpine/edge/main \
-U --allow-untrusted --root $__RootfsDir --arch $__AlpineArch --initdb \
add $__AlpinePackagesEdgeMain
if [[ -n "$__AlpinePackagesEdgeMain" ]]; then
$__ApkToolsDir/apk-tools-$__ApkToolsVersion/apk \
-X http://dl-cdn.alpinelinux.org/alpine/edge/main \
-U --allow-untrusted --root $__RootfsDir --arch $__AlpineArch --initdb \
add $__AlpinePackagesEdgeMain
fi
$__ApkToolsDir/apk-tools-$__ApkToolsVersion/apk \
-X http://dl-cdn.alpinelinux.org/alpine/edge/community \
-U --allow-untrusted --root $__RootfsDir --arch $__AlpineArch --initdb \
add $__AlpinePackagesEdgeCommunity
if [[ -n "$__AlpinePackagesEdgeCommunity" ]]; then
$__ApkToolsDir/apk-tools-$__ApkToolsVersion/apk \
-X http://dl-cdn.alpinelinux.org/alpine/edge/community \
-U --allow-untrusted --root $__RootfsDir --arch $__AlpineArch --initdb \
add $__AlpinePackagesEdgeCommunity
fi
rm -r $__ApkToolsDir
elif [[ "$__CodeName" == "freebsd" ]]; then
@ -329,6 +354,7 @@ elif [[ -n $__CodeName ]]; then
chroot $__RootfsDir apt-get -f -y install
chroot $__RootfsDir apt-get -y install $__UbuntuPackages
chroot $__RootfsDir symlinks -cr /usr
chroot $__RootfsDir apt-get clean
if [ $__SkipUnmount == 0 ]; then
umount $__RootfsDir/* || true
@ -340,6 +366,12 @@ elif [[ -n $__CodeName ]]; then
patch -p1 < $__CrossDir/$__BuildArch/trusty-lttng-2.4.patch
popd
fi
if [[ "$__BuildArch" == "armel" && "$__CodeName" == "jessie" ]]; then
pushd $__RootfsDir
patch -p1 < $__CrossDir/$__BuildArch/armel.jessie.patch
popd
fi
elif [[ "$__Tizen" == "tizen" ]]; then
ROOTFS_DIR=$__RootfsDir $__CrossDir/$__BuildArch/tizen-build-rootfs.sh
else

11
eng/common/cross/s390x/sources.list.bionic

@ -0,0 +1,11 @@
deb http://ports.ubuntu.com/ubuntu-ports/ bionic main restricted universe
deb-src http://ports.ubuntu.com/ubuntu-ports/ bionic main restricted universe
deb http://ports.ubuntu.com/ubuntu-ports/ bionic-updates main restricted universe
deb-src http://ports.ubuntu.com/ubuntu-ports/ bionic-updates main restricted universe
deb http://ports.ubuntu.com/ubuntu-ports/ bionic-backports main restricted
deb-src http://ports.ubuntu.com/ubuntu-ports/ bionic-backports main restricted
deb http://ports.ubuntu.com/ubuntu-ports/ bionic-security main restricted universe multiverse
deb-src http://ports.ubuntu.com/ubuntu-ports/ bionic-security main restricted universe multiverse

11
eng/common/cross/toolchain.cmake

@ -36,6 +36,9 @@ elseif(TARGET_ARCH_NAME STREQUAL "arm64")
if("$ENV{__DistroRid}" MATCHES "tizen.*")
set(TIZEN_TOOLCHAIN "aarch64-tizen-linux-gnu/9.2.0")
endif()
elseif(TARGET_ARCH_NAME STREQUAL "s390x")
set(CMAKE_SYSTEM_PROCESSOR s390x)
set(TOOLCHAIN "s390x-linux-gnu")
elseif(TARGET_ARCH_NAME STREQUAL "x86")
set(CMAKE_SYSTEM_PROCESSOR i686)
set(TOOLCHAIN "i686-linux-gnu")
@ -46,7 +49,7 @@ elseif (ILLUMOS)
set(CMAKE_SYSTEM_PROCESSOR "x86_64")
set(TOOLCHAIN "x86_64-illumos")
else()
message(FATAL_ERROR "Arch is ${TARGET_ARCH_NAME}. Only armel, arm, arm64 and x86 are supported!")
message(FATAL_ERROR "Arch is ${TARGET_ARCH_NAME}. Only armel, arm, arm64, s390x and x86 are supported!")
endif()
if(DEFINED ENV{TOOLCHAIN})
@ -139,6 +142,10 @@ function(add_toolchain_linker_flag Flag)
set("CMAKE_SHARED_LINKER_FLAGS${CONFIG_SUFFIX}" "${CMAKE_SHARED_LINKER_FLAGS${CONFIG_SUFFIX}} ${Flag}" PARENT_SCOPE)
endfunction()
if(CMAKE_SYSTEM_NAME STREQUAL "Linux")
add_toolchain_linker_flag("-Wl,--rpath-link=${CROSS_ROOTFS}/lib/${TOOLCHAIN}")
add_toolchain_linker_flag("-Wl,--rpath-link=${CROSS_ROOTFS}/usr/lib/${TOOLCHAIN}")
endif()
if(TARGET_ARCH_NAME STREQUAL "armel")
if(DEFINED TIZEN_TOOLCHAIN) # For Tizen only
@ -167,7 +174,7 @@ endif()
# Specify compile options
if((TARGET_ARCH_NAME MATCHES "^(arm|armel|arm64)$" AND NOT "$ENV{__DistroRid}" MATCHES "android.*") OR ILLUMOS)
if((TARGET_ARCH_NAME MATCHES "^(arm|armel|arm64|s390x)$" AND NOT "$ENV{__DistroRid}" MATCHES "android.*") OR ILLUMOS)
set(CMAKE_C_COMPILER_TARGET ${TOOLCHAIN})
set(CMAKE_CXX_COMPILER_TARGET ${TOOLCHAIN})
set(CMAKE_ASM_COMPILER_TARGET ${TOOLCHAIN})

2
eng/common/darc-init.sh

@ -6,7 +6,7 @@ versionEndpoint='https://maestro-prod.westus2.cloudapp.azure.com/api/assets/darc
verbosity='minimal'
while [[ $# > 0 ]]; do
opt="$(echo "$1" | awk '{print tolower($0)}')"
opt="$(echo "$1" | tr "[:upper:]" "[:lower:]")"
case "$opt" in
--darcversion)
darcVersion=$2

13
eng/common/dotnet-install.sh

@ -19,7 +19,7 @@ runtime='dotnet'
runtimeSourceFeed=''
runtimeSourceFeedKey=''
while [[ $# > 0 ]]; do
opt="$(echo "$1" | awk '{print tolower($0)}')"
opt="$(echo "$1" | tr "[:upper:]" "[:lower:]")"
case "$opt" in
-version|-v)
shift
@ -49,13 +49,8 @@ while [[ $# > 0 ]]; do
shift
done
# Use uname to determine what the CPU is.
cpuname=$(uname -p)
# Some Linux platforms report unknown for platform, but the arch for machine.
if [[ "$cpuname" == "unknown" ]]; then
cpuname=$(uname -m)
fi
# Use uname to determine what the CPU is, see https://en.wikipedia.org/wiki/Uname#Examples
cpuname=$(uname -m)
case $cpuname in
aarch64)
buildarch=arm64
@ -75,7 +70,7 @@ case $cpuname in
;;
esac
dotnetRoot="$repo_root/.dotnet"
dotnetRoot="${repo_root}.dotnet"
if [[ $architecture != "" ]] && [[ $architecture != $buildarch ]]; then
dotnetRoot="$dotnetRoot/$architecture"
fi

117
eng/common/generate-locproject.ps1

@ -0,0 +1,117 @@
Param(
[Parameter(Mandatory=$true)][string] $SourcesDirectory, # Directory where source files live; if using a Localize directory it should live in here
[string] $LanguageSet = 'VS_Main_Languages', # Language set to be used in the LocProject.json
[switch] $UseCheckedInLocProjectJson, # When set, generates a LocProject.json and compares it to one that already exists in the repo; otherwise just generates one
[switch] $CreateNeutralXlfs # Creates neutral xlf files. Only set to false when running locally
)
# Generates LocProject.json files for the OneLocBuild task. OneLocBuildTask is described here:
# https://ceapex.visualstudio.com/CEINTL/_wiki/wikis/CEINTL.wiki/107/Localization-with-OneLocBuild-Task
Set-StrictMode -Version 2.0
$ErrorActionPreference = "Stop"
. $PSScriptRoot\tools.ps1
Import-Module -Name (Join-Path $PSScriptRoot 'native\CommonLibrary.psm1')
$exclusionsFilePath = "$SourcesDirectory\eng\Localize\LocExclusions.json"
$exclusions = @{ Exclusions = @() }
if (Test-Path -Path $exclusionsFilePath)
{
$exclusions = Get-Content "$exclusionsFilePath" | ConvertFrom-Json
}
Push-Location "$SourcesDirectory" # push location for Resolve-Path -Relative to work
# Template files
$jsonFiles = @()
$jsonTemplateFiles = Get-ChildItem -Recurse -Path "$SourcesDirectory" | Where-Object { $_.FullName -Match "\.template\.config\\localize\\.+\.en\.json" } # .NET templating pattern
$jsonTemplateFiles | ForEach-Object {
$null = $_.Name -Match "(.+)\.[\w-]+\.json" # matches '[filename].[langcode].json
$destinationFile = "$($_.Directory.FullName)\$($Matches.1).json"
$jsonFiles += Copy-Item "$($_.FullName)" -Destination $destinationFile -PassThru
}
$jsonWinformsTemplateFiles = Get-ChildItem -Recurse -Path "$SourcesDirectory" | Where-Object { $_.FullName -Match "en\\strings\.json" } # current winforms pattern
$xlfFiles = @()
$allXlfFiles = Get-ChildItem -Recurse -Path "$SourcesDirectory\*\*.xlf"
$langXlfFiles = @()
if ($allXlfFiles) {
$null = $allXlfFiles[0].FullName -Match "\.([\w-]+)\.xlf" # matches '[langcode].xlf'
$firstLangCode = $Matches.1
$langXlfFiles = Get-ChildItem -Recurse -Path "$SourcesDirectory\*\*.$firstLangCode.xlf"
}
$langXlfFiles | ForEach-Object {
$null = $_.Name -Match "(.+)\.[\w-]+\.xlf" # matches '[filename].[langcode].xlf
$destinationFile = "$($_.Directory.FullName)\$($Matches.1).xlf"
$xlfFiles += Copy-Item "$($_.FullName)" -Destination $destinationFile -PassThru
}
$locFiles = $jsonFiles + $jsonWinformsTemplateFiles + $xlfFiles
$locJson = @{
Projects = @(
@{
LanguageSet = $LanguageSet
LocItems = @(
$locFiles | ForEach-Object {
$outputPath = "$(($_.DirectoryName | Resolve-Path -Relative) + "\")"
$continue = $true
foreach ($exclusion in $exclusions.Exclusions) {
if ($outputPath.Contains($exclusion))
{
$continue = $false
}
}
$sourceFile = ($_.FullName | Resolve-Path -Relative)
if (!$CreateNeutralXlfs -and $_.Extension -eq '.xlf') {
Remove-Item -Path $sourceFile
}
if ($continue)
{
if ($_.Directory.Name -eq 'en' -and $_.Extension -eq '.json') {
return @{
SourceFile = $sourceFile
CopyOption = "LangIDOnPath"
OutputPath = "$($_.Directory.Parent.FullName | Resolve-Path -Relative)\"
}
}
else {
return @{
SourceFile = $sourceFile
CopyOption = "LangIDOnName"
OutputPath = $outputPath
}
}
}
}
)
}
)
}
$json = ConvertTo-Json $locJson -Depth 5
Write-Host "LocProject.json generated:`n`n$json`n`n"
Pop-Location
if (!$UseCheckedInLocProjectJson) {
New-Item "$SourcesDirectory\eng\Localize\LocProject.json" -Force # Need this to make sure the Localize directory is created
Set-Content "$SourcesDirectory\eng\Localize\LocProject.json" $json
}
else {
New-Item "$SourcesDirectory\eng\Localize\LocProject-generated.json" -Force # Need this to make sure the Localize directory is created
Set-Content "$SourcesDirectory\eng\Localize\LocProject-generated.json" $json
if ((Get-FileHash "$SourcesDirectory\eng\Localize\LocProject-generated.json").Hash -ne (Get-FileHash "$SourcesDirectory\eng\Localize\LocProject.json").Hash) {
Write-PipelineTelemetryError -Category "OneLocBuild" -Message "Existing LocProject.json differs from generated LocProject.json. Download LocProject-generated.json and compare them."
exit 1
}
else {
Write-Host "Generated LocProject.json and current LocProject.json are identical."
}
}

113
eng/common/init-tools-native.sh

@ -16,7 +16,7 @@ declare -A native_assets
. $scriptroot/native/common-library.sh
while (($# > 0)); do
lowerI="$(echo $1 | awk '{print tolower($0)}')"
lowerI="$(echo $1 | tr "[:upper:]" "[:lower:]")"
case $lowerI in
--baseuri)
base_uri=$2
@ -76,24 +76,89 @@ while (($# > 0)); do
done
function ReadGlobalJsonNativeTools {
# Get the native-tools section from the global.json.
local native_tools_section=$(cat $global_json_file | awk '/"native-tools"/,/}/')
# Only extract the contents of the object.
local native_tools_list=$(echo $native_tools_section | awk -F"[{}]" '{print $2}')
native_tools_list=${native_tools_list//[\" ]/}
native_tools_list=$( echo "$native_tools_list" | sed 's/\s//g' | sed 's/,/\n/g' )
local old_IFS=$IFS
while read -r line; do
# Lines are of the form: 'tool:version'
IFS=:
while read -r key value; do
native_assets[$key]=$value
done <<< "$line"
done <<< "$native_tools_list"
IFS=$old_IFS
return 0;
# happy path: we have a proper JSON parsing tool `jq(1)` in PATH!
if command -v jq &> /dev/null; then
# jq: read each key/value pair under "native-tools" entry and emit:
# KEY="<entry-key>" VALUE="<entry-value>"
# followed by a null byte.
#
# bash: read line with null byte delimeter and push to array (for later `eval`uation).
while IFS= read -rd '' line; do
native_assets+=("$line")
done < <(jq -r '. |
select(has("native-tools")) |
."native-tools" |
keys[] as $k |
@sh "KEY=\($k) VALUE=\(.[$k])\u0000"' "$global_json_file")
return
fi
# Warning: falling back to manually parsing JSON, which is not recommended.
# Following routine matches the output and escaping logic of jq(1)'s @sh formatter used above.
# It has been tested with several weird strings with escaped characters in entries (key and value)
# and results were compared with the output of jq(1) in binary representation using xxd(1);
# just before the assignment to 'native_assets' array (above and below).
# try to capture the section under "native-tools".
if [[ ! "$(cat "$global_json_file")" =~ \"native-tools\"[[:space:]\:\{]*([^\}]+) ]]; then
return
fi
section="${BASH_REMATCH[1]}"
parseStarted=0
possibleEnd=0
escaping=0
escaped=0
isKey=1
for (( i=0; i<${#section}; i++ )); do
char="${section:$i:1}"
if ! ((parseStarted)) && [[ "$char" =~ [[:space:],:] ]]; then continue; fi
if ! ((escaping)) && [[ "$char" == "\\" ]]; then
escaping=1
elif ((escaping)) && ! ((escaped)); then
escaped=1
fi
if ! ((parseStarted)) && [[ "$char" == "\"" ]]; then
parseStarted=1
possibleEnd=0
elif [[ "$char" == "'" ]]; then
token="$token'\\\''"
possibleEnd=0
elif ((escaping)) || [[ "$char" != "\"" ]]; then
token="$token$char"
possibleEnd=1
fi
if ((possibleEnd)) && ! ((escaping)) && [[ "$char" == "\"" ]]; then
# Use printf to unescape token to match jq(1)'s @sh formatting rules.
# do not use 'token="$(printf "$token")"' syntax, as $() eats the trailing linefeed.
printf -v token "'$token'"
if ((isKey)); then
KEY="$token"
isKey=0
else
line="KEY=$KEY VALUE=$token"
native_assets+=("$line")
isKey=1
fi
# reset for next token
parseStarted=0
token=
elif ((escaping)) && ((escaped)); then
escaping=0
escaped=0
fi
done
}
native_base_dir=$install_directory
@ -111,14 +176,14 @@ if [[ ${#native_assets[@]} -eq 0 ]]; then
exit 0;
else
native_installer_dir="$scriptroot/native"
for tool in "${!native_assets[@]}"
do
tool_version=${native_assets[$tool]}
installer_path="$native_installer_dir/install-$tool.sh"
for index in "${!native_assets[@]}"; do
eval "${native_assets["$index"]}"
installer_path="$native_installer_dir/install-$KEY.sh"
installer_command="$installer_path"
installer_command+=" --baseuri $base_uri"
installer_command+=" --installpath $install_bin"
installer_command+=" --version $tool_version"
installer_command+=" --version $VALUE"
echo $installer_command
if [[ $force = true ]]; then

8
eng/common/internal-feed-operations.ps1

@ -45,11 +45,11 @@ function SetupCredProvider {
# Then, we set the 'VSS_NUGET_EXTERNAL_FEED_ENDPOINTS' environment variable to restore from the stable
# feeds successfully
$nugetConfigPath = "$RepoRoot\NuGet.config"
$nugetConfigPath = Join-Path $RepoRoot "NuGet.config"
if (-Not (Test-Path -Path $nugetConfigPath)) {
Write-PipelineTelemetryError -Category 'Build' -Message 'NuGet.config file not found in repo root!'
ExitWithExitCode 1
ExitWithExitCode 1
}
$endpoints = New-Object System.Collections.ArrayList
@ -63,8 +63,6 @@ function SetupCredProvider {
}
if (($endpoints | Measure-Object).Count -gt 0) {
# [SuppressMessage("Microsoft.Security", "CS002:SecretInNextLine", Justification="Endpoint code example with no real credentials.")]
# Create the JSON object. It should look like '{"endpointCredentials": [{"endpoint":"http://example.index.json", "username":"optional", "password":"accesstoken"}]}'
$endpointCredentials = @{endpointCredentials=$endpoints} | ConvertTo-Json -Compress
# Create the environment variables the AzDo way
@ -87,7 +85,7 @@ function SetupCredProvider {
#Workaround for https://github.com/microsoft/msbuild/issues/4430
function InstallDotNetSdkAndRestoreArcade {
$dotnetTempDir = "$RepoRoot\dotnet"
$dotnetTempDir = Join-Path $RepoRoot "dotnet"
$dotnetSdkVersion="2.1.507" # After experimentation we know this version works when restoring the SDK (compared to 3.0.*)
$dotnet = "$dotnetTempDir\dotnet.exe"
$restoreProjPath = "$PSScriptRoot\restore.proj"

6
eng/common/internal-feed-operations.sh

@ -39,7 +39,7 @@ function SetupCredProvider {
# Then, we set the 'VSS_NUGET_EXTERNAL_FEED_ENDPOINTS' environment variable to restore from the stable
# feeds successfully
local nugetConfigPath="$repo_root/NuGet.config"
local nugetConfigPath="{$repo_root}NuGet.config"
if [ ! "$nugetConfigPath" ]; then
Write-PipelineTelemetryError -category 'Build' "NuGet.config file not found in repo's root!"
@ -62,8 +62,6 @@ function SetupCredProvider {
endpoints+=']'
if [ ${#endpoints} -gt 2 ]; then
# [SuppressMessage("Microsoft.Security", "CS002:SecretInNextLine", Justification="Endpoint code example with no real credentials.")]
# Create the JSON object. It should look like '{"endpointCredentials": [{"endpoint":"http://example.index.json", "username":"optional", "password":"accesstoken"}]}'
local endpointCredentials="{\"endpointCredentials\": "$endpoints"}"
echo "##vso[task.setvariable variable=VSS_NUGET_EXTERNAL_FEED_ENDPOINTS]$endpointCredentials"
@ -103,7 +101,7 @@ authToken=''
repoName=''
while [[ $# > 0 ]]; do
opt="$(echo "$1" | awk '{print tolower($0)}')"
opt="$(echo "$1" | tr "[:upper:]" "[:lower:]")"
case "$opt" in
--operation)
operation=$2

1
eng/common/msbuild.ps1

@ -5,6 +5,7 @@ Param(
[bool] $nodeReuse = $true,
[switch] $ci,
[switch] $prepareMachine,
[switch] $excludePrereleaseVS,
[Parameter(ValueFromRemainingArguments=$true)][String[]]$extraArgs
)

2
eng/common/msbuild.sh

@ -19,7 +19,7 @@ prepare_machine=false
extra_args=''
while (($# > 0)); do
lowerI="$(echo $1 | awk '{print tolower($0)}')"
lowerI="$(echo $1 | tr "[:upper:]" "[:lower:]")"
case $lowerI in
--verbosity)
verbosity=$2

2
eng/common/native/CommonLibrary.psm1

@ -48,7 +48,7 @@ function DownloadAndExtract {
-Verbose:$Verbose
if ($DownloadStatus -Eq $False) {
Write-Error "Download failed"
Write-Error "Download failed from $Uri"
return $False
}

6
eng/common/native/install-cmake-test.sh

@ -14,7 +14,7 @@ download_retries=5
retry_wait_time_seconds=30
while (($# > 0)); do
lowerI="$(echo $1 | awk '{print tolower($0)}')"
lowerI="$(echo $1 | tr "[:upper:]" "[:lower:]")"
case $lowerI in
--baseuri)
base_uri=$2
@ -63,7 +63,7 @@ done
tool_name="cmake-test"
tool_os=$(GetCurrentOS)
tool_folder=$(echo $tool_os | awk '{print tolower($0)}')
tool_folder="$(echo $tool_os | tr "[:upper:]" "[:lower:]")"
tool_arch="x86_64"
tool_name_moniker="$tool_name-$version-$tool_os-$tool_arch"
tool_install_directory="$install_path/$tool_name/$version"
@ -114,4 +114,4 @@ if [[ $? != 0 ]]; then
exit 1
fi
exit 0
exit 0

6
eng/common/native/install-cmake.sh

@ -14,7 +14,7 @@ download_retries=5
retry_wait_time_seconds=30
while (($# > 0)); do
lowerI="$(echo $1 | awk '{print tolower($0)}')"
lowerI="$(echo $1 | tr "[:upper:]" "[:lower:]")"
case $lowerI in
--baseuri)
base_uri=$2
@ -63,7 +63,7 @@ done
tool_name="cmake"
tool_os=$(GetCurrentOS)
tool_folder=$(echo $tool_os | awk '{print tolower($0)}')
tool_folder="$(echo $tool_os | tr "[:upper:]" "[:lower:]")"
tool_arch="x86_64"
tool_name_moniker="$tool_name-$version-$tool_os-$tool_arch"
tool_install_directory="$install_path/$tool_name/$version"
@ -114,4 +114,4 @@ if [[ $? != 0 ]]; then
exit 1
fi
exit 0
exit 0

2
eng/common/native/install-tool.ps1

@ -105,7 +105,7 @@ try {
Write-Error "There are multiple copies of $ToolName in $($ToolInstallDirectory): `n$(@($ToolFilePath | out-string))"
exit 1
} elseif (@($ToolFilePath).Length -Lt 1) {
Write-Host "$ToolName was not found in $ToolFilePath."
Write-Host "$ToolName was not found in $ToolInstallDirectory."
exit 1
}

104
eng/common/pipeline-logging-functions.ps1

@ -29,14 +29,14 @@ function Write-PipelineTelemetryError {
[switch]$AsOutput,
[switch]$Force)
$PSBoundParameters.Remove('Category') | Out-Null
$PSBoundParameters.Remove('Category') | Out-Null
if($Force -Or ((Test-Path variable:ci) -And $ci)) {
$Message = "(NETCORE_ENGINEERING_TELEMETRY=$Category) $Message"
}
$PSBoundParameters.Remove('Message') | Out-Null
$PSBoundParameters.Add('Message', $Message)
Write-PipelineTaskError @PSBoundParameters
if ($Force -Or ((Test-Path variable:ci) -And $ci)) {
$Message = "(NETCORE_ENGINEERING_TELEMETRY=$Category) $Message"
}
$PSBoundParameters.Remove('Message') | Out-Null
$PSBoundParameters.Add('Message', $Message)
Write-PipelineTaskError @PSBoundParameters
}
# Specify "-Force" to force pipeline formatted output even if "$ci" is false or not set
@ -55,8 +55,8 @@ function Write-PipelineTaskError {
[switch]$Force
)
if(!$Force -And (-Not (Test-Path variable:ci) -Or !$ci)) {
if($Type -eq 'error') {
if (!$Force -And (-Not (Test-Path variable:ci) -Or !$ci)) {
if ($Type -eq 'error') {
Write-Host $Message -ForegroundColor Red
return
}
@ -66,47 +66,61 @@ function Write-PipelineTaskError {
}
}
if(($Type -ne 'error') -and ($Type -ne 'warning')) {
if (($Type -ne 'error') -and ($Type -ne 'warning')) {
Write-Host $Message
return
}
$PSBoundParameters.Remove('Force') | Out-Null
if(-not $PSBoundParameters.ContainsKey('Type')) {
if (-not $PSBoundParameters.ContainsKey('Type')) {
$PSBoundParameters.Add('Type', 'error')
}
Write-LogIssue @PSBoundParameters
}
}
function Write-PipelineSetVariable {
function Write-PipelineSetVariable {
[CmdletBinding()]
param(
[Parameter(Mandatory = $true)]
[string]$Name,
[string]$Value,
[switch]$Secret,
[switch]$AsOutput,
[bool]$IsMultiJobVariable=$true)
if((Test-Path variable:ci) -And $ci) {
[Parameter(Mandatory = $true)]
[string]$Name,
[string]$Value,
[switch]$Secret,
[switch]$AsOutput,
[bool]$IsMultiJobVariable = $true)
if ((Test-Path variable:ci) -And $ci) {
Write-LoggingCommand -Area 'task' -Event 'setvariable' -Data $Value -Properties @{
'variable' = $Name
'isSecret' = $Secret
'isOutput' = $IsMultiJobVariable
'variable' = $Name
'isSecret' = $Secret
'isOutput' = $IsMultiJobVariable
} -AsOutput:$AsOutput
}
}
}
}
function Write-PipelinePrependPath {
function Write-PipelinePrependPath {
[CmdletBinding()]
param(
[Parameter(Mandatory=$true)]
[string]$Path,
[switch]$AsOutput)
[Parameter(Mandatory = $true)]
[string]$Path,
[switch]$AsOutput)
if((Test-Path variable:ci) -And $ci) {
if ((Test-Path variable:ci) -And $ci) {
Write-LoggingCommand -Area 'task' -Event 'prependpath' -Data $Path -AsOutput:$AsOutput
}
}
}
}
function Write-PipelineSetResult {
[CmdletBinding()]
param(
[ValidateSet("Succeeded", "SucceededWithIssues", "Failed", "Cancelled", "Skipped")]
[Parameter(Mandatory = $true)]
[string]$Result,
[string]$Message)
if ((Test-Path variable:ci) -And $ci) {
Write-LoggingCommand -Area 'task' -Event 'complete' -Data $Message -Properties @{
'result' = $Result
}
}
}
<########################################
# Private functions.
@ -123,7 +137,8 @@ function Format-LoggingCommandData {
foreach ($mapping in $script:loggingCommandEscapeMappings) {
$Value = $Value.Replace($mapping.Token, $mapping.Replacement)
}
} else {
}
else {
for ($i = $script:loggingCommandEscapeMappings.Length - 1 ; $i -ge 0 ; $i--) {
$mapping = $script:loggingCommandEscapeMappings[$i]
$Value = $Value.Replace($mapping.Replacement, $mapping.Token)
@ -156,7 +171,8 @@ function Format-LoggingCommand {
if ($first) {
$null = $sb.Append(' ')
$first = $false
} else {
}
else {
$null = $sb.Append(';')
}
@ -193,7 +209,8 @@ function Write-LoggingCommand {
$command = Format-LoggingCommand -Area $Area -Event $Event -Data $Data -Properties $Properties
if ($AsOutput) {
$command
} else {
}
else {
Write-Host $command
}
}
@ -212,12 +229,12 @@ function Write-LogIssue {
[switch]$AsOutput)
$command = Format-LoggingCommand -Area 'task' -Event 'logissue' -Data $Message -Properties @{
'type' = $Type
'code' = $ErrCode
'sourcepath' = $SourcePath
'linenumber' = $LineNumber
'columnnumber' = $ColumnNumber
}
'type' = $Type
'code' = $ErrCode
'sourcepath' = $SourcePath
'linenumber' = $LineNumber
'columnnumber' = $ColumnNumber
}
if ($AsOutput) {
return $command
}
@ -229,7 +246,8 @@ function Write-LogIssue {
$foregroundColor = [System.ConsoleColor]::Red
$backgroundColor = [System.ConsoleColor]::Black
}
} else {
}
else {
$foregroundColor = $host.PrivateData.WarningForegroundColor
$backgroundColor = $host.PrivateData.WarningBackgroundColor
if ($foregroundColor -isnot [System.ConsoleColor] -or $backgroundColor -isnot [System.ConsoleColor]) {

34
eng/common/pipeline-logging-functions.sh

@ -6,7 +6,7 @@ function Write-PipelineTelemetryError {
local function_args=()
local message=''
while [[ $# -gt 0 ]]; do
opt="$(echo "${1/#--/-}" | awk '{print tolower($0)}')"
opt="$(echo "${1/#--/-}" | tr "[:upper:]" "[:lower:]")"
case "$opt" in
-category|-c)
telemetry_category=$2
@ -48,7 +48,7 @@ function Write-PipelineTaskError {
local force=false
while [[ $# -gt 0 ]]; do
opt="$(echo "${1/#--/-}" | awk '{print tolower($0)}')"
opt="$(echo "${1/#--/-}" | tr "[:upper:]" "[:lower:]")"
case "$opt" in
-type|-t)
message_type=$2
@ -122,7 +122,7 @@ function Write-PipelineSetVariable {
local is_multi_job_variable=true
while [[ $# -gt 0 ]]; do
opt="$(echo "${1/#--/-}" | awk '{print tolower($0)}')"
opt="$(echo "${1/#--/-}" | tr "[:upper:]" "[:lower:]")"
case "$opt" in
-name|-n)
name=$2
@ -164,7 +164,7 @@ function Write-PipelinePrependPath {
local prepend_path=''
while [[ $# -gt 0 ]]; do
opt="$(echo "${1/#--/-}" | awk '{print tolower($0)}')"
opt="$(echo "${1/#--/-}" | tr "[:upper:]" "[:lower:]")"
case "$opt" in
-path|-p)
prepend_path=$2
@ -179,4 +179,28 @@ function Write-PipelinePrependPath {
if [[ "$ci" == true ]]; then
echo "##vso[task.prependpath]$prepend_path"
fi
}
}
function Write-PipelineSetResult {
local result=''
local message=''
while [[ $# -gt 0 ]]; do
opt="$(echo "${1/#--/-}" | tr "[:upper:]" "[:lower:]")"
case "$opt" in
-result|-r)
result=$2
shift
;;
-message|-m)
message=$2
shift
;;
esac
shift
done
if [[ "$ci" == true ]]; then
echo "##vso[task.complete result=$result;]$message"
fi
}

6
eng/common/post-build/post-build-utils.ps1

@ -69,9 +69,9 @@ function Trigger-Subscription([string]$SubscriptionId) {
function Validate-MaestroVars {
try {
Get-Variable MaestroApiEndPoint -Scope Global | Out-Null
Get-Variable MaestroApiVersion -Scope Global | Out-Null
Get-Variable MaestroApiAccessToken -Scope Global | Out-Null
Get-Variable MaestroApiEndPoint | Out-Null
Get-Variable MaestroApiVersion | Out-Null
Get-Variable MaestroApiAccessToken | Out-Null
if (!($MaestroApiEndPoint -Match '^http[s]?://maestro-(int|prod).westus2.cloudapp.azure.com$')) {
Write-PipelineTelemetryError -Category 'MaestroVars' -Message "MaestroApiEndPoint is not a valid Maestro URL. '$MaestroApiEndPoint'"

14
eng/common/post-build/publish-using-darc.ps1

@ -10,21 +10,27 @@ param(
[Parameter(Mandatory=$false)][string] $EnableNugetValidation,
[Parameter(Mandatory=$false)][string] $PublishInstallersAndChecksums,
[Parameter(Mandatory=$false)][string] $ArtifactsPublishingAdditionalParameters,
[Parameter(Mandatory=$false)][string] $SymbolPublishingAdditionalParameters,
[Parameter(Mandatory=$false)][string] $SigningValidationAdditionalParameters
)
try {
. $PSScriptRoot\post-build-utils.ps1
# Hard coding darc version till the next arcade-services roll out, cos this version has required API changes for darc add-build-to-channel
$darc = Get-Darc "1.1.0-beta.20418.1"
$darc = Get-Darc
$optionalParams = [System.Collections.ArrayList]::new()
if ("" -ne $ArtifactsPublishingAdditionalParameters) {
$optionalParams.Add("artifact-publishing-parameters") | Out-Null
$optionalParams.Add("--artifact-publishing-parameters") | Out-Null
$optionalParams.Add($ArtifactsPublishingAdditionalParameters) | Out-Null
}
if ("" -ne $SymbolPublishingAdditionalParameters) {
$optionalParams.Add("--symbol-publishing-parameters") | Out-Null
$optionalParams.Add($SymbolPublishingAdditionalParameters) | Out-Null
}
if ("false" -eq $WaitPublishingFinish) {
$optionalParams.Add("--no-wait") | Out-Null
}
@ -54,7 +60,7 @@ try {
--id $buildId `
--publishing-infra-version $PublishingInfraVersion `
--default-channels `
--source-branch master `
--source-branch main `
--azdev-pat $AzdoToken `
--bar-uri $MaestroApiEndPoint `
--password $MaestroToken `

95
eng/common/post-build/sourcelink-validation.ps1

@ -14,7 +14,9 @@ param(
$global:RepoFiles = @{}
# Maximum number of jobs to run in parallel
$MaxParallelJobs = 6
$MaxParallelJobs = 16
$MaxRetries = 5
# Wait time between check for system load
$SecondsBetweenLoadChecks = 10
@ -29,7 +31,10 @@ $ValidatePackage = {
# Ensure input file exist
if (!(Test-Path $PackagePath)) {
Write-Host "Input file does not exist: $PackagePath"
return 1
return [pscustomobject]@{
result = 1
packagePath = $PackagePath
}
}
# Extensions for which we'll look for SourceLink information
@ -59,7 +64,10 @@ $ValidatePackage = {
# We ignore resource DLLs
if ($FileName.EndsWith('.resources.dll')) {
return
return [pscustomobject]@{
result = 0
packagePath = $PackagePath
}
}
[System.IO.Compression.ZipFileExtensions]::ExtractToFile($_, $TargetFile, $true)
@ -91,36 +99,49 @@ $ValidatePackage = {
$Status = 200
$Cache = $using:RepoFiles
if ( !($Cache.ContainsKey($FilePath)) ) {
try {
$Uri = $Link -as [System.URI]
# Only GitHub links are valid
if ($Uri.AbsoluteURI -ne $null -and ($Uri.Host -match 'github' -or $Uri.Host -match 'githubusercontent')) {
$Status = (Invoke-WebRequest -Uri $Link -UseBasicParsing -Method HEAD -TimeoutSec 5).StatusCode
$totalRetries = 0
while ($totalRetries -lt $using:MaxRetries) {
if ( !($Cache.ContainsKey($FilePath)) ) {
try {
$Uri = $Link -as [System.URI]
# Only GitHub links are valid
if ($Uri.AbsoluteURI -ne $null -and ($Uri.Host -match 'github' -or $Uri.Host -match 'githubusercontent')) {
$Status = (Invoke-WebRequest -Uri $Link -UseBasicParsing -Method HEAD -TimeoutSec 5).StatusCode
}
else {
# If it's not a github link, we want to break out of the loop and not retry.
$Status = 0
$totalRetries = $using:MaxRetries
}
}
else {
catch {
Write-Host $_
$Status = 0
}
}
catch {
write-host $_
$Status = 0
}
}
if ($Status -ne 200) {
if ($NumFailedLinks -eq 0) {
if ($FailedFiles.Value -eq 0) {
Write-Host
if ($Status -ne 200) {
$totalRetries++
if ($totalRetries -ge $using:MaxRetries) {
if ($NumFailedLinks -eq 0) {
if ($FailedFiles.Value -eq 0) {
Write-Host
}
Write-Host "`tFile $RealPath has broken links:"
}
Write-Host "`t`tFailed to retrieve $Link"
$NumFailedLinks++
}
Write-Host "`tFile $RealPath has broken links:"
}
Write-Host "`t`tFailed to retrieve $Link"
$NumFailedLinks++
else {
break
}
}
}
}
@ -136,7 +157,7 @@ $ValidatePackage = {
}
}
catch {
Write-Host $_
}
finally {
$zip.Dispose()
@ -161,9 +182,12 @@ $ValidatePackage = {
function CheckJobResult(
$result,
$packagePath,
[ref]$ValidationFailures) {
if ($jobResult.result -ne '0') {
Write-PipelineTelemetryError -Category 'SourceLink' -Message "$packagePath has broken SourceLink links."
[ref]$ValidationFailures,
[switch]$logErrors) {
if ($result -ne '0') {
if ($logErrors) {
Write-PipelineTelemetryError -Category 'SourceLink' -Message "$packagePath has broken SourceLink links."
}
$ValidationFailures.Value++
}
}
@ -217,6 +241,7 @@ function ValidateSourceLinkLinks {
# Process each NuGet package in parallel
Get-ChildItem "$InputPath\*.symbols.nupkg" |
ForEach-Object {
Write-Host "Starting $($_.FullName)"
Start-Job -ScriptBlock $ValidatePackage -ArgumentList $_.FullName | Out-Null
$NumJobs = @(Get-Job -State 'Running').Count
@ -228,16 +253,14 @@ function ValidateSourceLinkLinks {
foreach ($Job in @(Get-Job -State 'Completed')) {
$jobResult = Wait-Job -Id $Job.Id | Receive-Job
CheckJobResult $jobResult.result $jobResult.packagePath ([ref]$ValidationFailures)
CheckJobResult $jobResult.result $jobResult.packagePath ([ref]$ValidationFailures) -LogErrors
Remove-Job -Id $Job.Id
}
}
foreach ($Job in @(Get-Job)) {
$jobResult = Wait-Job -Id $Job.Id | Receive-Job
if ($jobResult -ne '0') {
$ValidationFailures++
}
CheckJobResult $jobResult.result $jobResult.packagePath ([ref]$ValidationFailures)
Remove-Job -Id $Job.Id
}
if ($ValidationFailures -gt 0) {
@ -266,6 +289,10 @@ function InstallSourcelinkCli {
try {
InstallSourcelinkCli
foreach ($Job in @(Get-Job)) {
Remove-Job -Id $Job.Id
}
ValidateSourceLinkLinks
}
catch {

182
eng/common/post-build/symbols-validation.ps1

@ -1,30 +1,49 @@
param(
[Parameter(Mandatory=$true)][string] $InputPath, # Full path to directory where NuGet packages to be checked are stored
[Parameter(Mandatory=$true)][string] $ExtractPath, # Full path to directory where the packages will be extracted during validation
[Parameter(Mandatory=$true)][string] $DotnetSymbolVersion, # Version of dotnet symbol to use
[Parameter(Mandatory=$false)][switch] $ContinueOnError, # If we should keep checking symbols after an error
[Parameter(Mandatory=$false)][switch] $Clean # Clean extracted symbols directory after checking symbols
[Parameter(Mandatory = $true)][string] $InputPath, # Full path to directory where NuGet packages to be checked are stored
[Parameter(Mandatory = $true)][string] $ExtractPath, # Full path to directory where the packages will be extracted during validation
[Parameter(Mandatory = $true)][string] $DotnetSymbolVersion, # Version of dotnet symbol to use
[Parameter(Mandatory = $false)][switch] $CheckForWindowsPdbs, # If we should check for the existence of windows pdbs in addition to portable PDBs
[Parameter(Mandatory = $false)][switch] $ContinueOnError, # If we should keep checking symbols after an error
[Parameter(Mandatory = $false)][switch] $Clean # Clean extracted symbols directory after checking symbols
)
# Maximum number of jobs to run in parallel
$MaxParallelJobs = 6
$MaxParallelJobs = 16
# Max number of retries
$MaxRetry = 5
# Wait time between check for system load
$SecondsBetweenLoadChecks = 10
# Set error codes
Set-Variable -Name "ERROR_BADEXTRACT" -Option Constant -Value -1
Set-Variable -Name "ERROR_FILEDOESNOTEXIST" -Option Constant -Value -2
$WindowsPdbVerificationParam = ""
if ($CheckForWindowsPdbs) {
$WindowsPdbVerificationParam = "--windows-pdbs"
}
$CountMissingSymbols = {
param(
[string] $PackagePath # Path to a NuGet package
[string] $PackagePath, # Path to a NuGet package
[string] $WindowsPdbVerificationParam # If we should check for the existence of windows pdbs in addition to portable PDBs
)
. $using:PSScriptRoot\..\tools.ps1
Add-Type -AssemblyName System.IO.Compression.FileSystem
Write-Host "Validating $PackagePath "
# Ensure input file exist
if (!(Test-Path $PackagePath)) {
Write-PipelineTaskError "Input file does not exist: $PackagePath"
return -2
return [pscustomobject]@{
result = $using:ERROR_FILEDOESNOTEXIST
packagePath = $PackagePath
}
}
# Extensions for which we'll look for symbols
@ -45,24 +64,25 @@ $CountMissingSymbols = {
Write-Host "Something went wrong extracting $PackagePath"
Write-Host $_
return [pscustomobject]@{
result = -1
result = $using:ERROR_BADEXTRACT
packagePath = $PackagePath
}
}
Get-ChildItem -Recurse $ExtractPath |
Where-Object {$RelevantExtensions -contains $_.Extension} |
ForEach-Object {
$FileName = $_.FullName
if ($FileName -Match '\\ref\\') {
Write-Host "`t Ignoring reference assembly file " $FileName
return
}
Where-Object { $RelevantExtensions -contains $_.Extension } |
ForEach-Object {
$FileName = $_.FullName
if ($FileName -Match '\\ref\\') {
Write-Host "`t Ignoring reference assembly file " $FileName
return
}
$FirstMatchingSymbolDescriptionOrDefault = {
$FirstMatchingSymbolDescriptionOrDefault = {
param(
[string] $FullPath, # Full path to the module that has to be checked
[string] $TargetServerParam, # Parameter to pass to `Symbol Tool` indicating the server to lookup for symbols
[string] $FullPath, # Full path to the module that has to be checked
[string] $TargetServerParam, # Parameter to pass to `Symbol Tool` indicating the server to lookup for symbols
[string] $WindowsPdbVerificationParam, # Parameter to pass to potential check for windows-pdbs.
[string] $SymbolsPath
)
@ -87,56 +107,76 @@ $CountMissingSymbols = {
# DWARF file for a .dylib
$DylibDwarf = $SymbolPath.Replace($Extension, '.dylib.dwarf')
$dotnetSymbolExe = "$env:USERPROFILE\.dotnet\tools"
$dotnetSymbolExe = Resolve-Path "$dotnetSymbolExe\dotnet-symbol.exe"
& $dotnetSymbolExe --symbols --modules --windows-pdbs $TargetServerParam $FullPath -o $SymbolsPath | Out-Null
$totalRetries = 0
if (Test-Path $PdbPath) {
return 'PDB'
}
elseif (Test-Path $NGenPdb) {
return 'NGen PDB'
}
elseif (Test-Path $SODbg) {
return 'DBG for SO'
}
elseif (Test-Path $DylibDwarf) {
return 'Dwarf for Dylib'
}
elseif (Test-Path $SymbolPath) {
return 'Module'
}
else {
return $null
while ($totalRetries -lt $using:MaxRetry) {
# Save the output and get diagnostic output
$output = & $dotnetSymbolExe --symbols --modules $WindowsPdbVerificationParam $TargetServerParam $FullPath -o $SymbolsPath --diagnostics | Out-String
if (Test-Path $PdbPath) {
return 'PDB'
}
elseif (Test-Path $NGenPdb) {
return 'NGen PDB'
}
elseif (Test-Path $SODbg) {
return 'DBG for SO'
}
elseif (Test-Path $DylibDwarf) {
return 'Dwarf for Dylib'
}
elseif (Test-Path $SymbolPath) {
return 'Module'
}
else
{
$totalRetries++
}
}
return $null
}
$SymbolsOnMSDL = & $FirstMatchingSymbolDescriptionOrDefault $FileName '--microsoft-symbol-server' $SymbolsPath
$SymbolsOnSymWeb = & $FirstMatchingSymbolDescriptionOrDefault $FileName '--internal-server' $SymbolsPath
Write-Host -NoNewLine "`t Checking file " $FileName "... "
$FileGuid = New-Guid
$ExpandedSymbolsPath = Join-Path -Path $SymbolsPath -ChildPath $FileGuid
$SymbolsOnMSDL = & $FirstMatchingSymbolDescriptionOrDefault `
-FullPath $FileName `
-TargetServerParam '--microsoft-symbol-server' `
-SymbolsPath "$ExpandedSymbolsPath-msdl" `
-WindowsPdbVerificationParam $WindowsPdbVerificationParam
$SymbolsOnSymWeb = & $FirstMatchingSymbolDescriptionOrDefault `
-FullPath $FileName `
-TargetServerParam '--internal-server' `
-SymbolsPath "$ExpandedSymbolsPath-symweb" `
-WindowsPdbVerificationParam $WindowsPdbVerificationParam
Write-Host -NoNewLine "`t Checking file " $FileName "... "
if ($SymbolsOnMSDL -ne $null -and $SymbolsOnSymWeb -ne $null) {
Write-Host "Symbols found on MSDL ($SymbolsOnMSDL) and SymWeb ($SymbolsOnSymWeb)"
if ($SymbolsOnMSDL -ne $null -and $SymbolsOnSymWeb -ne $null) {
Write-Host "Symbols found on MSDL ($SymbolsOnMSDL) and SymWeb ($SymbolsOnSymWeb)"
}
else {
$MissingSymbols++
if ($SymbolsOnMSDL -eq $null -and $SymbolsOnSymWeb -eq $null) {
Write-Host 'No symbols found on MSDL or SymWeb!'
}
else {
$MissingSymbols++
if ($SymbolsOnMSDL -eq $null -and $SymbolsOnSymWeb -eq $null) {
Write-Host 'No symbols found on MSDL or SymWeb!'
if ($SymbolsOnMSDL -eq $null) {
Write-Host 'No symbols found on MSDL!'
}
else {
if ($SymbolsOnMSDL -eq $null) {
Write-Host 'No symbols found on MSDL!'
}
else {
Write-Host 'No symbols found on SymWeb!'
}
Write-Host 'No symbols found on SymWeb!'
}
}
}
}
if ($using:Clean) {
Remove-Item $ExtractPath -Recurse -Force
@ -145,24 +185,31 @@ $CountMissingSymbols = {
Pop-Location
return [pscustomobject]@{
result = $MissingSymbols
packagePath = $PackagePath
}
result = $MissingSymbols
packagePath = $PackagePath
}
}
function CheckJobResult(
$result,
$packagePath,
[ref]$DupedSymbols,
[ref]$TotalFailures) {
if ($result -eq '-1') {
$result,
$packagePath,
[ref]$DupedSymbols,
[ref]$TotalFailures) {
if ($result -eq $ERROR_BADEXTRACT) {
Write-PipelineTelemetryError -Category 'CheckSymbols' -Message "$packagePath has duplicated symbol files"
$DupedSymbols.Value++
}
elseif ($jobResult.result -ne '0') {
elseif ($result -eq $ERROR_FILEDOESNOTEXIST) {
Write-PipelineTelemetryError -Category 'CheckSymbols' -Message "$packagePath does not exist"
$TotalFailures.Value++
}
elseif ($result -gt '0') {
Write-PipelineTelemetryError -Category 'CheckSymbols' -Message "Missing symbols for $result modules in the package $packagePath"
$TotalFailures.Value++
}
else {
Write-Host "All symbols verified for package $packagePath"
}
}
function CheckSymbolsAvailable {
@ -170,6 +217,7 @@ function CheckSymbolsAvailable {
Remove-Item $ExtractPath -Force -Recurse -ErrorAction SilentlyContinue
}
$TotalPackages = 0
$TotalFailures = 0
$DupedSymbols = 0
@ -192,9 +240,9 @@ function CheckSymbolsAvailable {
return
}
Write-Host "Validating $FileName "
$TotalPackages++
Start-Job -ScriptBlock $CountMissingSymbols -ArgumentList $FullName | Out-Null
Start-Job -ScriptBlock $CountMissingSymbols -ArgumentList @($FullName,$WindowsPdbVerificationParam) | Out-Null
$NumJobs = @(Get-Job -State 'Running').Count
@ -219,11 +267,11 @@ function CheckSymbolsAvailable {
if ($TotalFailures -gt 0 -or $DupedSymbols -gt 0) {
if ($TotalFailures -gt 0) {
Write-PipelineTelemetryError -Category 'CheckSymbols' -Message "Symbols missing for $TotalFailures packages"
Write-PipelineTelemetryError -Category 'CheckSymbols' -Message "Symbols missing for $TotalFailures/$TotalPackages packages"
}
if ($DupedSymbols -gt 0) {
Write-PipelineTelemetryError -Category 'CheckSymbols' -Message "$DupedSymbols packages had duplicated symbol files"
Write-PipelineTelemetryError -Category 'CheckSymbols' -Message "$DupedSymbols/$TotalPackages packages had duplicated symbol files and could not be extracted"
}
ExitWithExitCode 1

8
eng/common/sdk-task.ps1

@ -34,7 +34,7 @@ function Print-Usage() {
function Build([string]$target) {
$logSuffix = if ($target -eq 'Execute') { '' } else { ".$target" }
$log = Join-Path $LogDir "$task$logSuffix.binlog"
$outputPath = Join-Path $ToolsetDir "$task\\"
$outputPath = Join-Path $ToolsetDir "$task\"
MSBuild $taskProject `
/bl:$log `
@ -53,7 +53,7 @@ try {
}
if ($task -eq "") {
Write-PipelineTelemetryError -Category 'Build' -Message "Missing required parameter '-task <value>'" -ForegroundColor Red
Write-PipelineTelemetryError -Category 'Build' -Message "Missing required parameter '-task <value>'"
Print-Usage
ExitWithExitCode 1
}
@ -64,7 +64,7 @@ try {
$GlobalJson.tools | Add-Member -Name "vs" -Value (ConvertFrom-Json "{ `"version`": `"16.5`" }") -MemberType NoteProperty
}
if( -not ($GlobalJson.tools.PSObject.Properties.Name -match "xcopy-msbuild" )) {
$GlobalJson.tools | Add-Member -Name "xcopy-msbuild" -Value "16.8.0-preview3" -MemberType NoteProperty
$GlobalJson.tools | Add-Member -Name "xcopy-msbuild" -Value "16.10.0-preview2" -MemberType NoteProperty
}
if ($GlobalJson.tools."xcopy-msbuild".Trim() -ine "none") {
$xcopyMSBuildToolsFolder = InitializeXCopyMSBuild $GlobalJson.tools."xcopy-msbuild" -install $true
@ -78,7 +78,7 @@ try {
$taskProject = GetSdkTaskProject $task
if (!(Test-Path $taskProject)) {
Write-PipelineTelemetryError -Category 'Build' -Message "Unknown task: $task" -ForegroundColor Red
Write-PipelineTelemetryError -Category 'Build' -Message "Unknown task: $task"
ExitWithExitCode 1
}

6
eng/common/sdl/execute-all-sdl-tools.ps1

@ -32,7 +32,7 @@ try {
$ErrorActionPreference = 'Stop'
Set-StrictMode -Version 2.0
$disableConfigureToolsetImport = $true
$LASTEXITCODE = 0
$global:LASTEXITCODE = 0
# `tools.ps1` checks $ci to perform some actions. Since the SDL
# scripts don't necessarily execute in the same agent that run the
@ -87,10 +87,6 @@ try {
& $(Join-Path $PSScriptRoot 'run-sdl.ps1') -GuardianCliLocation $guardianCliLocation -WorkingDirectory $workingDirectory -TargetDirectory $SourceDirectory -GdnFolder $gdnFolder -ToolsList $SourceToolsList -AzureDevOpsAccessToken $AzureDevOpsAccessToken -UpdateBaseline $UpdateBaseline -GuardianLoggerLevel $GuardianLoggerLevel -CrScanAdditionalRunConfigParams $CrScanAdditionalRunConfigParams -PoliCheckAdditionalRunConfigParams $PoliCheckAdditionalRunConfigParams
}
if ($UpdateBaseline) {
& (Join-Path $PSScriptRoot 'push-gdn.ps1') -Repository $RepoName -BranchName $BranchName -GdnFolder $GdnFolder -AzureDevOpsAccessToken $AzureDevOpsAccessToken -PushReason 'Update baseline'
}
if ($TsaPublish) {
if ($TsaBranchName -and $BuildNumber) {
if (-not $TsaRepositoryName) {

16
eng/common/sdl/init-sdl.ps1

@ -10,7 +10,7 @@ Param(
$ErrorActionPreference = 'Stop'
Set-StrictMode -Version 2.0
$disableConfigureToolsetImport = $true
$LASTEXITCODE = 0
$global:LASTEXITCODE = 0
# `tools.ps1` checks $ci to perform some actions. Since the SDL
# scripts don't necessarily execute in the same agent that run the
@ -29,18 +29,7 @@ $zipFile = "$WorkingDirectory/gdn.zip"
Add-Type -AssemblyName System.IO.Compression.FileSystem
$gdnFolder = (Join-Path $WorkingDirectory '.gdn')
try {
# We try to download the zip; if the request fails (e.g. the file doesn't exist), we catch it and init guardian instead
Write-Host 'Downloading gdn folder from internal config repostiory...'
Invoke-WebRequest -Headers @{ "Accept"="application/zip"; "Authorization"="Basic $encodedPat" } -Uri $uri -OutFile $zipFile
if (Test-Path $gdnFolder) {
# Remove the gdn folder if it exists (it shouldn't unless there's too much caching; this is just in case)
Remove-Item -Force -Recurse $gdnFolder
}
[System.IO.Compression.ZipFile]::ExtractToDirectory($zipFile, $WorkingDirectory)
Write-Host $gdnFolder
ExitWithExitCode 0
} catch [System.Net.WebException] { } # Catch and ignore webexception
try {
# if the folder does not exist, we'll do a guardian init and push it to the remote repository
Write-Host 'Initializing Guardian...'
@ -57,7 +46,6 @@ try {
Write-PipelineTelemetryError -Force -Category 'Build' -Message "Guardian baseline failed with exit code $LASTEXITCODE."
ExitWithExitCode $LASTEXITCODE
}
& $(Join-Path $PSScriptRoot 'push-gdn.ps1') -Repository $Repository -BranchName $BranchName -GdnFolder $gdnFolder -AzureDevOpsAccessToken $AzureDevOpsAccessToken -PushReason 'Initialize gdn folder'
ExitWithExitCode 0
}
catch {

2
eng/common/sdl/packages.config

@ -1,4 +1,4 @@
<?xml version="1.0" encoding="utf-8"?>
<packages>
<package id="Microsoft.Guardian.Cli.win10-x64" version="0.20.1"/>
<package id="Microsoft.Guardian.Cli" version="0.53.3"/>
</packages>

2
eng/common/sdl/run-sdl.ps1

@ -13,7 +13,7 @@ Param(
$ErrorActionPreference = 'Stop'
Set-StrictMode -Version 2.0
$disableConfigureToolsetImport = $true
$LASTEXITCODE = 0
$global:LASTEXITCODE = 0
try {
# `tools.ps1` checks $ci to perform some actions. Since the SDL

4
eng/common/templates/job/execute-sdl.yml

@ -45,6 +45,7 @@ jobs:
buildId: $(AzDOBuildId)
artifactName: ${{ artifactName }}
downloadPath: $(Build.ArtifactStagingDirectory)\artifacts
checkDownloadedFiles: true
- ${{ if eq(parameters.artifactNames, '') }}:
- task: DownloadBuildArtifacts@0
displayName: Download Build Artifacts
@ -57,6 +58,7 @@ jobs:
downloadType: specific files
itemPattern: "**"
downloadPath: $(Build.ArtifactStagingDirectory)\artifacts
checkDownloadedFiles: true
- powershell: eng/common/sdl/extract-artifact-packages.ps1
-InputPath $(Build.ArtifactStagingDirectory)\artifacts\BlobArtifacts
-ExtractPath $(Build.ArtifactStagingDirectory)\artifacts\BlobArtifacts
@ -83,7 +85,7 @@ jobs:
continueOnError: ${{ parameters.sdlContinueOnError }}
- ${{ if eq(parameters.overrideParameters, '') }}:
- powershell: eng/common/sdl/execute-all-sdl-tools.ps1
-GuardianPackageName Microsoft.Guardian.Cli.win10-x64.0.20.1
-GuardianPackageName Microsoft.Guardian.Cli.0.53.3
-NugetPackageDirectory $(Build.SourcesDirectory)\.packages
-AzureDevOpsAccessToken $(dn-bot-dotnet-build-rw-code-rw)
${{ parameters.additionalParameters }}

18
eng/common/templates/job/job.yml

@ -24,9 +24,9 @@ parameters:
enablePublishBuildAssets: false
enablePublishTestResults: false
enablePublishUsingPipelines: false
useBuildManifest: false
mergeTestResults: false
testRunTitle: ''
testResultsFormat: ''
name: ''
preSteps: []
runAsPublic: false
@ -131,8 +131,8 @@ jobs:
- task: RichCodeNavIndexer@0
displayName: RichCodeNav Upload
inputs:
languages: 'csharp'
environment: ${{ coalesce(parameters.richCodeNavigationEnvironment, 'prod') }}
languages: ${{ coalesce(parameters.richCodeNavigationLanguage, 'csharp') }}
environment: ${{ coalesce(parameters.richCodeNavigationEnvironment, 'production') }}
richNavLogOutputDirectory: $(Build.SourcesDirectory)/artifacts/bin
continueOnError: true
@ -202,7 +202,7 @@ jobs:
continueOnError: true
condition: always()
- ${{ if eq(parameters.enablePublishTestResults, 'true') }}:
- ${{ if or(and(eq(parameters.enablePublishTestResults, 'true'), eq(parameters.testResultsFormat, '')), eq(parameters.testResultsFormat, 'xunit')) }}:
- task: PublishTestResults@2
displayName: Publish XUnit Test Results
inputs:
@ -213,6 +213,7 @@ jobs:
mergeTestResults: ${{ parameters.mergeTestResults }}
continueOnError: true
condition: always()
- ${{ if or(and(eq(parameters.enablePublishTestResults, 'true'), eq(parameters.testResultsFormat, '')), eq(parameters.testResultsFormat, 'vstest')) }}:
- task: PublishTestResults@2
displayName: Publish TRX Test Results
inputs:
@ -241,12 +242,3 @@ jobs:
ArtifactName: AssetManifests
continueOnError: ${{ parameters.continueOnError }}
condition: and(succeeded(), eq(variables['_DotNetPublishToBlobFeed'], 'true'))
- ${{ if eq(parameters.useBuildManifest, true) }}:
- task: PublishBuildArtifacts@1
displayName: Publish Build Manifest
inputs:
PathToPublish: '$(Build.SourcesDirectory)/artifacts/log/$(_BuildConfig)/manifest.props'
PublishLocation: Container
ArtifactName: BuildManifests
continueOnError: ${{ parameters.continueOnError }}

93
eng/common/templates/job/onelocbuild.yml

@ -0,0 +1,93 @@
parameters:
# Optional: dependencies of the job
dependsOn: ''
# Optional: A defined YAML pool - https://docs.microsoft.com/en-us/azure/devops/pipelines/yaml-schema?view=vsts&tabs=schema#pool
pool:
vmImage: vs2017-win2016
CeapexPat: $(dn-bot-ceapex-package-r) # PAT for the loc AzDO instance https://dev.azure.com/ceapex
GithubPat: $(BotAccount-dotnet-bot-repo-PAT)
SourcesDirectory: $(Build.SourcesDirectory)
CreatePr: true
AutoCompletePr: false
UseLfLineEndings: true
UseCheckedInLocProjectJson: false
LanguageSet: VS_Main_Languages
LclSource: lclFilesInRepo
LclPackageId: ''
RepoType: gitHub
GitHubOrg: dotnet
MirrorRepo: ''
MirrorBranch: main
condition: ''
jobs:
- job: OneLocBuild
dependsOn: ${{ parameters.dependsOn }}
displayName: OneLocBuild
pool: ${{ parameters.pool }}
variables:
- group: OneLocBuildVariables # Contains the CeapexPat and GithubPat
- name: _GenerateLocProjectArguments
value: -SourcesDirectory ${{ parameters.SourcesDirectory }}
-LanguageSet "${{ parameters.LanguageSet }}"
-CreateNeutralXlfs
- ${{ if eq(parameters.UseCheckedInLocProjectJson, 'true') }}:
- name: _GenerateLocProjectArguments
value: ${{ variables._GenerateLocProjectArguments }} -UseCheckedInLocProjectJson
steps:
- task: Powershell@2
inputs:
filePath: $(Build.SourcesDirectory)/eng/common/generate-locproject.ps1
arguments: $(_GenerateLocProjectArguments)
displayName: Generate LocProject.json
condition: ${{ parameters.condition }}
- task: OneLocBuild@2
displayName: OneLocBuild
env:
SYSTEM_ACCESSTOKEN: $(System.AccessToken)
inputs:
locProj: eng/Localize/LocProject.json
outDir: $(Build.ArtifactStagingDirectory)
lclSource: ${{ parameters.LclSource }}
lclPackageId: ${{ parameters.LclPackageId }}
isCreatePrSelected: ${{ parameters.CreatePr }}
${{ if eq(parameters.CreatePr, true) }}:
isAutoCompletePrSelected: ${{ parameters.AutoCompletePr }}
isUseLfLineEndingsSelected: ${{ parameters.UseLfLineEndings }}
packageSourceAuth: patAuth
patVariable: ${{ parameters.CeapexPat }}
${{ if eq(parameters.RepoType, 'gitHub') }}:
repoType: ${{ parameters.RepoType }}
gitHubPatVariable: "${{ parameters.GithubPat }}"
${{ if ne(parameters.MirrorRepo, '') }}:
isMirrorRepoSelected: true
gitHubOrganization: ${{ parameters.GitHubOrg }}
mirrorRepo: ${{ parameters.MirrorRepo }}
mirrorBranch: ${{ parameters.MirrorBranch }}
condition: ${{ parameters.condition }}
- task: PublishBuildArtifacts@1
displayName: Publish Localization Files
inputs:
PathtoPublish: '$(Build.ArtifactStagingDirectory)/loc'
PublishLocation: Container
ArtifactName: Loc
condition: ${{ parameters.condition }}
- task: PublishBuildArtifacts@1
displayName: Publish LocProject.json
inputs:
PathtoPublish: '$(Build.SourcesDirectory)/eng/Localize/'
PublishLocation: Container
ArtifactName: Loc
condition: ${{ parameters.condition }}

8
eng/common/templates/job/publish-build-assets.yml

@ -37,6 +37,7 @@ jobs:
- name: _BuildConfig
value: ${{ parameters.configuration }}
- group: Publish-Build-Assets
- group: AzureDevOps-Artifact-Feeds-Pats
# Skip component governance and codesign validation for SDL. These jobs
# create no content.
- name: skipComponentGovernanceDetection
@ -51,12 +52,19 @@ jobs:
inputs:
artifactName: AssetManifests
downloadPath: '$(Build.StagingDirectory)/Download'
checkDownloadedFiles: true
condition: ${{ parameters.condition }}
continueOnError: ${{ parameters.continueOnError }}
- ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}:
- task: NuGetAuthenticate@0
- task: PowerShell@2
displayName: Enable cross-org NuGet feed authentication
inputs:
filePath: $(Build.SourcesDirectory)/eng/common/enable-cross-org-publishing.ps1
arguments: -token $(dn-bot-all-orgs-artifact-feeds-rw)
- task: PowerShell@2
displayName: Publish Build Assets
inputs:

11
eng/common/templates/job/source-build.yml

@ -15,6 +15,9 @@ parameters:
# nonPortable: false
# Enables non-portable mode. This means a more specific RID (e.g. fedora.32-x64 rather than
# linux-x64), and compiling against distro-provided packages rather than portable ones.
# skipPublishValidation: false
# Disables publishing validation. By default, a check is performed to ensure no packages are
# published by source-build.
# container: ''
# A container to use. Runs in docker.
# pool: {}
@ -28,6 +31,11 @@ parameters:
# container and pool.
platform: {}
# The default VM host AzDO pool. This should be capable of running Docker containers: almost all
# source-build builds run in Docker, including the default managed platform.
defaultContainerHostPool:
vmImage: ubuntu-20.04
jobs:
- job: ${{ parameters.jobNamePrefix }}_${{ parameters.platform.name }}
displayName: Source-Build (${{ parameters.platform.name }})
@ -37,6 +45,9 @@ jobs:
${{ if ne(parameters.platform.container, '') }}:
container: ${{ parameters.platform.container }}
${{ if eq(parameters.platform.pool, '') }}:
pool: ${{ parameters.defaultContainerHostPool }}
${{ if ne(parameters.platform.pool, '') }}:
pool: ${{ parameters.platform.pool }}

62
eng/common/templates/job/source-index-stage1.yml

@ -0,0 +1,62 @@
parameters:
runAsPublic: false
sourceIndexPackageVersion: 1.0.1-20210614.1
sourceIndexPackageSource: https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools/nuget/v3/index.json
sourceIndexBuildCommand: powershell -NoLogo -NoProfile -ExecutionPolicy Bypass -Command "eng/common/build.ps1 -restore -build -binarylog -ci"
preSteps: []
binlogPath: artifacts/log/Debug/Build.binlog
pool:
vmImage: vs2017-win2016
condition: ''
dependsOn: ''
jobs:
- job: SourceIndexStage1
dependsOn: ${{ parameters.dependsOn }}
condition: ${{ parameters.condition }}
variables:
- name: SourceIndexPackageVersion
value: ${{ parameters.sourceIndexPackageVersion }}
- name: SourceIndexPackageSource
value: ${{ parameters.sourceIndexPackageSource }}
- name: BinlogPath
value: ${{ parameters.binlogPath }}
- ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}:
- group: source-dot-net stage1 variables
pool: ${{ parameters.pool }}
steps:
- ${{ each preStep in parameters.preSteps }}:
- ${{ preStep }}
- task: UseDotNet@2
displayName: Use .NET Core sdk 3.1
inputs:
packageType: sdk
version: 3.1.x
- task: UseDotNet@2
displayName: Use .NET Core sdk
inputs:
useGlobalJson: true
- script: |
dotnet tool install BinLogToSln --version $(SourceIndexPackageVersion) --add-source $(SourceIndexPackageSource) --tool-path .source-index/tools
dotnet tool install UploadIndexStage1 --version $(SourceIndexPackageVersion) --add-source $(SourceIndexPackageSource) --tool-path .source-index/tools
echo ##vso[task.prependpath]$(Build.SourcesDirectory)/.source-index/tools
displayName: Download Tools
- script: ${{ parameters.sourceIndexBuildCommand }}
displayName: Build Repository
- script: BinLogToSln -i $(BinlogPath) -r $(Build.SourcesDirectory) -n $(Build.Repository.Name) -o .source-index/stage1output
displayName: Process Binlog into indexable sln
env:
DOTNET_ROLL_FORWARD_ON_NO_CANDIDATE_FX: 2
- ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}:
- script: UploadIndexStage1 -i .source-index/stage1output -n $(Build.Repository.Name)
displayName: Upload stage1 artifacts to source index
env:
BLOB_CONTAINER_URL: $(source-dot-net-stage1-blob-container-url)
DOTNET_ROLL_FORWARD_ON_NO_CANDIDATE_FX: 2

29
eng/common/templates/jobs/jobs.yml

@ -7,7 +7,14 @@ parameters:
# Optional: Enable publishing using release pipelines
enablePublishUsingPipelines: false
# Optional: Enable running the source-build jobs to build repo from source
enableSourceBuild: false
# Optional: Parameters for source-build template.
# See /eng/common/templates/jobs/source-build.yml for options
sourceBuildParameters: []
graphFileGeneration:
# Optional: Enable generating the graph files at the end of the build
enabled: false
@ -24,12 +31,8 @@ parameters:
# if 'true', the build won't run any of the internal only steps, even if it is running in non-public projects.
runAsPublic: false
# Optional: Enable running the source-build jobs to build repo from source
runSourceBuild: false
# Optional: Parameters for source-build template.
# See /eng/common/templates/jobs/source-build.yml for options
sourceBuildParameters: []
enableSourceIndex: false
sourceIndexParams: {}
# Internal resources (telemetry, microbuild) can only be accessed from non-public projects,
# and some (Microbuild) should only be applied to non-PR cases for internal builds.
@ -50,14 +53,22 @@ jobs:
name: ${{ job.job }}
- ${{ if eq(parameters.runSourceBuild, true) }}:
- ${{ if eq(parameters.enableSourceBuild, true) }}:
- template: /eng/common/templates/jobs/source-build.yml
parameters:
allCompletedJobId: Source_Build_Complete
${{ each parameter in parameters.sourceBuildParameters }}:
${{ parameter.key }}: ${{ parameter.value }}
- ${{ if eq(parameters.enableSourceIndex, 'true') }}:
- template: ../job/source-index-stage1.yml
parameters:
runAsPublic: ${{ parameters.runAsPublic }}
${{ each parameter in parameters.sourceIndexParams }}:
${{ parameter.key }}: ${{ parameter.value }}
- ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}:
- ${{ if or(eq(parameters.enablePublishBuildAssets, true), eq(parameters.artifacts.publish.manifests, 'true'), ne(parameters.artifacts.publish.manifests, '')) }}:
- template: ../job/publish-build-assets.yml
parameters:
@ -69,7 +80,7 @@ jobs:
- ${{ if eq(parameters.publishBuildAssetsDependsOn, '') }}:
- ${{ each job in parameters.jobs }}:
- ${{ job.job }}
- ${{ if eq(parameters.runSourceBuild, true) }}:
- ${{ if eq(parameters.enableSourceBuild, true) }}:
- Source_Build_Complete
pool:
vmImage: vs2017-win2016

12
eng/common/templates/jobs/source-build.yml

@ -11,16 +11,14 @@ parameters:
# See /eng/common/templates/job/source-build.yml
jobNamePrefix: 'Source_Build'
# If changed to true, causes this template to include the default platform for a managed-only
# repo. The exact Docker image used for this build will be provided by Arcade. This has some risk,
# but since the repo is supposed to be managed-only, the risk should be very low.
includeDefaultManagedPlatform: false
# This is the default platform provided by Arcade, intended for use by a managed-only repo.
defaultManagedPlatform:
name: 'Managed'
container: 'mcr.microsoft.com/dotnet-buildtools/prereqs:centos-7-3e800f1-20190501005343'
# Defines the platforms on which to run build jobs. One job is created for each platform, and the
# object in this array is sent to the job template as 'platform'.
# object in this array is sent to the job template as 'platform'. If no platforms are specified,
# one job runs on 'defaultManagedPlatform'.
platforms: []
jobs:
@ -32,7 +30,7 @@ jobs:
dependsOn:
- ${{ each platform in parameters.platforms }}:
- ${{ parameters.jobNamePrefix }}_${{ platform.name }}
- ${{ if eq(parameters.includeDefaultManagedPlatform, true) }}:
- ${{ if eq(length(parameters.platforms), 0) }}:
- ${{ parameters.jobNamePrefix }}_${{ parameters.defaultManagedPlatform.name }}
- ${{ each platform in parameters.platforms }}:
@ -41,7 +39,7 @@ jobs:
jobNamePrefix: ${{ parameters.jobNamePrefix }}
platform: ${{ platform }}
- ${{ if eq(parameters.includeDefaultManagedPlatform, true) }}:
- ${{ if eq(length(parameters.platforms), 0) }}:
- template: /eng/common/templates/job/source-build.yml
parameters:
jobNamePrefix: ${{ parameters.jobNamePrefix }}

1
eng/common/templates/phases/publish-build-assets.yml

@ -20,6 +20,7 @@ phases:
inputs:
artifactName: AssetManifests
downloadPath: '$(Build.StagingDirectory)/Download'
checkDownloadedFiles: true
condition: ${{ parameters.condition }}
continueOnError: ${{ parameters.continueOnError }}
- task: AzureKeyVault@1

8
eng/common/templates/post-build/channels/generic-internal-channel.yml

@ -40,6 +40,9 @@ stages:
pool:
vmImage: 'windows-2019'
steps:
- script: echo "##vso[task.logissue type=warning]Going forward, v2 Arcade publishing is no longer supported. Please read https://github.com/dotnet/arcade/blob/main/Documentation/CorePackages/Publishing.md for details, then contact dnceng if you have further questions."
displayName: Warn about v2 Arcade Publishing Usage
# This is necessary whenever we want to publish/restore to an AzDO private feed
- task: NuGetAuthenticate@0
displayName: 'Authenticate to AzDO Feeds'
@ -58,6 +61,7 @@ stages:
PdbArtifacts/**
BlobArtifacts/**
downloadPath: '$(Build.ArtifactStagingDirectory)'
checkDownloadedFiles: true
# This is necessary whenever we want to publish/restore to an AzDO private feed
# Since sdk-task.ps1 tries to restore packages we need to do this authentication here
@ -109,6 +113,9 @@ stages:
pool:
vmImage: 'windows-2019'
steps:
- script: echo "##vso[task.logissue type=warning]Going forward, v2 Arcade publishing is no longer supported. Please read https://github.com/dotnet/arcade/blob/main/Documentation/CorePackages/Publishing.md for details, then contact dnceng if you have further questions."
displayName: Warn about v2 Arcade Publishing Usage
- task: DownloadBuildArtifacts@0
displayName: Download Build Assets
continueOnError: true
@ -124,6 +131,7 @@ stages:
BlobArtifacts/**
AssetManifests/**
downloadPath: '$(Build.ArtifactStagingDirectory)'
checkDownloadedFiles: true
- task: NuGetToolInstaller@1
displayName: 'Install NuGet.exe'

8
eng/common/templates/post-build/channels/generic-public-channel.yml

@ -42,6 +42,9 @@ stages:
pool:
vmImage: 'windows-2019'
steps:
- script: echo "##vso[task.logissue type=warning]Going forward, v2 Arcade publishing is no longer supported. Please read https://github.com/dotnet/arcade/blob/main/Documentation/CorePackages/Publishing.md for details, then contact dnceng if you have further questions."
displayName: Warn about v2 Arcade Publishing Usage
- task: DownloadBuildArtifacts@0
displayName: Download Build Assets
continueOnError: true
@ -56,6 +59,7 @@ stages:
PdbArtifacts/**
BlobArtifacts/**
downloadPath: '$(Build.ArtifactStagingDirectory)'
checkDownloadedFiles: true
# This is necessary whenever we want to publish/restore to an AzDO private feed
# Since sdk-task.ps1 tries to restore packages we need to do this authentication here
@ -108,6 +112,9 @@ stages:
pool:
vmImage: 'windows-2019'
steps:
- script: echo "##vso[task.logissue type=warning]Going forward, v2 Arcade publishing is no longer supported. Please read https://github.com/dotnet/arcade/blob/main/Documentation/CorePackages/Publishing.md for details, then contact dnceng if you have further questions."
displayName: Warn about v2 Arcade Publishing Usage
- task: DownloadBuildArtifacts@0
displayName: Download Build Assets
continueOnError: true
@ -123,6 +130,7 @@ stages:
BlobArtifacts/**
AssetManifests/**
downloadPath: '$(Build.ArtifactStagingDirectory)'
checkDownloadedFiles: true
- task: NuGetToolInstaller@1
displayName: 'Install NuGet.exe'

391
eng/common/templates/post-build/post-build.yml

@ -32,7 +32,6 @@ parameters:
symbolPublishingAdditionalParameters: ''
artifactsPublishingAdditionalParameters: ''
signingValidationAdditionalParameters: ''
useBuildManifest: false
# Which stages should finish execution before post-build stages start
validateDependsOn:
@ -54,9 +53,6 @@ parameters:
NETCoreExperimentalChannelId: 562
NetEngServicesIntChannelId: 678
NetEngServicesProdChannelId: 679
Net5Preview8ChannelId: 1155
Net5RC1ChannelId: 1157
Net5RC2ChannelId: 1329
NetCoreSDK313xxChannelId: 759
NetCoreSDK313xxInternalChannelId: 760
NetCoreSDK314xxChannelId: 921
@ -65,178 +61,180 @@ parameters:
VS167ChannelId: 1011
VS168ChannelId: 1154
VSMasterChannelId: 1012
VS169ChannelId: 1473
VS1610ChannelId: 1692
stages:
- stage: Validate
dependsOn: ${{ parameters.validateDependsOn }}
displayName: Validate Build Assets
variables:
- template: common-variables.yml
jobs:
- template: setup-maestro-vars.yml
parameters:
BARBuildId: ${{ parameters.BARBuildId }}
PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }}
- ${{ if or(and(le(parameters.publishingInfraVersion, 2), eq(parameters.inline, 'true')), eq( parameters.enableNugetValidation, 'true'), eq(parameters.enableSigningValidation, 'true'), eq(parameters.enableSourceLinkValidation, 'true'), eq(parameters.SDLValidationParameters.enable, 'true')) }}:
- stage: Validate
dependsOn: ${{ parameters.validateDependsOn }}
displayName: Validate Build Assets
variables:
- template: common-variables.yml
jobs:
- template: setup-maestro-vars.yml
parameters:
BARBuildId: ${{ parameters.BARBuildId }}
PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }}
- ${{ if and(le(parameters.publishingInfraVersion, 2), eq(parameters.inline, 'true')) }}:
- job:
displayName: Post-build Checks
dependsOn: setupMaestroVars
variables:
- name: TargetChannels
value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.TargetChannels'] ]
pool:
vmImage: 'windows-2019'
steps:
- task: PowerShell@2
displayName: Maestro Channels Consistency
inputs:
filePath: $(Build.SourcesDirectory)/eng/common/post-build/check-channel-consistency.ps1
arguments: -PromoteToChannels "$(TargetChannels)"
-AvailableChannelIds ${{parameters.NetEngLatestChannelId}},${{parameters.NetEngValidationChannelId}},${{parameters.NetDev5ChannelId}},${{parameters.NetDev6ChannelId}},${{parameters.GeneralTestingChannelId}},${{parameters.NETCoreToolingDevChannelId}},${{parameters.NETCoreToolingReleaseChannelId}},${{parameters.NETInternalToolingChannelId}},${{parameters.NETCoreExperimentalChannelId}},${{parameters.NetEngServicesIntChannelId}},${{parameters.NetEngServicesProdChannelId}},${{parameters.NetCoreSDK313xxChannelId}},${{parameters.NetCoreSDK313xxInternalChannelId}},${{parameters.NetCoreSDK314xxChannelId}},${{parameters.NetCoreSDK314xxInternalChannelId}},${{parameters.VS166ChannelId}},${{parameters.VS167ChannelId}},${{parameters.VS168ChannelId}},${{parameters.VSMasterChannelId}},${{parameters.VS169ChannelId}},${{parameters.VS1610ChannelId}}
- ${{ if and(le(parameters.publishingInfraVersion, 2), eq(parameters.inline, 'true')) }}:
- job:
displayName: Post-build Checks
displayName: NuGet Validation
dependsOn: setupMaestroVars
variables:
- name: TargetChannels
value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.TargetChannels'] ]
condition: eq( ${{ parameters.enableNugetValidation }}, 'true')
pool:
vmImage: 'windows-2019'
variables:
- name: AzDOProjectName
value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOProjectName'] ]
- name: AzDOPipelineId
value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOPipelineId'] ]
- name: AzDOBuildId
value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOBuildId'] ]
steps:
- task: DownloadBuildArtifacts@0
displayName: Download Package Artifacts
inputs:
buildType: specific
buildVersionToDownload: specific
project: $(AzDOProjectName)
pipeline: $(AzDOPipelineId)
buildId: $(AzDOBuildId)
artifactName: PackageArtifacts
checkDownloadedFiles: true
- task: PowerShell@2
displayName: Maestro Channels Consistency
displayName: Validate
inputs:
filePath: $(Build.SourcesDirectory)/eng/common/post-build/check-channel-consistency.ps1
arguments: -PromoteToChannels "$(TargetChannels)"
-AvailableChannelIds ${{parameters.NetEngLatestChannelId}},${{parameters.NetEngValidationChannelId}},${{parameters.NetDev5ChannelId}},${{parameters.NetDev6ChannelId}},${{parameters.GeneralTestingChannelId}},${{parameters.NETCoreToolingDevChannelId}},${{parameters.NETCoreToolingReleaseChannelId}},${{parameters.NETInternalToolingChannelId}},${{parameters.NETCoreExperimentalChannelId}},${{parameters.NetEngServicesIntChannelId}},${{parameters.NetEngServicesProdChannelId}},${{parameters.Net5Preview8ChannelId}},${{parameters.Net5RC1ChannelId}},${{parameters.Net5RC2ChannelId}},${{parameters.NetCoreSDK313xxChannelId}},${{parameters.NetCoreSDK313xxInternalChannelId}},${{parameters.NetCoreSDK314xxChannelId}},${{parameters.NetCoreSDK314xxInternalChannelId}},${{parameters.VS166ChannelId}},${{parameters.VS167ChannelId}},${{parameters.VS168ChannelId}},${{parameters.VSMasterChannelId}}
- job:
displayName: NuGet Validation
dependsOn: setupMaestroVars
condition: eq( ${{ parameters.enableNugetValidation }}, 'true')
pool:
vmImage: 'windows-2019'
variables:
- name: AzDOProjectName
value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOProjectName'] ]
- name: AzDOPipelineId
value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOPipelineId'] ]
- name: AzDOBuildId
value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOBuildId'] ]
steps:
- task: DownloadBuildArtifacts@0
displayName: Download Package Artifacts
inputs:
buildType: specific
buildVersionToDownload: specific
project: $(AzDOProjectName)
pipeline: $(AzDOPipelineId)
buildId: $(AzDOBuildId)
artifactName: PackageArtifacts
- task: PowerShell@2
displayName: Validate
inputs:
filePath: $(Build.SourcesDirectory)/eng/common/post-build/nuget-validation.ps1
arguments: -PackagesPath $(Build.ArtifactStagingDirectory)/PackageArtifacts/
-ToolDestinationPath $(Agent.BuildDirectory)/Extract/
- job:
displayName: Signing Validation
dependsOn: setupMaestroVars
condition: eq( ${{ parameters.enableSigningValidation }}, 'true')
variables:
- template: common-variables.yml
- name: AzDOProjectName
value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOProjectName'] ]
- name: AzDOPipelineId
value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOPipelineId'] ]
- name: AzDOBuildId
value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOBuildId'] ]
pool:
vmImage: 'windows-2019'
steps:
- ${{ if eq(parameters.useBuildManifest, true) }}:
filePath: $(Build.SourcesDirectory)/eng/common/post-build/nuget-validation.ps1
arguments: -PackagesPath $(Build.ArtifactStagingDirectory)/PackageArtifacts/
-ToolDestinationPath $(Agent.BuildDirectory)/Extract/
- job:
displayName: Signing Validation
dependsOn: setupMaestroVars
condition: and( eq( ${{ parameters.enableSigningValidation }}, 'true'), ne( variables['PostBuildSign'], 'true'))
variables:
- template: common-variables.yml
- name: AzDOProjectName
value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOProjectName'] ]
- name: AzDOPipelineId
value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOPipelineId'] ]
- name: AzDOBuildId
value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOBuildId'] ]
pool:
vmImage: 'windows-2019'
steps:
- task: DownloadBuildArtifacts@0
displayName: Download build manifest
displayName: Download Package Artifacts
inputs:
buildType: specific
buildVersionToDownload: specific
project: $(AzDOProjectName)
pipeline: $(AzDOPipelineId)
buildId: $(AzDOBuildId)
artifactName: BuildManifests
- task: DownloadBuildArtifacts@0
displayName: Download Package Artifacts
inputs:
buildType: specific
buildVersionToDownload: specific
project: $(AzDOProjectName)
pipeline: $(AzDOPipelineId)
buildId: $(AzDOBuildId)
artifactName: PackageArtifacts
# This is necessary whenever we want to publish/restore to an AzDO private feed
# Since sdk-task.ps1 tries to restore packages we need to do this authentication here
# otherwise it'll complain about accessing a private feed.
- task: NuGetAuthenticate@0
displayName: 'Authenticate to AzDO Feeds'
- task: PowerShell@2
displayName: Enable cross-org publishing
inputs:
filePath: eng\common\enable-cross-org-publishing.ps1
arguments: -token $(dn-bot-dnceng-artifact-feeds-rw)
# Signing validation will optionally work with the buildmanifest file which is downloaded from
# Azure DevOps above.
- task: PowerShell@2
displayName: Validate
inputs:
filePath: eng\common\sdk-task.ps1
arguments: -task SigningValidation -restore -msbuildEngine vs
/p:PackageBasePath='$(Build.ArtifactStagingDirectory)/PackageArtifacts'
/p:SignCheckExclusionsFile='$(Build.SourcesDirectory)/eng/SignCheckExclusionsFile.txt'
${{ parameters.signingValidationAdditionalParameters }}
- template: ../steps/publish-logs.yml
parameters:
StageLabel: 'Validation'
JobLabel: 'Signing'
- job:
displayName: SourceLink Validation
dependsOn: setupMaestroVars
condition: eq( ${{ parameters.enableSourceLinkValidation }}, 'true')
variables:
- template: common-variables.yml
- name: AzDOProjectName
value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOProjectName'] ]
- name: AzDOPipelineId
value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOPipelineId'] ]
- name: AzDOBuildId
value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOBuildId'] ]
pool:
vmImage: 'windows-2019'
steps:
- task: DownloadBuildArtifacts@0
displayName: Download Blob Artifacts
inputs:
buildType: specific
buildVersionToDownload: specific
project: $(AzDOProjectName)
pipeline: $(AzDOPipelineId)
buildId: $(AzDOBuildId)
artifactName: BlobArtifacts
- task: PowerShell@2
displayName: Validate
inputs:
filePath: $(Build.SourcesDirectory)/eng/common/post-build/sourcelink-validation.ps1
arguments: -InputPath $(Build.ArtifactStagingDirectory)/BlobArtifacts/
-ExtractPath $(Agent.BuildDirectory)/Extract/
-GHRepoName $(Build.Repository.Name)
-GHCommit $(Build.SourceVersion)
-SourcelinkCliVersion $(SourceLinkCLIVersion)
continueOnError: true
- template: /eng/common/templates/job/execute-sdl.yml
parameters:
enable: ${{ parameters.SDLValidationParameters.enable }}
artifactName: PackageArtifacts
checkDownloadedFiles: true
itemPattern: |
**
!**/Microsoft.SourceBuild.Intermediate.*.nupkg
# This is necessary whenever we want to publish/restore to an AzDO private feed
# Since sdk-task.ps1 tries to restore packages we need to do this authentication here
# otherwise it'll complain about accessing a private feed.
- task: NuGetAuthenticate@0
displayName: 'Authenticate to AzDO Feeds'
- task: PowerShell@2
displayName: Enable cross-org publishing
inputs:
filePath: eng\common\enable-cross-org-publishing.ps1
arguments: -token $(dn-bot-dnceng-artifact-feeds-rw)
# Signing validation will optionally work with the buildmanifest file which is downloaded from
# Azure DevOps above.
- task: PowerShell@2
displayName: Validate
inputs:
filePath: eng\common\sdk-task.ps1
arguments: -task SigningValidation -restore -msbuildEngine vs
/p:PackageBasePath='$(Build.ArtifactStagingDirectory)/PackageArtifacts'
/p:SignCheckExclusionsFile='$(Build.SourcesDirectory)/eng/SignCheckExclusionsFile.txt'
${{ parameters.signingValidationAdditionalParameters }}
- template: ../steps/publish-logs.yml
parameters:
StageLabel: 'Validation'
JobLabel: 'Signing'
- job:
displayName: SourceLink Validation
dependsOn: setupMaestroVars
additionalParameters: ${{ parameters.SDLValidationParameters.params }}
continueOnError: ${{ parameters.SDLValidationParameters.continueOnError }}
artifactNames: ${{ parameters.SDLValidationParameters.artifactNames }}
downloadArtifacts: ${{ parameters.SDLValidationParameters.downloadArtifacts }}
condition: eq( ${{ parameters.enableSourceLinkValidation }}, 'true')
variables:
- template: common-variables.yml
- name: AzDOProjectName
value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOProjectName'] ]
- name: AzDOPipelineId
value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOPipelineId'] ]
- name: AzDOBuildId
value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOBuildId'] ]
pool:
vmImage: 'windows-2019'
steps:
- task: DownloadBuildArtifacts@0
displayName: Download Blob Artifacts
inputs:
buildType: specific
buildVersionToDownload: specific
project: $(AzDOProjectName)
pipeline: $(AzDOPipelineId)
buildId: $(AzDOBuildId)
artifactName: BlobArtifacts
checkDownloadedFiles: true
- task: PowerShell@2
displayName: Validate
inputs:
filePath: $(Build.SourcesDirectory)/eng/common/post-build/sourcelink-validation.ps1
arguments: -InputPath $(Build.ArtifactStagingDirectory)/BlobArtifacts/
-ExtractPath $(Agent.BuildDirectory)/Extract/
-GHRepoName $(Build.Repository.Name)
-GHCommit $(Build.SourceVersion)
-SourcelinkCliVersion $(SourceLinkCLIVersion)
continueOnError: true
- template: /eng/common/templates/job/execute-sdl.yml
parameters:
enable: ${{ parameters.SDLValidationParameters.enable }}
dependsOn: setupMaestroVars
additionalParameters: ${{ parameters.SDLValidationParameters.params }}
continueOnError: ${{ parameters.SDLValidationParameters.continueOnError }}
artifactNames: ${{ parameters.SDLValidationParameters.artifactNames }}
downloadArtifacts: ${{ parameters.SDLValidationParameters.downloadArtifacts }}
- ${{ if or(ge(parameters.publishingInfraVersion, 3), eq(parameters.inline, 'false')) }}:
- stage: publish_using_darc
dependsOn: Validate
${{ if or(eq(parameters.enableNugetValidation, 'true'), eq(parameters.enableSigningValidation, 'true'), eq(parameters.enableSourceLinkValidation, 'true'), eq(parameters.SDLValidationParameters.enable, 'true')) }}:
dependsOn: ${{ parameters.publishDependsOn }}
${{ if and(ne(parameters.enableNugetValidation, 'true'), ne(parameters.enableSigningValidation, 'true'), ne(parameters.enableSourceLinkValidation, 'true'), ne(parameters.SDLValidationParameters.enable, 'true')) }}:
dependsOn: ${{ parameters.validateDependsOn }}
displayName: Publish using Darc
variables:
- template: common-variables.yml
@ -249,6 +247,7 @@ stages:
- job:
displayName: Publish Using Darc
dependsOn: setupMaestroVars
timeoutInMinutes: 120
variables:
- name: BARBuildId
value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.BARBuildId'] ]
@ -265,6 +264,8 @@ stages:
-MaestroToken '$(MaestroApiAccessToken)'
-WaitPublishingFinish ${{ parameters.waitPublishingFinish }}
-PublishInstallersAndChecksums ${{ parameters.publishInstallersAndChecksums }}
-ArtifactsPublishingAdditionalParameters '${{ parameters.artifactsPublishingAdditionalParameters }}'
-SymbolPublishingAdditionalParameters '${{ parameters.symbolPublishingAdditionalParameters }}'
- ${{ if and(le(parameters.publishingInfraVersion, 2), eq(parameters.inline, 'true')) }}:
- template: \eng\common\templates\post-build\channels\generic-public-channel.yml
@ -299,54 +300,6 @@ stages:
shippingFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet6/nuget/v3/index.json'
symbolsFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet6-symbols/nuget/v3/index.json'
- template: \eng\common\templates\post-build\channels\generic-internal-channel.yml
parameters:
BARBuildId: ${{ parameters.BARBuildId }}
PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }}
artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }}
dependsOn: ${{ parameters.publishDependsOn }}
publishInstallersAndChecksums: ${{ parameters.publishInstallersAndChecksums }}
symbolPublishingAdditionalParameters: ${{ parameters.symbolPublishingAdditionalParameters }}
stageName: 'Net5_Preview8_Publish'
channelName: '.NET 5 Preview 8'
akaMSChannelName: 'net5/preview8'
channelId: ${{ parameters.Net5Preview8ChannelId }}
transportFeed: 'https://pkgs.dev.azure.com/dnceng/internal/_packaging/dotnet5-internal-transport/nuget/v3/index.json'
shippingFeed: 'https://pkgs.dev.azure.com/dnceng/internal/_packaging/dotnet5-internal/nuget/v3/index.json'
symbolsFeed: 'https://pkgs.dev.azure.com/dnceng/internal/_packaging/dotnet5-internal-symbols/nuget/v3/index.json'
- template: \eng\common\templates\post-build\channels\generic-public-channel.yml
parameters:
BARBuildId: ${{ parameters.BARBuildId }}
PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }}
artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }}
dependsOn: ${{ parameters.publishDependsOn }}
publishInstallersAndChecksums: ${{ parameters.publishInstallersAndChecksums }}
symbolPublishingAdditionalParameters: ${{ parameters.symbolPublishingAdditionalParameters }}
stageName: 'Net5_RC1_Publish'
channelName: '.NET 5 RC 1'
akaMSChannelName: 'net5/rc1'
channelId: ${{ parameters.Net5RC1ChannelId }}
transportFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet5-transport/nuget/v3/index.json'
shippingFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet5/nuget/v3/index.json'
symbolsFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet5-symbols/nuget/v3/index.json'
- template: \eng\common\templates\post-build\channels\generic-public-channel.yml
parameters:
BARBuildId: ${{ parameters.BARBuildId }}
PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }}
artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }}
dependsOn: ${{ parameters.publishDependsOn }}
publishInstallersAndChecksums: ${{ parameters.publishInstallersAndChecksums }}
symbolPublishingAdditionalParameters: ${{ parameters.symbolPublishingAdditionalParameters }}
stageName: 'Net5_RC2_Publish'
channelName: '.NET 5 RC 2'
akaMSChannelName: 'net5/rc2'
channelId: ${{ parameters.Net5RC2ChannelId }}
transportFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet5-transport/nuget/v3/index.json'
shippingFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet5/nuget/v3/index.json'
symbolsFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet5-symbols/nuget/v3/index.json'
- template: \eng\common\templates\post-build\channels\generic-public-channel.yml
parameters:
BARBuildId: ${{ parameters.BARBuildId }}
@ -604,3 +557,33 @@ stages:
transportFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools-transport/nuget/v3/index.json'
shippingFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools/nuget/v3/index.json'
symbolsFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools-symbols/nuget/v3/index.json'
- template: \eng\common\templates\post-build\channels\generic-public-channel.yml
parameters:
BARBuildId: ${{ parameters.BARBuildId }}
PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }}
artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }}
dependsOn: ${{ parameters.publishDependsOn }}
publishInstallersAndChecksums: ${{ parameters.publishInstallersAndChecksums }}
symbolPublishingAdditionalParameters: ${{ parameters.symbolPublishingAdditionalParameters }}
stageName: 'VS_16_9_Publishing'
channelName: 'VS 16.9'
channelId: ${{ parameters.VS169ChannelId }}
transportFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools-transport/nuget/v3/index.json'
shippingFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools/nuget/v3/index.json'
symbolsFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools-symbols/nuget/v3/index.json'
- template: \eng\common\templates\post-build\channels\generic-public-channel.yml
parameters:
BARBuildId: ${{ parameters.BARBuildId }}
PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }}
artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }}
dependsOn: ${{ parameters.publishDependsOn }}
publishInstallersAndChecksums: ${{ parameters.publishInstallersAndChecksums }}
symbolPublishingAdditionalParameters: ${{ parameters.symbolPublishingAdditionalParameters }}
stageName: 'VS_16_10_Publishing'
channelName: 'VS 16.10'
channelId: ${{ parameters.VS1610ChannelId }}
transportFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools-transport/nuget/v3/index.json'
shippingFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools/nuget/v3/index.json'
symbolsFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools-symbols/nuget/v3/index.json'

1
eng/common/templates/post-build/setup-maestro-vars.yml

@ -18,6 +18,7 @@ jobs:
inputs:
buildType: current
artifactName: ReleaseConfigs
checkDownloadedFiles: true
- task: PowerShell@2
name: setReleaseVars

4
eng/common/templates/steps/send-to-helix.yml

@ -18,8 +18,8 @@ parameters:
XUnitRuntimeTargetFramework: '' # optional -- framework to use for the xUnit console runner
XUnitRunnerVersion: '' # optional -- version of the xUnit nuget package you wish to use on Helix; required for XUnitProjects
IncludeDotNetCli: false # optional -- true will download a version of the .NET CLI onto the Helix machine as a correlation payload; requires DotNetCliPackageType and DotNetCliVersion
DotNetCliPackageType: '' # optional -- either 'sdk', 'runtime' or 'aspnetcore-runtime'; determines whether the sdk or runtime will be sent to Helix; see https://raw.githubusercontent.com/dotnet/core/master/release-notes/releases-index.json
DotNetCliVersion: '' # optional -- version of the CLI to send to Helix; based on this: https://raw.githubusercontent.com/dotnet/core/master/release-notes/releases-index.json
DotNetCliPackageType: '' # optional -- either 'sdk', 'runtime' or 'aspnetcore-runtime'; determines whether the sdk or runtime will be sent to Helix; see https://raw.githubusercontent.com/dotnet/core/main/release-notes/releases-index.json
DotNetCliVersion: '' # optional -- version of the CLI to send to Helix; based on this: https://raw.githubusercontent.com/dotnet/core/main/release-notes/releases-index.json
EnableXUnitReporter: false # optional -- true enables XUnit result reporting to Mission Control
WaitForWorkItemCompletion: true # optional -- true will make the task wait until work items have been completed and fail the build if work items fail. False is "fire and forget."
IsExternal: false # [DEPRECATED] -- doesn't do anything, jobs are external if HelixAccessToken is empty and Creator is set

7
eng/common/templates/steps/source-build.yml

@ -34,9 +34,14 @@ steps:
targetRidArgs='/p:TargetRid=${{ parameters.platform.targetRID }}'
fi
publishArgs=
if [ '${{ parameters.platform.skipPublishValidation }}' != 'true' ]; then
publishArgs='--publish'
fi
${{ coalesce(parameters.platform.buildScript, './build.sh') }} --ci \
--configuration $buildConfig \
--restore --build --pack --publish \
--restore --build --pack $publishArgs -bl \
$officialBuildArgs \
$targetRidArgs \
/p:SourceBuildNonPortable=${{ parameters.platform.nonPortable }} \

164
eng/common/tools.ps1

@ -48,6 +48,9 @@
# True to use global NuGet cache instead of restoring packages to repository-local directory.
[bool]$useGlobalNuGetCache = if (Test-Path variable:useGlobalNuGetCache) { $useGlobalNuGetCache } else { !$ci }
# True to exclude prerelease versions Visual Studio during build
[bool]$excludePrereleaseVS = if (Test-Path variable:excludePrereleaseVS) { $excludePrereleaseVS } else { $false }
# An array of names of processes to stop on script exit if prepareMachine is true.
$processesToStopOnExit = if (Test-Path variable:processesToStopOnExit) { $processesToStopOnExit } else { @('msbuild', 'dotnet', 'vbcscompiler') }
@ -57,15 +60,11 @@ set-strictmode -version 2.0
$ErrorActionPreference = 'Stop'
[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12
# If specified, provides an alternate path for getting .NET Core SDKs and Runtimes. This script will still try public sources first.
# If specifies, provides an alternate path for getting .NET Core SDKs and Runtimes. This script will still try public sources first.
[string]$runtimeSourceFeed = if (Test-Path variable:runtimeSourceFeed) { $runtimeSourceFeed } else { $null }
# Base-64 encoded SAS token that has permission to storage container described by $runtimeSourceFeed
[string]$runtimeSourceFeedKey = if (Test-Path variable:runtimeSourceFeedKey) { $runtimeSourceFeedKey } else { $null }
# If false, use copy of dotnet-install from /eng/common/dotnet-install-scripts (for custom behaviors).
# otherwise will fetch from public location.
[bool]$useDefaultDotnetInstall = if (Test-Path variable:useDefaultDotnetInstall) { $useDefaultDotnetInstall } else { $false }
function Create-Directory ([string[]] $path) {
New-Item -Path $path -Force -ItemType 'Directory' | Out-Null
}
@ -145,7 +144,7 @@ function InitializeDotNetCli([bool]$install, [bool]$createSdkLocationFile) {
# Use dotnet installation specified in DOTNET_INSTALL_DIR if it contains the required SDK version,
# otherwise install the dotnet CLI and SDK to repo local .dotnet directory to avoid potential permission issues.
if ((-not $globalJsonHasRuntimes) -and ($env:DOTNET_INSTALL_DIR -ne $null) -and (Test-Path(Join-Path $env:DOTNET_INSTALL_DIR "sdk\$dotnetSdkVersion"))) {
if ((-not $globalJsonHasRuntimes) -and (-not [string]::IsNullOrEmpty($env:DOTNET_INSTALL_DIR)) -and (Test-Path(Join-Path $env:DOTNET_INSTALL_DIR "sdk\$dotnetSdkVersion"))) {
$dotnetRoot = $env:DOTNET_INSTALL_DIR
} else {
$dotnetRoot = Join-Path $RepoRoot '.dotnet'
@ -173,7 +172,7 @@ function InitializeDotNetCli([bool]$install, [bool]$createSdkLocationFile) {
Set-Content -Path $sdkCacheFileTemp -Value $dotnetRoot
try {
Rename-Item -Force -Path $sdkCacheFileTemp 'sdk.txt'
Move-Item -Force $sdkCacheFileTemp (Join-Path $ToolsetDir 'sdk.txt')
} catch {
# Somebody beat us
Remove-Item -Path $sdkCacheFileTemp
@ -194,49 +193,44 @@ function InitializeDotNetCli([bool]$install, [bool]$createSdkLocationFile) {
return $global:_DotNetInstallDir = $dotnetRoot
}
function GetDotNetInstallScript([string] $dotnetRoot) {
$installScript = Join-Path $dotnetRoot 'dotnet-install.ps1'
if (!(Test-Path $installScript)) {
create-directory $dotnetroot
function Retry($downloadBlock, $maxRetries = 5) {
$retries = 1
if ($useDefaultDotnetInstall)
{
$progresspreference = 'silentlycontinue' # don't display the console progress ui - it's a huge perf hit
while($true) {
try {
& $downloadBlock
break
}
catch {
Write-PipelineTelemetryError -Category 'InitializeToolset' -Message $_
}
$maxretries = 5
$retries = 1
if (++$retries -le $maxRetries) {
$delayInSeconds = [math]::Pow(2, $retries) - 1 # Exponential backoff
Write-Host "Retrying. Waiting for $delayInSeconds seconds before next attempt ($retries of $maxRetries)."
Start-Sleep -Seconds $delayInSeconds
}
else {
Write-PipelineTelemetryError -Category 'InitializeToolset' -Message "Unable to download file in $maxRetries attempts."
break
}
$uri = "https://dot.net/$dotnetinstallscriptversion/dotnet-install.ps1"
}
}
while($true) {
try {
write-host "get $uri"
invoke-webrequest $uri -outfile $installscript
break
}
catch {
write-host "failed to download '$uri'"
write-error $_.exception.message -erroraction continue
}
function GetDotNetInstallScript([string] $dotnetRoot) {
$installScript = Join-Path $dotnetRoot 'dotnet-install.ps1'
if (!(Test-Path $installScript)) {
Create-Directory $dotnetRoot
$ProgressPreference = 'SilentlyContinue' # Don't display the console progress UI - it's a huge perf hit
$uri = "https://dot.net/$dotnetInstallScriptVersion/dotnet-install.ps1"
if (++$retries -le $maxretries) {
$delayinseconds = [math]::pow(2, $retries) - 1 # exponential backoff
write-host "retrying. waiting for $delayinseconds seconds before next attempt ($retries of $maxretries)."
start-sleep -seconds $delayinseconds
}
else {
throw "unable to download file in $maxretries attempts."
}
}
}
else
{
# Use a special version of the script from eng/common that understands the existence of a "productVersion.txt" in a dotnet path.
# See https://github.com/dotnet/arcade/issues/6047 for details
$engCommonCopy = Resolve-Path (Join-Path $PSScriptRoot 'dotnet-install-scripts\dotnet-install.ps1')
Copy-Item $engCommonCopy -Destination $installScript -Force
}
Retry({
Write-Host "GET $uri"
Invoke-WebRequest $uri -OutFile $installScript
})
}
return $installScript
}
@ -318,8 +312,8 @@ function InitializeVisualStudioMSBuild([bool]$install, [object]$vsRequirements =
# If the version of msbuild is going to be xcopied,
# use this version. Version matches a package here:
# https://dev.azure.com/dnceng/public/_packaging?_a=package&feed=dotnet-eng&package=RoslynTools.MSBuild&protocolType=NuGet&version=16.8.0-preview3&view=overview
$defaultXCopyMSBuildVersion = '16.8.0-preview3'
# https://dev.azure.com/dnceng/public/_packaging?_a=package&feed=dotnet-eng&package=RoslynTools.MSBuild&protocolType=NuGet&version=16.10.0-preview2&view=overview
$defaultXCopyMSBuildVersion = '16.10.0-preview2'
if (!$vsRequirements) { $vsRequirements = $GlobalJson.tools.vs }
$vsMinVersionStr = if ($vsRequirements.version) { $vsRequirements.version } else { $vsMinVersionReqdStr }
@ -413,9 +407,13 @@ function InitializeXCopyMSBuild([string]$packageVersion, [bool]$install) {
}
Create-Directory $packageDir
Write-Host "Downloading $packageName $packageVersion"
$ProgressPreference = 'SilentlyContinue' # Don't display the console progress UI - it's a huge perf hit
Invoke-WebRequest "https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-eng/nuget/v3/flat2/$packageName/$packageVersion/$packageName.$packageVersion.nupkg" -OutFile $packagePath
Retry({
Invoke-WebRequest "https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-eng/nuget/v3/flat2/$packageName/$packageVersion/$packageName.$packageVersion.nupkg" -OutFile $packagePath
})
Unzip $packagePath $packageDir
}
@ -452,16 +450,17 @@ function LocateVisualStudio([object]$vsRequirements = $null){
if (!(Test-Path $vsWhereExe)) {
Create-Directory $vsWhereDir
Write-Host 'Downloading vswhere'
try {
Retry({
Invoke-WebRequest "https://netcorenativeassets.blob.core.windows.net/resource-packages/external/windows/vswhere/$vswhereVersion/vswhere.exe" -OutFile $vswhereExe
}
catch {
Write-PipelineTelemetryError -Category 'InitializeToolset' -Message $_
}
})
}
if (!$vsRequirements) { $vsRequirements = $GlobalJson.tools.vs }
$args = @('-latest', '-prerelease', '-format', 'json', '-requires', 'Microsoft.Component.MSBuild', '-products', '*')
$args = @('-latest', '-format', 'json', '-requires', 'Microsoft.Component.MSBuild', '-products', '*')
if (!$excludePrereleaseVS) {
$args += '-prerelease'
}
if (Get-Member -InputObject $vsRequirements -Name 'version') {
$args += '-version'
@ -487,7 +486,13 @@ function LocateVisualStudio([object]$vsRequirements = $null){
function InitializeBuildTool() {
if (Test-Path variable:global:_BuildTool) {
return $global:_BuildTool
# If the requested msbuild parameters do not match, clear the cached variables.
if($global:_BuildTool.Contains('ExcludePrereleaseVS') -and $global:_BuildTool.ExcludePrereleaseVS -ne $excludePrereleaseVS) {
Remove-Item variable:global:_BuildTool
Remove-Item variable:global:_MSBuildExe
} else {
return $global:_BuildTool
}
}
if (-not $msbuildEngine) {
@ -506,7 +511,7 @@ function InitializeBuildTool() {
ExitWithExitCode 1
}
$dotnetPath = Join-Path $dotnetRoot (GetExecutableFileName 'dotnet')
$buildTool = @{ Path = $dotnetPath; Command = 'msbuild'; Tool = 'dotnet'; Framework = 'netcoreapp2.1' }
$buildTool = @{ Path = $dotnetPath; Command = 'msbuild'; Tool = 'dotnet'; Framework = 'netcoreapp3.1' }
} elseif ($msbuildEngine -eq "vs") {
try {
$msbuildPath = InitializeVisualStudioMSBuild -install:$restore
@ -515,7 +520,7 @@ function InitializeBuildTool() {
ExitWithExitCode 1
}
$buildTool = @{ Path = $msbuildPath; Command = ""; Tool = "vs"; Framework = "net472" }
$buildTool = @{ Path = $msbuildPath; Command = ""; Tool = "vs"; Framework = "net472"; ExcludePrereleaseVS = $excludePrereleaseVS }
} else {
Write-PipelineTelemetryError -Category 'InitializeToolset' -Message "Unexpected value of -msbuildEngine: '$msbuildEngine'."
ExitWithExitCode 1
@ -540,7 +545,7 @@ function GetDefaultMSBuildEngine() {
function GetNuGetPackageCachePath() {
if ($env:NUGET_PACKAGES -eq $null) {
# Use local cache on CI to ensure deterministic build.
# Use local cache on CI to ensure deterministic build.
# Avoid using the http cache as workaround for https://github.com/NuGet/Home/issues/3116
# use global cache in dev builds to avoid cost of downloading packages.
# For directory normalization, see also: https://github.com/NuGet/Home/issues/7968
@ -642,9 +647,26 @@ function MSBuild() {
}
$toolsetBuildProject = InitializeToolset
$path = Split-Path -parent $toolsetBuildProject
$path = Join-Path $path (Join-Path $buildTool.Framework 'Microsoft.DotNet.Arcade.Sdk.dll')
$args += "/logger:$path"
$basePath = Split-Path -parent $toolsetBuildProject
$possiblePaths = @(
# new scripts need to work with old packages, so we need to look for the old names/versions
(Join-Path $basePath (Join-Path $buildTool.Framework 'Microsoft.DotNet.ArcadeLogging.dll')),
(Join-Path $basePath (Join-Path $buildTool.Framework 'Microsoft.DotNet.Arcade.Sdk.dll')),
(Join-Path $basePath (Join-Path netcoreapp2.1 'Microsoft.DotNet.ArcadeLogging.dll')),
(Join-Path $basePath (Join-Path netcoreapp2.1 'Microsoft.DotNet.Arcade.Sdk.dll'))
)
$selectedPath = $null
foreach ($path in $possiblePaths) {
if (Test-Path $path -PathType Leaf) {
$selectedPath = $path
break
}
}
if (-not $selectedPath) {
Write-PipelineTelemetryError -Category 'Build' -Message 'Unable to find arcade sdk logger assembly.'
ExitWithExitCode 1
}
$args += "/logger:$selectedPath"
}
MSBuild-Core @args
@ -680,7 +702,10 @@ function MSBuild-Core() {
}
foreach ($arg in $args) {
if ($arg -ne $null -and $arg.Trim() -ne "") {
if ($null -ne $arg -and $arg.Trim() -ne "") {
if ($arg.EndsWith('\')) {
$arg = $arg + "\"
}
$cmdArgs += " `"$arg`""
}
}
@ -690,14 +715,23 @@ function MSBuild-Core() {
$exitCode = Exec-Process $buildTool.Path $cmdArgs
if ($exitCode -ne 0) {
Write-PipelineTelemetryError -Category 'Build' -Message 'Build failed.'
# We should not Write-PipelineTaskError here because that message shows up in the build summary
# The build already logged an error, that's the reason it failed. Producing an error here only adds noise.
Write-Host "Build failed with exit code $exitCode. Check errors above." -ForegroundColor Red
$buildLog = GetMSBuildBinaryLogCommandLineArgument $args
if ($buildLog -ne $null) {
if ($null -ne $buildLog) {
Write-Host "See log: $buildLog" -ForegroundColor DarkGray
}
ExitWithExitCode $exitCode
if ($ci) {
Write-PipelineSetResult -Result "Failed" -Message "msbuild execution failed."
# Exiting with an exit code causes the azure pipelines task to log yet another "noise" error
# The above Write-PipelineSetResult will cause the task to be marked as failure without adding yet another error
ExitWithExitCode 0
} else {
ExitWithExitCode $exitCode
}
}
}
@ -743,7 +777,7 @@ function Get-Darc($version) {
. $PSScriptRoot\pipeline-logging-functions.ps1
$RepoRoot = Resolve-Path (Join-Path $PSScriptRoot '..\..')
$RepoRoot = Resolve-Path (Join-Path $PSScriptRoot '..\..\')
$EngRoot = Resolve-Path (Join-Path $PSScriptRoot '..')
$ArtifactsDir = Join-Path $RepoRoot 'artifacts'
$ToolsetDir = Join-Path $ArtifactsDir 'toolset'

104
eng/common/tools.sh

@ -68,10 +68,6 @@ fi
runtime_source_feed=${runtime_source_feed:-''}
runtime_source_feed_key=${runtime_source_feed_key:-''}
# Determines if dotnet-install.sh comes from the eng/common folder or the internet
# (default = public version)
use_default_dotnet_install=${use_default_dotnet_install:-false}
# Resolve any symlinks in the given path.
function ResolvePath {
local path=$1
@ -93,16 +89,16 @@ function ResolvePath {
function ReadGlobalVersion {
local key=$1
local line=$(awk "/$key/ {print; exit}" "$global_json_file")
local pattern="\"$key\" *: *\"(.*)\""
if command -v jq &> /dev/null; then
_ReadGlobalVersion="$(jq -r ".[] | select(has(\"$key\")) | .\"$key\"" "$global_json_file")"
elif [[ "$(cat "$global_json_file")" =~ \"$key\"[[:space:]\:]*\"([^\"]+) ]]; then
_ReadGlobalVersion=${BASH_REMATCH[1]}
fi
if [[ ! $line =~ $pattern ]]; then
if [[ -z "$_ReadGlobalVersion" ]]; then
Write-PipelineTelemetryError -category 'Build' "Error: Cannot find \"$key\" in $global_json_file"
ExitWithExitCode 1
fi
# return value
_ReadGlobalVersion=${BASH_REMATCH[1]}
}
function InitializeDotNetCli {
@ -253,7 +249,7 @@ function with_retries {
return 0
fi
timeout=$((2**$retries-1))
timeout=$((3**$retries-1))
echo "Failed to execute '$@'. Waiting $timeout seconds before next attempt ($retries out of $maxRetries)." 1>&2
sleep $timeout
done
@ -271,30 +267,31 @@ function GetDotNetInstallScript {
if [[ ! -a "$install_script" ]]; then
mkdir -p "$root"
if [[ "$use_default_dotnet_install" == true ]]; then
echo "Downloading '$install_script_url'"
# Use curl if available, otherwise use wget
if command -v curl > /dev/null; then
with_retries curl "$install_script_url" -sSL --retry 10 --create-dirs -o "$install_script" || {
local exit_code=$?
Write-PipelineTelemetryError -category 'InitializeToolset' "Failed to acquire dotnet install script (exit code '$exit_code')."
ExitWithExitCode $exit_code
}
else
with_retries wget -v -O "$install_script" "$install_script_url" || {
echo "Downloading '$install_script_url'"
# Use curl if available, otherwise use wget
if command -v curl > /dev/null; then
# first, try directly, if this fails we will retry with verbose logging
curl "$install_script_url" -sSL --retry 10 --create-dirs -o "$install_script" || {
if command -v openssl &> /dev/null; then
echo "Curl failed; dumping some information about dotnet.microsoft.com for later investigation"
echo | openssl s_client -showcerts -servername dotnet.microsoft.com -connect dotnet.microsoft.com:443
fi
echo "Will now retry the same URL with verbose logging."
with_retries curl "$install_script_url" -sSL --verbose --retry 10 --create-dirs -o "$install_script" || {
local exit_code=$?
Write-PipelineTelemetryError -category 'InitializeToolset' "Failed to acquire dotnet install script (exit code '$exit_code')."
ExitWithExitCode $exit_code
}
fi
}
else
# Use a special version of the script from eng/common that understands the existence of a "productVersion.txt" in a dotnet path.
# See https://github.com/dotnet/arcade/issues/6047 for details
cp $repo_root/eng/common/dotnet-install-scripts/dotnet-install.sh $install_script
with_retries wget -v -O "$install_script" "$install_script_url" || {
local exit_code=$?
Write-PipelineTelemetryError -category 'InitializeToolset' "Failed to acquire dotnet install script (exit code '$exit_code')."
ExitWithExitCode $exit_code
}
fi
fi
# return value
_GetDotNetInstallScript="$install_script"
}
@ -309,7 +306,7 @@ function InitializeBuildTool {
# return values
_InitializeBuildTool="$_InitializeDotNetCli/dotnet"
_InitializeBuildToolCommand="msbuild"
_InitializeBuildToolFramework="netcoreapp2.1"
_InitializeBuildToolFramework="netcoreapp3.1"
}
# Set RestoreNoCache as a workaround for https://github.com/NuGet/Home/issues/3116
@ -416,8 +413,24 @@ function MSBuild {
fi
local toolset_dir="${_InitializeToolset%/*}"
local logger_path="$toolset_dir/$_InitializeBuildToolFramework/Microsoft.DotNet.Arcade.Sdk.dll"
args=( "${args[@]}" "-logger:$logger_path" )
# new scripts need to work with old packages, so we need to look for the old names/versions
local selectedPath=
local possiblePaths=()
possiblePaths+=( "$toolset_dir/$_InitializeBuildToolFramework/Microsoft.DotNet.ArcadeLogging.dll" )
possiblePaths+=( "$toolset_dir/$_InitializeBuildToolFramework/Microsoft.DotNet.Arcade.Sdk.dll" )
possiblePaths+=( "$toolset_dir/netcoreapp2.1/Microsoft.DotNet.ArcadeLogging.dll" )
possiblePaths+=( "$toolset_dir/netcoreapp2.1/Microsoft.DotNet.Arcade.Sdk.dll" )
for path in "${possiblePaths[@]}"; do
if [[ -f $path ]]; then
selectedPath=$path
break
fi
done
if [[ -z "$selectedPath" ]]; then
Write-PipelineTelemetryError -category 'Build' "Unable to find arcade sdk logger assembly."
ExitWithExitCode 1
fi
args+=( "-logger:$selectedPath" )
fi
MSBuild-Core ${args[@]}
@ -448,8 +461,17 @@ function MSBuild-Core {
"$_InitializeBuildTool" "$@" || {
local exit_code=$?
Write-PipelineTaskError "Build failed (exit code '$exit_code')."
ExitWithExitCode $exit_code
# We should not Write-PipelineTaskError here because that message shows up in the build summary
# The build already logged an error, that's the reason it failed. Producing an error here only adds noise.
echo "Build failed with exit code $exit_code. Check errors above."
if [[ "$ci" == "true" ]]; then
Write-PipelineSetResult -result "Failed" -message "msbuild execution failed."
# Exiting with an exit code causes the azure pipelines task to log yet another "noise" error
# The above Write-PipelineSetResult will cause the task to be marked as failure without adding yet another error
ExitWithExitCode 0
else
ExitWithExitCode $exit_code
fi
}
}
@ -463,23 +485,27 @@ _script_dir=`dirname "$_ResolvePath"`
eng_root=`cd -P "$_script_dir/.." && pwd`
repo_root=`cd -P "$_script_dir/../.." && pwd`
artifacts_dir="$repo_root/artifacts"
repo_root="${repo_root}/"
artifacts_dir="${repo_root}artifacts"
toolset_dir="$artifacts_dir/toolset"
tools_dir="$repo_root/.tools"
tools_dir="${repo_root}.tools"
log_dir="$artifacts_dir/log/$configuration"
temp_dir="$artifacts_dir/tmp/$configuration"
global_json_file="$repo_root/global.json"
global_json_file="${repo_root}global.json"
# determine if global.json contains a "runtimes" entry
global_json_has_runtimes=false
dotnetlocal_key=$(awk "/runtimes/ {print; exit}" "$global_json_file") || true
if [[ -n "$dotnetlocal_key" ]]; then
if command -v jq &> /dev/null; then
if jq -er '. | select(has("runtimes"))' "$global_json_file" &> /dev/null; then
global_json_has_runtimes=true
fi
elif [[ "$(cat "$global_json_file")" =~ \"runtimes\"[[:space:]\:]*\{ ]]; then
global_json_has_runtimes=true
fi
# HOME may not be defined in some scenarios, but it is required by NuGet
if [[ -z $HOME ]]; then
export HOME="$repo_root/artifacts/.home/"
export HOME="${repo_root}artifacts/.home/"
mkdir -p "$HOME"
fi

9
global.json

@ -1,17 +1,18 @@
{
"tools": {
"dotnet": "5.0.301",
"dotnet": "6.0.100-rc.1.21458.32",
"runtimes": {
"aspnetcore": [
"2.1.28",
"3.1.16"
"3.1.16",
"5.0.7"
]
}
},
"msbuild-sdks": {
"Microsoft.DotNet.Arcade.Sdk": "5.0.0-beta.20510.1",
"Microsoft.DotNet.Helix.Sdk": "5.0.0-beta.20510.1"
"Microsoft.DotNet.Arcade.Sdk": "6.0.0-beta.21316.1",
"Microsoft.DotNet.Helix.Sdk": "6.0.0-beta.21316.1"
}
}

Loading…
Cancel
Save