Browse Source

[main] Update dependencies from dotnet/arcade (#1141)

[main] Update dependencies from dotnet/arcade


 - Move build tooling to .NET 6 GA to avoid RC1 static web asset build issue.
pull/1218/head
dotnet-maestro[bot] 5 years ago
committed by GitHub
parent
commit
995c669219
No known key found for this signature in database GPG Key ID: 4AEE18F83AFDEB23
  1. 20
      eng/Version.Details.xml
  2. 4
      eng/common/build.sh
  3. 74
      eng/common/cross/build-rootfs.sh
  4. 9
      eng/common/cross/toolchain.cmake
  5. 2
      eng/common/init-tools-native.sh
  6. 1
      eng/common/internal/Tools.csproj
  7. 1
      eng/common/msbuild.ps1
  8. 8
      eng/common/native/common-library.sh
  9. 102
      eng/common/native/init-compiler.sh
  10. 30
      eng/common/post-build/sourcelink-validation.ps1
  11. 95
      eng/common/post-build/symbols-validation.ps1
  12. 3
      eng/common/sdk-task.ps1
  13. 6
      eng/common/sdl/configure-sdl-tool.ps1
  14. 6
      eng/common/templates/job/execute-sdl.yml
  15. 2
      eng/common/templates/job/job.yml
  16. 5
      eng/common/templates/job/onelocbuild.yml
  17. 26
      eng/common/templates/job/publish-build-assets.yml
  18. 15
      eng/common/templates/job/source-build.yml
  19. 31
      eng/common/templates/job/source-index-stage1.yml
  20. 4
      eng/common/templates/jobs/jobs.yml
  21. 130
      eng/common/templates/phases/base.yml
  22. 52
      eng/common/templates/phases/publish-build-assets.yml
  23. 3
      eng/common/templates/steps/send-to-helix.yml
  24. 35
      eng/common/templates/steps/source-build.yml
  25. 42
      eng/common/tools.ps1
  26. 16
      eng/common/tools.sh
  27. 4
      global.json

20
eng/Version.Details.xml

@ -3,25 +3,25 @@
<ProductDependencies>
</ProductDependencies>
<ToolsetDependencies>
<Dependency Name="Microsoft.DotNet.Arcade.Sdk" Version="6.0.0-beta.21379.2">
<Dependency Name="Microsoft.DotNet.Arcade.Sdk" Version="7.0.0-beta.21559.3">
<Uri>https://github.com/dotnet/arcade</Uri>
<Sha>62a8aafffd4c68ef887680f6837abdff906a662c</Sha>
<Sha>fecf65bedcee9036b8ba9d8d7feef5413f294914</Sha>
</Dependency>
<Dependency Name="Microsoft.DotNet.Build.Tasks.Feed" Version="6.0.0-beta.21379.2">
<Dependency Name="Microsoft.DotNet.Build.Tasks.Feed" Version="7.0.0-beta.21559.3">
<Uri>https://github.com/dotnet/arcade</Uri>
<Sha>62a8aafffd4c68ef887680f6837abdff906a662c</Sha>
<Sha>fecf65bedcee9036b8ba9d8d7feef5413f294914</Sha>
</Dependency>
<Dependency Name="Microsoft.DotNet.SignTool" Version="6.0.0-beta.21379.2">
<Dependency Name="Microsoft.DotNet.SignTool" Version="7.0.0-beta.21559.3">
<Uri>https://github.com/dotnet/arcade</Uri>
<Sha>62a8aafffd4c68ef887680f6837abdff906a662c</Sha>
<Sha>fecf65bedcee9036b8ba9d8d7feef5413f294914</Sha>
</Dependency>
<Dependency Name="Microsoft.DotNet.Helix.Sdk" Version="6.0.0-beta.21379.2">
<Dependency Name="Microsoft.DotNet.Helix.Sdk" Version="7.0.0-beta.21559.3">
<Uri>https://github.com/dotnet/arcade</Uri>
<Sha>62a8aafffd4c68ef887680f6837abdff906a662c</Sha>
<Sha>fecf65bedcee9036b8ba9d8d7feef5413f294914</Sha>
</Dependency>
<Dependency Name="Microsoft.DotNet.SwaggerGenerator.MSBuild" Version="6.0.0-beta.21379.2">
<Dependency Name="Microsoft.DotNet.SwaggerGenerator.MSBuild" Version="7.0.0-beta.21559.3">
<Uri>https://github.com/dotnet/arcade</Uri>
<Sha>62a8aafffd4c68ef887680f6837abdff906a662c</Sha>
<Sha>fecf65bedcee9036b8ba9d8d7feef5413f294914</Sha>
</Dependency>
<Dependency Name="Microsoft.DotNet.Maestro.Client" Version="1.1.0-beta.19556.4">
<Uri>https://github.com/dotnet/arcade-services</Uri>

4
eng/common/build.sh

@ -187,6 +187,10 @@ function InitializeCustomToolset {
}
function Build {
if [[ "$ci" == true ]]; then
TryLogClientIpAddress
fi
InitializeToolset
InitializeCustomToolset

74
eng/common/cross/build-rootfs.sh

@ -6,10 +6,10 @@ usage()
{
echo "Usage: $0 [BuildArch] [CodeName] [lldbx.y] [--skipunmount] --rootfsdir <directory>]"
echo "BuildArch can be: arm(default), armel, arm64, x86"
echo "CodeName - optional, Code name for Linux, can be: trusty, xenial(default), zesty, bionic, alpine, alpine3.9 or alpine3.13. If BuildArch is armel, LinuxCodeName is jessie(default) or tizen."
echo " for FreeBSD can be: freebsd11 or freebsd12."
echo "CodeName - optional, Code name for Linux, can be: xenial(default), zesty, bionic, alpine, alpine3.13 or alpine3.14. If BuildArch is armel, LinuxCodeName is jessie(default) or tizen."
echo " for FreeBSD can be: freebsd11, freebsd12, freebsd13"
echo " for illumos can be: illumos."
echo "lldbx.y - optional, LLDB version, can be: lldb3.9(default), lldb4.0, lldb5.0, lldb6.0 no-lldb. Ignored for alpine and FReeBSD"
echo "lldbx.y - optional, LLDB version, can be: lldb3.9(default), lldb4.0, lldb5.0, lldb6.0 no-lldb. Ignored for alpine and FreeBSD"
echo "--skipunmount - optional, will skip the unmount of rootfs folder."
echo "--use-mirror - optional, use mirror URL to fetch resources, when available."
exit 1
@ -32,9 +32,9 @@ __UbuntuPackages="build-essential"
__AlpinePackages="alpine-base"
__AlpinePackages+=" build-base"
__AlpinePackages+=" linux-headers"
__AlpinePackagesEdgeCommunity=" lldb-dev"
__AlpinePackagesEdgeMain+=" python3"
__AlpinePackagesEdgeMain+=" libedit"
__AlpinePackages+=" lldb-dev"
__AlpinePackages+=" python3"
__AlpinePackages+=" libedit"
# symlinks fixer
__UbuntuPackages+=" symlinks"
@ -60,13 +60,15 @@ __AlpinePackages+=" krb5-dev"
__AlpinePackages+=" openssl-dev"
__AlpinePackages+=" zlib-dev"
__FreeBSDBase="12.1-RELEASE"
__FreeBSDBase="12.2-RELEASE"
__FreeBSDPkg="1.12.0"
__FreeBSDABI="12"
__FreeBSDPackages="libunwind"
__FreeBSDPackages+=" icu"
__FreeBSDPackages+=" libinotify"
__FreeBSDPackages+=" lttng-ust"
__FreeBSDPackages+=" krb5"
__FreeBSDPackages+=" terminfo-db"
__IllumosPackages="icu-64.2nb2"
__IllumosPackages+=" mit-krb5-1.16.2nb4"
@ -144,11 +146,6 @@ while :; do
no-lldb)
unset __LLDB_Package
;;
trusty) # Ubuntu 14.04
if [ "$__CodeName" != "jessie" ]; then
__CodeName=trusty
fi
;;
xenial) # Ubuntu 16.04
if [ "$__CodeName" != "jessie" ]; then
__CodeName=xenial
@ -188,32 +185,34 @@ while :; do
__UbuntuRepo=
__Tizen=tizen
;;
alpine|alpine3.9)
alpine|alpine3.13)
__CodeName=alpine
__UbuntuRepo=
__AlpineVersion=3.9
__AlpinePackagesEdgeMain+=" llvm11-libs"
__AlpinePackagesEdgeMain+=" clang-libs"
__AlpineVersion=3.13
__AlpinePackages+=" llvm10-libs"
;;
alpine3.13)
alpine3.14)
__CodeName=alpine
__UbuntuRepo=
__AlpineVersion=3.13
# Alpine 3.13 has all the packages we need in the 3.13 repository
__AlpinePackages+=$__AlpinePackagesEdgeCommunity
__AlpinePackagesEdgeCommunity=
__AlpinePackages+=$__AlpinePackagesEdgeMain
__AlpinePackagesEdgeMain=
__AlpinePackages+=" llvm10-libs"
__AlpineVersion=3.14
__AlpinePackages+=" llvm11-libs"
;;
freebsd11)
__FreeBSDBase="11.3-RELEASE"
__FreeBSDABI="11"
;&
freebsd12)
__CodeName=freebsd
__BuildArch=x64
__SkipUnmount=1
;;
freebsd13)
__CodeName=freebsd
__FreeBSDBase="13.0-RELEASE"
__FreeBSDABI="13"
__BuildArch=x64
__SkipUnmount=1
;;
illumos)
__CodeName=illumos
__BuildArch=x64
@ -274,26 +273,12 @@ if [[ "$__CodeName" == "alpine" ]]; then
-U --allow-untrusted --root $__RootfsDir --arch $__AlpineArch --initdb \
add $__AlpinePackages
if [[ -n "$__AlpinePackagesEdgeMain" ]]; then
$__ApkToolsDir/apk-tools-$__ApkToolsVersion/apk \
-X http://dl-cdn.alpinelinux.org/alpine/edge/main \
-U --allow-untrusted --root $__RootfsDir --arch $__AlpineArch --initdb \
add $__AlpinePackagesEdgeMain
fi
if [[ -n "$__AlpinePackagesEdgeCommunity" ]]; then
$__ApkToolsDir/apk-tools-$__ApkToolsVersion/apk \
-X http://dl-cdn.alpinelinux.org/alpine/edge/community \
-U --allow-untrusted --root $__RootfsDir --arch $__AlpineArch --initdb \
add $__AlpinePackagesEdgeCommunity
fi
rm -r $__ApkToolsDir
elif [[ "$__CodeName" == "freebsd" ]]; then
mkdir -p $__RootfsDir/usr/local/etc
JOBS="$(getconf _NPROCESSORS_ONLN)"
wget -O - https://download.freebsd.org/ftp/releases/amd64/${__FreeBSDBase}/base.txz | tar -C $__RootfsDir -Jxf - ./lib ./usr/lib ./usr/libdata ./usr/include ./usr/share/keys ./etc ./bin/freebsd-version
# For now, ask for 11 ABI even on 12. This can be revisited later.
echo "ABI = \"FreeBSD:11:amd64\"; FINGERPRINTS = \"${__RootfsDir}/usr/share/keys\"; REPOS_DIR = [\"${__RootfsDir}/etc/pkg\"]; REPO_AUTOUPDATE = NO; RUN_SCRIPTS = NO;" > ${__RootfsDir}/usr/local/etc/pkg.conf
echo "ABI = \"FreeBSD:${__FreeBSDABI}:amd64\"; FINGERPRINTS = \"${__RootfsDir}/usr/share/keys\"; REPOS_DIR = [\"${__RootfsDir}/etc/pkg\"]; REPO_AUTOUPDATE = NO; RUN_SCRIPTS = NO;" > ${__RootfsDir}/usr/local/etc/pkg.conf
echo "FreeBSD: { url: "pkg+http://pkg.FreeBSD.org/\${ABI}/quarterly", mirror_type: \"srv\", signature_type: \"fingerprints\", fingerprints: \"${__RootfsDir}/usr/share/keys/pkg\", enabled: yes }" > ${__RootfsDir}/etc/pkg/FreeBSD.conf
mkdir -p $__RootfsDir/tmp
# get and build package manager
@ -301,7 +286,7 @@ elif [[ "$__CodeName" == "freebsd" ]]; then
cd $__RootfsDir/tmp/pkg-${__FreeBSDPkg}
# needed for install to succeed
mkdir -p $__RootfsDir/host/etc
./autogen.sh && ./configure --prefix=$__RootfsDir/host && make && make install
./autogen.sh && ./configure --prefix=$__RootfsDir/host && make -j "$JOBS" && make install
rm -rf $__RootfsDir/tmp/pkg-${__FreeBSDPkg}
# install packages we need.
INSTALL_AS_USER=$(whoami) $__RootfsDir/host/sbin/pkg -r $__RootfsDir -C $__RootfsDir/usr/local/etc/pkg.conf update
@ -364,13 +349,6 @@ elif [[ -n $__CodeName ]]; then
umount $__RootfsDir/* || true
fi
if [[ "$__BuildArch" == "arm" && "$__CodeName" == "trusty" ]]; then
pushd $__RootfsDir
patch -p1 < $__CrossDir/$__BuildArch/trusty.patch
patch -p1 < $__CrossDir/$__BuildArch/trusty-lttng-2.4.patch
popd
fi
if [[ "$__BuildArch" == "armel" && "$__CodeName" == "jessie" ]]; then
pushd $__RootfsDir
patch -p1 < $__CrossDir/$__BuildArch/armel.jessie.patch

9
eng/common/cross/toolchain.cmake

@ -44,7 +44,7 @@ elseif(TARGET_ARCH_NAME STREQUAL "x86")
set(TOOLCHAIN "i686-linux-gnu")
elseif (CMAKE_SYSTEM_NAME STREQUAL "FreeBSD")
set(CMAKE_SYSTEM_PROCESSOR "x86_64")
set(triple "x86_64-unknown-freebsd11")
set(triple "x86_64-unknown-freebsd12")
elseif (ILLUMOS)
set(CMAKE_SYSTEM_PROCESSOR "x86_64")
set(TOOLCHAIN "x86_64-illumos")
@ -91,6 +91,9 @@ elseif(CMAKE_SYSTEM_NAME STREQUAL "FreeBSD")
set(CMAKE_CXX_COMPILER_TARGET ${triple})
set(CMAKE_ASM_COMPILER_TARGET ${triple})
set(CMAKE_SYSROOT "${CROSS_ROOTFS}")
set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -fuse-ld=lld")
set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} -fuse-ld=lld")
set(CMAKE_MODULE_LINKER_FLAGS "${CMAKE_MODULE_LINKER_FLAGS} -fuse-ld=lld")
elseif(ILLUMOS)
set(CMAKE_SYSROOT "${CROSS_ROOTFS}")
@ -138,8 +141,8 @@ function(add_toolchain_linker_flag Flag)
if (NOT Config STREQUAL "")
set(CONFIG_SUFFIX "_${Config}")
endif()
set("CMAKE_EXE_LINKER_FLAGS${CONFIG_SUFFIX}" "${CMAKE_EXE_LINKER_FLAGS${CONFIG_SUFFIX}} ${Flag}" PARENT_SCOPE)
set("CMAKE_SHARED_LINKER_FLAGS${CONFIG_SUFFIX}" "${CMAKE_SHARED_LINKER_FLAGS${CONFIG_SUFFIX}} ${Flag}" PARENT_SCOPE)
set("CMAKE_EXE_LINKER_FLAGS${CONFIG_SUFFIX}_INIT" "${CMAKE_EXE_LINKER_FLAGS${CONFIG_SUFFIX}_INIT} ${Flag}" PARENT_SCOPE)
set("CMAKE_SHARED_LINKER_FLAGS${CONFIG_SUFFIX}_INIT" "${CMAKE_SHARED_LINKER_FLAGS${CONFIG_SUFFIX}_INIT} ${Flag}" PARENT_SCOPE)
endfunction()
if(CMAKE_SYSTEM_NAME STREQUAL "Linux")

2
eng/common/init-tools-native.sh

@ -10,7 +10,7 @@ force=false
download_retries=5
retry_wait_time_seconds=30
global_json_file="$(dirname "$(dirname "${scriptroot}")")/global.json"
declare -A native_assets
declare -a native_assets
. $scriptroot/pipeline-logging-functions.sh
. $scriptroot/native/common-library.sh

1
eng/common/internal/Tools.csproj

@ -1,5 +1,4 @@
<!-- Licensed to the .NET Foundation under one or more agreements. The .NET Foundation licenses this file to you under the MIT license. -->
<?xml version="1.0" encoding="utf-8"?>
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<TargetFramework>net472</TargetFramework>

1
eng/common/msbuild.ps1

@ -6,6 +6,7 @@ Param(
[switch] $ci,
[switch] $prepareMachine,
[switch] $excludePrereleaseVS,
[string] $msbuildEngine = $null,
[Parameter(ValueFromRemainingArguments=$true)][String[]]$extraArgs
)

8
eng/common/native/common-library.sh

@ -148,8 +148,12 @@ function NewScriptShim {
fi
if [[ ! -f $tool_file_path ]]; then
Write-PipelineTelemetryError -category 'NativeToolsBootstrap' "Specified tool file path:'$tool_file_path' does not exist"
return 1
# try to see if the path is lower cased
tool_file_path="$(echo $tool_file_path | tr "[:upper:]" "[:lower:]")"
if [[ ! -f $tool_file_path ]]; then
Write-PipelineTelemetryError -category 'NativeToolsBootstrap' "Specified tool file path:'$tool_file_path' does not exist"
return 1
fi
fi
local shim_contents=$'#!/usr/bin/env bash\n'

102
eng/common/native/find-native-compiler.sh → eng/common/native/init-compiler.sh

@ -1,39 +1,34 @@
#!/usr/bin/env bash
#
# This file locates the native compiler with the given name and version and sets the environment variables to locate it.
# This file detects the C/C++ compiler and exports it to the CC/CXX environment variables
#
source="${BASH_SOURCE[0]}"
# resolve $SOURCE until the file is no longer a symlink
while [[ -h $source ]]; do
scriptroot="$( cd -P "$( dirname "$source" )" && pwd )"
source="$(readlink "$source")"
# if $source was a relative symlink, we need to resolve it relative to the path where the
# symlink file was located
[[ $source != /* ]] && source="$scriptroot/$source"
done
scriptroot="$( cd -P "$( dirname "$source" )" && pwd )"
if [ $# -lt 0 ]
then
if [[ "$#" -lt 3 ]]; then
echo "Usage..."
echo "find-native-compiler.sh <compiler> <compiler major version> <compiler minor version>"
echo "init-compiler.sh <script directory> <Architecture> <compiler> <compiler major version> <compiler minor version>"
echo "Specify the script directory."
echo "Specify the target architecture."
echo "Specify the name of compiler (clang or gcc)."
echo "Specify the major version of compiler."
echo "Specify the minor version of compiler."
exit 1
fi
. $scriptroot/../pipeline-logging-functions.sh
compiler="$1"
nativescriptroot="$1"
build_arch="$2"
compiler="$3"
cxxCompiler="$compiler++"
majorVersion="$2"
minorVersion="$3"
majorVersion="$4"
minorVersion="$5"
. "$nativescriptroot"/../pipeline-logging-functions.sh
# clear the existing CC and CXX from environment
CC=
CXX=
LDFLAGS=
if [ "$compiler" = "gcc" ]; then cxxCompiler="g++"; fi
if [[ "$compiler" == "gcc" ]]; then cxxCompiler="g++"; fi
check_version_exists() {
desired_version=-1
@ -50,38 +45,38 @@ check_version_exists() {
echo "$desired_version"
}
if [ -z "$CLR_CC" ]; then
if [[ -z "$CLR_CC" ]]; then
# Set default versions
if [ -z "$majorVersion" ]; then
if [[ -z "$majorVersion" ]]; then
# note: gcc (all versions) and clang versions higher than 6 do not have minor version in file name, if it is zero.
if [ "$compiler" = "clang" ]; then versions=( 9 8 7 6.0 5.0 4.0 3.9 3.8 3.7 3.6 3.5 )
elif [ "$compiler" = "gcc" ]; then versions=( 9 8 7 6 5 4.9 ); fi
if [[ "$compiler" == "clang" ]]; then versions=( 13 12 11 10 9 8 7 6.0 5.0 4.0 3.9 3.8 3.7 3.6 3.5 )
elif [[ "$compiler" == "gcc" ]]; then versions=( 12 11 10 9 8 7 6 5 4.9 ); fi
for version in "${versions[@]}"; do
parts=(${version//./ })
desired_version="$(check_version_exists "${parts[0]}" "${parts[1]}")"
if [ "$desired_version" != "-1" ]; then majorVersion="${parts[0]}"; break; fi
if [[ "$desired_version" != "-1" ]]; then majorVersion="${parts[0]}"; break; fi
done
if [ -z "$majorVersion" ]; then
if [[ -z "$majorVersion" ]]; then
if command -v "$compiler" > /dev/null; then
if [ "$(uname)" != "Darwin" ]; then
if [[ "$(uname)" != "Darwin" ]]; then
Write-PipelineTelemetryError -category "Build" -type "warning" "Specific version of $compiler not found, falling back to use the one in PATH."
fi
export CC="$(command -v "$compiler")"
export CXX="$(command -v "$cxxCompiler")"
CC="$(command -v "$compiler")"
CXX="$(command -v "$cxxCompiler")"
else
Write-PipelineTelemetryError -category "Build" "No usable version of $compiler found."
exit 1
fi
else
if [ "$compiler" = "clang" ] && [ "$majorVersion" -lt 5 ]; then
if [ "$build_arch" = "arm" ] || [ "$build_arch" = "armel" ]; then
if [[ "$compiler" == "clang" && "$majorVersion" -lt 5 ]]; then
if [[ "$build_arch" == "arm" || "$build_arch" == "armel" ]]; then
if command -v "$compiler" > /dev/null; then
Write-PipelineTelemetryError -category "Build" -type "warning" "Found clang version $majorVersion which is not supported on arm/armel architectures, falling back to use clang from PATH."
export CC="$(command -v "$compiler")"
export CXX="$(command -v "$cxxCompiler")"
CC="$(command -v "$compiler")"
CXX="$(command -v "$cxxCompiler")"
else
Write-PipelineTelemetryError -category "Build" "Found clang version $majorVersion which is not supported on arm/armel architectures, and there is no clang in PATH."
exit 1
@ -91,31 +86,40 @@ if [ -z "$CLR_CC" ]; then
fi
else
desired_version="$(check_version_exists "$majorVersion" "$minorVersion")"
if [ "$desired_version" = "-1" ]; then
if [[ "$desired_version" == "-1" ]]; then
Write-PipelineTelemetryError -category "Build" "Could not find specific version of $compiler: $majorVersion $minorVersion."
exit 1
fi
fi
if [ -z "$CC" ]; then
export CC="$(command -v "$compiler$desired_version")"
export CXX="$(command -v "$cxxCompiler$desired_version")"
if [ -z "$CXX" ]; then export CXX="$(command -v "$cxxCompiler")"; fi
if [[ -z "$CC" ]]; then
CC="$(command -v "$compiler$desired_version")"
CXX="$(command -v "$cxxCompiler$desired_version")"
if [[ -z "$CXX" ]]; then CXX="$(command -v "$cxxCompiler")"; fi
fi
else
if [ ! -f "$CLR_CC" ]; then
if [[ ! -f "$CLR_CC" ]]; then
Write-PipelineTelemetryError -category "Build" "CLR_CC is set but path '$CLR_CC' does not exist"
exit 1
fi
export CC="$CLR_CC"
export CXX="$CLR_CXX"
CC="$CLR_CC"
CXX="$CLR_CXX"
fi
if [ -z "$CC" ]; then
Write-PipelineTelemetryError -category "Build" "Unable to find $compiler."
if [[ -z "$CC" ]]; then
Write-PipelineTelemetryError -category "Build" "Unable to find $compiler."
exit 1
fi
export CCC_CC="$CC"
export CCC_CXX="$CXX"
export SCAN_BUILD_COMMAND="$(command -v "scan-build$desired_version")"
if [[ "$compiler" == "clang" ]]; then
if command -v "lld$desired_version" > /dev/null; then
# Only lld version >= 9 can be considered stable
if [[ "$majorVersion" -ge 9 ]]; then
LDFLAGS="-fuse-ld=lld"
fi
fi
fi
SCAN_BUILD_COMMAND="$(command -v "scan-build$desired_version")"
export CC CXX LDFLAGS SCAN_BUILD_COMMAND

30
eng/common/post-build/sourcelink-validation.ps1

@ -17,10 +17,16 @@ $global:RepoFiles = @{}
$MaxParallelJobs = 16
$MaxRetries = 5
$RetryWaitTimeInSeconds = 30
# Wait time between check for system load
$SecondsBetweenLoadChecks = 10
if (!$InputPath -or !(Test-Path $InputPath)){
Write-Host "No files to validate."
ExitWithExitCode 0
}
$ValidatePackage = {
param(
[string] $PackagePath # Full path to a Symbols.NuGet package
@ -99,21 +105,25 @@ $ValidatePackage = {
$Status = 200
$Cache = $using:RepoFiles
$totalRetries = 0
$attempts = 0
while ($totalRetries -lt $using:MaxRetries) {
while ($attempts -lt $using:MaxRetries) {
if ( !($Cache.ContainsKey($FilePath)) ) {
try {
$Uri = $Link -as [System.URI]
# Only GitHub links are valid
if ($Uri.AbsoluteURI -ne $null -and ($Uri.Host -match 'github' -or $Uri.Host -match 'githubusercontent')) {
if ($Link -match "submodules") {
# Skip submodule links until sourcelink properly handles submodules
$Status = 200
}
elseif ($Uri.AbsoluteURI -ne $null -and ($Uri.Host -match 'github' -or $Uri.Host -match 'githubusercontent')) {
# Only GitHub links are valid
$Status = (Invoke-WebRequest -Uri $Link -UseBasicParsing -Method HEAD -TimeoutSec 5).StatusCode
}
else {
# If it's not a github link, we want to break out of the loop and not retry.
$Status = 0
$totalRetries = $using:MaxRetries
$attempts = $using:MaxRetries
}
}
catch {
@ -123,9 +133,15 @@ $ValidatePackage = {
}
if ($Status -ne 200) {
$totalRetries++
$attempts++
if ($totalRetries -ge $using:MaxRetries) {
if ($attempts -lt $using:MaxRetries)
{
$attemptsLeft = $using:MaxRetries - $attempts
Write-Warning "Download failed, $attemptsLeft attempts remaining, will retry in $using:RetryWaitTimeInSeconds seconds"
Start-Sleep -Seconds $using:RetryWaitTimeInSeconds
}
else {
if ($NumFailedLinks -eq 0) {
if ($FailedFiles.Value -eq 0) {
Write-Host

95
eng/common/post-build/symbols-validation.ps1

@ -4,9 +4,11 @@ param(
[Parameter(Mandatory = $true)][string] $DotnetSymbolVersion, # Version of dotnet symbol to use
[Parameter(Mandatory = $false)][switch] $CheckForWindowsPdbs, # If we should check for the existence of windows pdbs in addition to portable PDBs
[Parameter(Mandatory = $false)][switch] $ContinueOnError, # If we should keep checking symbols after an error
[Parameter(Mandatory = $false)][switch] $Clean # Clean extracted symbols directory after checking symbols
[Parameter(Mandatory = $false)][switch] $Clean, # Clean extracted symbols directory after checking symbols
[Parameter(Mandatory = $false)][string] $SymbolExclusionFile # Exclude the symbols in the file from publishing to symbol server
)
. $PSScriptRoot\..\tools.ps1
# Maximum number of jobs to run in parallel
$MaxParallelJobs = 16
@ -25,14 +27,28 @@ if ($CheckForWindowsPdbs) {
$WindowsPdbVerificationParam = "--windows-pdbs"
}
$ExclusionSet = New-Object System.Collections.Generic.HashSet[string];
if (!$InputPath -or !(Test-Path $InputPath)){
Write-Host "No symbols to validate."
ExitWithExitCode 0
}
#Check if the path exists
if ($SymbolExclusionFile -and (Test-Path $SymbolExclusionFile)){
[string[]]$Exclusions = Get-Content "$SymbolExclusionFile"
$Exclusions | foreach { if($_ -and $_.Trim()){$ExclusionSet.Add($_)} }
}
else{
Write-Host "Symbol Exclusion file does not exists. No symbols to exclude."
}
$CountMissingSymbols = {
param(
[string] $PackagePath, # Path to a NuGet package
[string] $WindowsPdbVerificationParam # If we should check for the existence of windows pdbs in addition to portable PDBs
)
. $using:PSScriptRoot\..\tools.ps1
Add-Type -AssemblyName System.IO.Compression.FileSystem
Write-Host "Validating $PackagePath "
@ -118,17 +134,17 @@ $CountMissingSymbols = {
# Save the output and get diagnostic output
$output = & $dotnetSymbolExe --symbols --modules $WindowsPdbVerificationParam $TargetServerParam $FullPath -o $SymbolsPath --diagnostics | Out-String
if (Test-Path $PdbPath) {
return 'PDB'
if ((Test-Path $PdbPath) -and (Test-path $SymbolPath)) {
return 'Module and PDB for Module'
}
elseif (Test-Path $NGenPdb) {
return 'NGen PDB'
elseif ((Test-Path $NGenPdb) -and (Test-Path $PdbPath) -and (Test-Path $SymbolPath)) {
return 'Dll, PDB and NGen PDB'
}
elseif (Test-Path $SODbg) {
return 'DBG for SO'
elseif ((Test-Path $SODbg) -and (Test-Path $SymbolPath)) {
return 'So and DBG for SO'
}
elseif (Test-Path $DylibDwarf) {
return 'Dwarf for Dylib'
elseif ((Test-Path $DylibDwarf) -and (Test-Path $SymbolPath)) {
return 'Dylib and Dwarf for Dylib'
}
elseif (Test-Path $SymbolPath) {
return 'Module'
@ -142,37 +158,44 @@ $CountMissingSymbols = {
return $null
}
$FileGuid = New-Guid
$ExpandedSymbolsPath = Join-Path -Path $SymbolsPath -ChildPath $FileGuid
$SymbolsOnMSDL = & $FirstMatchingSymbolDescriptionOrDefault `
-FullPath $FileName `
-TargetServerParam '--microsoft-symbol-server' `
-SymbolsPath "$ExpandedSymbolsPath-msdl" `
-WindowsPdbVerificationParam $WindowsPdbVerificationParam
$SymbolsOnSymWeb = & $FirstMatchingSymbolDescriptionOrDefault `
-FullPath $FileName `
-TargetServerParam '--internal-server' `
-SymbolsPath "$ExpandedSymbolsPath-symweb" `
-WindowsPdbVerificationParam $WindowsPdbVerificationParam
Write-Host -NoNewLine "`t Checking file " $FileName "... "
if ($SymbolsOnMSDL -ne $null -and $SymbolsOnSymWeb -ne $null) {
Write-Host "Symbols found on MSDL ($SymbolsOnMSDL) and SymWeb ($SymbolsOnSymWeb)"
$FileRelativePath = $FileName.Replace("$ExtractPath\", "")
if (($($using:ExclusionSet) -ne $null) -and ($($using:ExclusionSet).Contains($FileRelativePath) -or ($($using:ExclusionSet).Contains($FileRelativePath.Replace("\", "/"))))){
Write-Host "Skipping $FileName from symbol validation"
}
else {
$MissingSymbols++
if ($SymbolsOnMSDL -eq $null -and $SymbolsOnSymWeb -eq $null) {
Write-Host 'No symbols found on MSDL or SymWeb!'
else {
$FileGuid = New-Guid
$ExpandedSymbolsPath = Join-Path -Path $SymbolsPath -ChildPath $FileGuid
$SymbolsOnMSDL = & $FirstMatchingSymbolDescriptionOrDefault `
-FullPath $FileName `
-TargetServerParam '--microsoft-symbol-server' `
-SymbolsPath "$ExpandedSymbolsPath-msdl" `
-WindowsPdbVerificationParam $WindowsPdbVerificationParam
$SymbolsOnSymWeb = & $FirstMatchingSymbolDescriptionOrDefault `
-FullPath $FileName `
-TargetServerParam '--internal-server' `
-SymbolsPath "$ExpandedSymbolsPath-symweb" `
-WindowsPdbVerificationParam $WindowsPdbVerificationParam
Write-Host -NoNewLine "`t Checking file " $FileName "... "
if ($SymbolsOnMSDL -ne $null -and $SymbolsOnSymWeb -ne $null) {
Write-Host "Symbols found on MSDL ($SymbolsOnMSDL) and SymWeb ($SymbolsOnSymWeb)"
}
else {
if ($SymbolsOnMSDL -eq $null) {
Write-Host 'No symbols found on MSDL!'
$MissingSymbols++
if ($SymbolsOnMSDL -eq $null -and $SymbolsOnSymWeb -eq $null) {
Write-Host 'No symbols found on MSDL or SymWeb!'
}
else {
Write-Host 'No symbols found on SymWeb!'
if ($SymbolsOnMSDL -eq $null) {
Write-Host 'No symbols found on MSDL!'
}
else {
Write-Host 'No symbols found on SymWeb!'
}
}
}
}

3
eng/common/sdk-task.ps1

@ -83,6 +83,9 @@ try {
}
if ($restore) {
if ($ci) {
Try-LogClientIpAddress
}
Build 'Restore'
}

6
eng/common/sdl/configure-sdl-tool.ps1

@ -69,13 +69,13 @@ try {
# For some tools, add default and automatic args.
if ($tool.Name -eq 'credscan') {
if ($targetDirectory) {
$tool.Args += "TargetDirectory < $TargetDirectory"
$tool.Args += "`"TargetDirectory < $TargetDirectory`""
}
$tool.Args += "OutputType < pre"
$tool.Args += "`"OutputType < pre`""
$tool.Args += $CrScanAdditionalRunConfigParams
} elseif ($tool.Name -eq 'policheck') {
if ($targetDirectory) {
$tool.Args += "Target < $TargetDirectory"
$tool.Args += "`"Target < $TargetDirectory`""
}
$tool.Args += $PoliCheckAdditionalRunConfigParams
}

6
eng/common/templates/job/execute-sdl.yml

@ -60,11 +60,7 @@ jobs:
- name: GuardianPackagesConfigFile
value: $(Build.SourcesDirectory)\eng\common\sdl\packages.config
pool:
# To extract archives (.tar.gz, .zip), we need access to "tar", added in Windows 10/2019.
${{ if eq(parameters.extractArchiveArtifacts, 'false') }}:
name: Hosted VS2017
${{ if ne(parameters.extractArchiveArtifacts, 'false') }}:
vmImage: windows-2019
vmImage: windows-2019
steps:
- checkout: self
clean: true

2
eng/common/templates/job/job.yml

@ -103,7 +103,7 @@ jobs:
- ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}:
- ${{ if eq(parameters.enableMicrobuild, 'true') }}:
- task: MicroBuildSigningPlugin@2
- task: MicroBuildSigningPlugin@3
displayName: Install MicroBuild plugin
inputs:
signType: $(_SignType)

5
eng/common/templates/job/onelocbuild.yml

@ -4,7 +4,7 @@ parameters:
# Optional: A defined YAML pool - https://docs.microsoft.com/en-us/azure/devops/pipelines/yaml-schema?view=vsts&tabs=schema#pool
pool:
vmImage: vs2017-win2016
vmImage: 'windows-2019'
CeapexPat: $(dn-bot-ceapex-package-r) # PAT for the loc AzDO instance https://dev.azure.com/ceapex
GithubPat: $(BotAccount-dotnet-bot-repo-PAT)
@ -12,6 +12,7 @@ parameters:
SourcesDirectory: $(Build.SourcesDirectory)
CreatePr: true
AutoCompletePr: false
ReusePr: true
UseLfLineEndings: true
UseCheckedInLocProjectJson: false
LanguageSet: VS_Main_Languages
@ -64,6 +65,8 @@ jobs:
${{ if eq(parameters.CreatePr, true) }}:
isAutoCompletePrSelected: ${{ parameters.AutoCompletePr }}
isUseLfLineEndingsSelected: ${{ parameters.UseLfLineEndings }}
${{ if eq(parameters.RepoType, 'gitHub') }}:
isShouldReusePrSelected: ${{ parameters.ReusePr }}
packageSourceAuth: patAuth
patVariable: ${{ parameters.CeapexPat }}
${{ if eq(parameters.RepoType, 'gitHub') }}:

26
eng/common/templates/job/publish-build-assets.yml

@ -94,7 +94,31 @@ jobs:
PathtoPublish: '$(Build.StagingDirectory)/ReleaseConfigs.txt'
PublishLocation: Container
ArtifactName: ReleaseConfigs
- task: powershell@2
displayName: Check if SymbolPublishingExclusionsFile.txt exists
inputs:
targetType: inline
script: |
$symbolExclusionfile = "$(Build.SourcesDirectory)/eng/SymbolPublishingExclusionsFile.txt"
if(Test-Path -Path $symbolExclusionfile)
{
Write-Host "SymbolExclusionFile exists"
Write-Host "##vso[task.setvariable variable=SymbolExclusionFile]true"
}
else{
Write-Host "Symbols Exclusion file does not exists"
Write-Host "##vso[task.setvariable variable=SymbolExclusionFile]false"
}
- task: PublishBuildArtifacts@1
displayName: Publish SymbolPublishingExclusionsFile Artifact
condition: eq(variables['SymbolExclusionFile'], 'true')
inputs:
PathtoPublish: '$(Build.SourcesDirectory)/eng/SymbolPublishingExclusionsFile.txt'
PublishLocation: Container
ArtifactName: ReleaseConfigs
- ${{ if eq(parameters.enablePublishBuildArtifacts, 'true') }}:
- template: /eng/common/templates/steps/publish-logs.yml
parameters:

15
eng/common/templates/job/source-build.yml

@ -31,11 +31,6 @@ parameters:
# container and pool.
platform: {}
# The default VM host AzDO pool. This should be capable of running Docker containers: almost all
# source-build builds run in Docker, including the default managed platform.
defaultContainerHostPool:
vmImage: ubuntu-20.04
jobs:
- job: ${{ parameters.jobNamePrefix }}_${{ parameters.platform.name }}
displayName: Source-Build (${{ parameters.platform.name }})
@ -47,7 +42,15 @@ jobs:
container: ${{ parameters.platform.container }}
${{ if eq(parameters.platform.pool, '') }}:
pool: ${{ parameters.defaultContainerHostPool }}
# The default VM host AzDO pool. This should be capable of running Docker containers: almost all
# source-build builds run in Docker, including the default managed platform.
pool:
${{ if eq(variables['System.TeamProject'], 'public') }}:
name: NetCore1ESPool-Public
demands: ImageOverride -equals Build.Ubuntu.1804.Amd64.Open
${{ if eq(variables['System.TeamProject'], 'internal') }}:
name: NetCore1ESPool-Internal
demands: ImageOverride -equals Build.Ubuntu.1804.Amd64
${{ if ne(parameters.platform.pool, '') }}:
pool: ${{ parameters.platform.pool }}

31
eng/common/templates/job/source-index-stage1.yml

@ -5,8 +5,6 @@ parameters:
sourceIndexBuildCommand: powershell -NoLogo -NoProfile -ExecutionPolicy Bypass -Command "eng/common/build.ps1 -restore -build -binarylog -ci"
preSteps: []
binlogPath: artifacts/log/Debug/Build.binlog
pool:
vmImage: vs2017-win2016
condition: ''
dependsOn: ''
@ -24,7 +22,13 @@ jobs:
- ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}:
- group: source-dot-net stage1 variables
pool: ${{ parameters.pool }}
pool:
${{ if eq(variables['System.TeamProject'], 'public') }}:
name: NetCore1ESPool-Public
demands: ImageOverride -equals Build.Server.Amd64.VS2019.Open
${{ if eq(variables['System.TeamProject'], 'internal') }}:
name: NetCore1ESPool-Internal
demands: ImageOverride -equals Build.Server.Amd64.VS2019
steps:
- ${{ each preStep in parameters.preSteps }}:
- ${{ preStep }}
@ -34,29 +38,24 @@ jobs:
inputs:
packageType: sdk
version: 3.1.x
- task: UseDotNet@2
displayName: Use .NET Core sdk
inputs:
useGlobalJson: true
installationPath: $(Agent.TempDirectory)/dotnet
workingDirectory: $(Agent.TempDirectory)
- script: |
dotnet tool install BinLogToSln --version $(SourceIndexPackageVersion) --add-source $(SourceIndexPackageSource) --tool-path .source-index/tools
dotnet tool install UploadIndexStage1 --version $(SourceIndexPackageVersion) --add-source $(SourceIndexPackageSource) --tool-path .source-index/tools
echo ##vso[task.prependpath]$(Build.SourcesDirectory)/.source-index/tools
$(Agent.TempDirectory)/dotnet/dotnet tool install BinLogToSln --version $(SourceIndexPackageVersion) --add-source $(SourceIndexPackageSource) --tool-path $(Agent.TempDirectory)/.source-index/tools
$(Agent.TempDirectory)/dotnet/dotnet tool install UploadIndexStage1 --version $(SourceIndexPackageVersion) --add-source $(SourceIndexPackageSource) --tool-path $(Agent.TempDirectory)/.source-index/tools
displayName: Download Tools
# Set working directory to temp directory so 'dotnet' doesn't try to use global.json and use the repo's sdk.
workingDirectory: $(Agent.TempDirectory)
- script: ${{ parameters.sourceIndexBuildCommand }}
displayName: Build Repository
- script: BinLogToSln -i $(BinlogPath) -r $(Build.SourcesDirectory) -n $(Build.Repository.Name) -o .source-index/stage1output
- script: $(Agent.TempDirectory)/.source-index/tools/BinLogToSln -i $(BinlogPath) -r $(Build.SourcesDirectory) -n $(Build.Repository.Name) -o .source-index/stage1output
displayName: Process Binlog into indexable sln
env:
DOTNET_ROLL_FORWARD_ON_NO_CANDIDATE_FX: 2
- ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}:
- script: UploadIndexStage1 -i .source-index/stage1output -n $(Build.Repository.Name)
- script: $(Agent.TempDirectory)/.source-index/tools/UploadIndexStage1 -i .source-index/stage1output -n $(Build.Repository.Name)
displayName: Upload stage1 artifacts to source index
env:
BLOB_CONTAINER_URL: $(source-dot-net-stage1-blob-container-url)
DOTNET_ROLL_FORWARD_ON_NO_CANDIDATE_FX: 2

4
eng/common/templates/jobs/jobs.yml

@ -83,7 +83,7 @@ jobs:
- ${{ if eq(parameters.enableSourceBuild, true) }}:
- Source_Build_Complete
pool:
vmImage: vs2017-win2016
vmImage: 'windows-2019'
runAsPublic: ${{ parameters.runAsPublic }}
publishUsingPipelines: ${{ parameters.enablePublishUsingPipelines }}
enablePublishBuildArtifacts: ${{ parameters.enablePublishBuildArtifacts }}
@ -96,4 +96,4 @@ jobs:
dependsOn:
- Asset_Registry_Publish
pool:
vmImage: vs2017-win2016
vmImage: 'windows-2019'

130
eng/common/templates/phases/base.yml

@ -1,130 +0,0 @@
parameters:
# Optional: Clean sources before building
clean: true
# Optional: Git fetch depth
fetchDepth: ''
# Optional: name of the phase (not specifying phase name may cause name collisions)
name: ''
# Optional: display name of the phase
displayName: ''
# Optional: condition for the job to run
condition: ''
# Optional: dependencies of the phase
dependsOn: ''
# Required: A defined YAML queue
queue: {}
# Required: build steps
steps: []
# Optional: variables
variables: {}
# Optional: should run as a public build even in the internal project
# if 'true', the build won't run any of the internal only steps, even if it is running in non-public projects.
runAsPublic: false
## Telemetry variables
# Optional: enable sending telemetry
# if 'true', these "variables" must be specified in the variables object or as part of the queue matrix
# _HelixBuildConfig - differentiate between Debug, Release, other
# _HelixSource - Example: build/product
# _HelixType - Example: official/dotnet/arcade/$(Build.SourceBranch)
enableTelemetry: false
# Optional: Enable installing Microbuild plugin
# if 'true', these "variables" must be specified in the variables object or as part of the queue matrix
# _TeamName - the name of your team
# _SignType - 'test' or 'real'
enableMicrobuild: false
# Internal resources (telemetry, microbuild) can only be accessed from non-public projects,
# and some (Microbuild) should only be applied to non-PR cases for internal builds.
phases:
- phase: ${{ parameters.name }}
${{ if ne(parameters.displayName, '') }}:
displayName: ${{ parameters.displayName }}
${{ if ne(parameters.condition, '') }}:
condition: ${{ parameters.condition }}
${{ if ne(parameters.dependsOn, '') }}:
dependsOn: ${{ parameters.dependsOn }}
queue: ${{ parameters.queue }}
${{ if ne(parameters.variables, '') }}:
variables:
${{ insert }}: ${{ parameters.variables }}
steps:
- checkout: self
clean: ${{ parameters.clean }}
${{ if ne(parameters.fetchDepth, '') }}:
fetchDepth: ${{ parameters.fetchDepth }}
- ${{ if eq(parameters.enableTelemetry, 'true') }}:
- template: /eng/common/templates/steps/telemetry-start.yml
parameters:
buildConfig: $(_HelixBuildConfig)
helixSource: $(_HelixSource)
helixType: $(_HelixType)
runAsPublic: ${{ parameters.runAsPublic }}
- ${{ if eq(parameters.enableMicrobuild, 'true') }}:
# Internal only resource, and Microbuild signing shouldn't be applied to PRs.
- ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}:
- task: MicroBuildSigningPlugin@2
displayName: Install MicroBuild plugin
inputs:
signType: $(_SignType)
zipSources: false
feedSource: https://dnceng.pkgs.visualstudio.com/_packaging/MicroBuildToolset/nuget/v3/index.json
env:
TeamName: $(_TeamName)
continueOnError: false
condition: and(succeeded(), in(variables['_SignType'], 'real', 'test'), eq(variables['Agent.Os'], 'Windows_NT'))
# Run provided build steps
- ${{ parameters.steps }}
- ${{ if eq(parameters.enableMicrobuild, 'true') }}:
# Internal only resources
- ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}:
- task: MicroBuildCleanup@1
displayName: Execute Microbuild cleanup tasks
condition: and(always(), in(variables['_SignType'], 'real', 'test'), eq(variables['Agent.Os'], 'Windows_NT'))
env:
TeamName: $(_TeamName)
- ${{ if eq(parameters.enableTelemetry, 'true') }}:
- template: /eng/common/templates/steps/telemetry-end.yml
parameters:
helixSource: $(_HelixSource)
helixType: $(_HelixType)
- ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}:
- task: CopyFiles@2
displayName: Gather Asset Manifests
inputs:
SourceFolder: '$(Build.SourcesDirectory)/artifacts/log/$(_BuildConfig)/AssetManifest'
TargetFolder: '$(Build.StagingDirectory)/AssetManifests'
continueOnError: false
condition: and(succeeded(), eq(variables['_DotNetPublishToBlobFeed'], 'true'))
- task: PublishBuildArtifacts@1
displayName: Push Asset Manifests
inputs:
PathtoPublish: '$(Build.StagingDirectory)/AssetManifests'
PublishLocation: Container
ArtifactName: AssetManifests
continueOnError: false
condition: and(succeeded(), eq(variables['_DotNetPublishToBlobFeed'], 'true'))

52
eng/common/templates/phases/publish-build-assets.yml

@ -1,52 +0,0 @@
parameters:
dependsOn: ''
queue: {}
configuration: 'Debug'
condition: succeeded()
continueOnError: false
runAsPublic: false
publishUsingPipelines: false
phases:
- phase: Asset_Registry_Publish
displayName: Publish to Build Asset Registry
dependsOn: ${{ parameters.dependsOn }}
queue: ${{ parameters.queue }}
variables:
_BuildConfig: ${{ parameters.configuration }}
steps:
- ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}:
- task: DownloadBuildArtifacts@0
displayName: Download artifact
inputs:
artifactName: AssetManifests
downloadPath: '$(Build.StagingDirectory)/Download'
checkDownloadedFiles: true
condition: ${{ parameters.condition }}
continueOnError: ${{ parameters.continueOnError }}
- task: AzureKeyVault@1
inputs:
azureSubscription: 'DotNet-Engineering-Services_KeyVault'
KeyVaultName: EngKeyVault
SecretsFilter: 'MaestroAccessToken'
condition: ${{ parameters.condition }}
continueOnError: ${{ parameters.continueOnError }}
- task: PowerShell@2
displayName: Publish Build Assets
inputs:
filePath: eng\common\sdk-task.ps1
arguments: -task PublishBuildAssets -restore -msbuildEngine dotnet
/p:ManifestsPath='$(Build.StagingDirectory)/Download/AssetManifests'
/p:BuildAssetRegistryToken=$(MaestroAccessToken)
/p:MaestroApiEndpoint=https://maestro-prod.westus2.cloudapp.azure.com
/p:PublishUsingPipelines=${{ parameters.publishUsingPipelines }}
/p:Configuration=$(_BuildConfig)
condition: ${{ parameters.condition }}
continueOnError: ${{ parameters.continueOnError }}
- task: PublishBuildArtifacts@1
displayName: Publish Logs to VSTS
inputs:
PathtoPublish: '$(Build.SourcesDirectory)/artifacts/log/$(_BuildConfig)'
PublishLocation: Container
ArtifactName: $(Agent.Os)_Asset_Registry_Publish
continueOnError: true
condition: always()

3
eng/common/templates/steps/send-to-helix.yml

@ -20,7 +20,6 @@ parameters:
IncludeDotNetCli: false # optional -- true will download a version of the .NET CLI onto the Helix machine as a correlation payload; requires DotNetCliPackageType and DotNetCliVersion
DotNetCliPackageType: '' # optional -- either 'sdk', 'runtime' or 'aspnetcore-runtime'; determines whether the sdk or runtime will be sent to Helix; see https://raw.githubusercontent.com/dotnet/core/main/release-notes/releases-index.json
DotNetCliVersion: '' # optional -- version of the CLI to send to Helix; based on this: https://raw.githubusercontent.com/dotnet/core/main/release-notes/releases-index.json
EnableXUnitReporter: false # optional -- true enables XUnit result reporting to Mission Control
WaitForWorkItemCompletion: true # optional -- true will make the task wait until work items have been completed and fail the build if work items fail. False is "fire and forget."
IsExternal: false # [DEPRECATED] -- doesn't do anything, jobs are external if HelixAccessToken is empty and Creator is set
HelixBaseUri: 'https://helix.dot.net/' # optional -- sets the Helix API base URI (allows targeting int)
@ -54,7 +53,6 @@ steps:
IncludeDotNetCli: ${{ parameters.IncludeDotNetCli }}
DotNetCliPackageType: ${{ parameters.DotNetCliPackageType }}
DotNetCliVersion: ${{ parameters.DotNetCliVersion }}
EnableXUnitReporter: ${{ parameters.EnableXUnitReporter }}
WaitForWorkItemCompletion: ${{ parameters.WaitForWorkItemCompletion }}
HelixBaseUri: ${{ parameters.HelixBaseUri }}
Creator: ${{ parameters.Creator }}
@ -85,7 +83,6 @@ steps:
IncludeDotNetCli: ${{ parameters.IncludeDotNetCli }}
DotNetCliPackageType: ${{ parameters.DotNetCliPackageType }}
DotNetCliVersion: ${{ parameters.DotNetCliVersion }}
EnableXUnitReporter: ${{ parameters.EnableXUnitReporter }}
WaitForWorkItemCompletion: ${{ parameters.WaitForWorkItemCompletion }}
HelixBaseUri: ${{ parameters.HelixBaseUri }}
Creator: ${{ parameters.Creator }}

35
eng/common/templates/steps/source-build.yml

@ -18,6 +18,35 @@ steps:
set -x
df -h
# If building on the internal project, the artifact feeds variable may be available (usually only if needed)
# In that case, call the feed setup script to add internal feeds corresponding to public ones.
# In addition, add an msbuild argument to copy the WIP from the repo to the target build location.
# This is because SetupNuGetSources.sh will alter the current NuGet.config file, and we need to preserve those
# changes.
$internalRestoreArgs=
if [ '$(dn-bot-dnceng-artifact-feeds-rw)' != '$''(dn-bot-dnceng-artifact-feeds-rw)' ]; then
# Temporarily work around https://github.com/dotnet/arcade/issues/7709
chmod +x $(Build.SourcesDirectory)/eng/common/SetupNugetSources.sh
$(Build.SourcesDirectory)/eng/common/SetupNugetSources.sh $(Build.SourcesDirectory)/NuGet.config $(dn-bot-dnceng-artifact-feeds-rw)
internalRestoreArgs='/p:CopyWipIntoInnerSourceBuildRepo=true'
# The 'Copy WIP' feature of source build uses git stash to apply changes from the original repo.
# This only works if there is a username/email configured, which won't be the case in most CI runs.
git config --get user.email
if [ $? -ne 0 ]; then
git config user.email dn-bot@microsoft.com
git config user.name dn-bot
fi
fi
# If building on the internal project, the internal storage variable may be available (usually only if needed)
# In that case, add variables to allow the download of internal runtimes if the specified versions are not found
# in the default public locations.
internalRuntimeDownloadArgs=
if [ '$(dotnetclimsrc-read-sas-token-base64)' != '$''(dotnetclimsrc-read-sas-token-base64)' ]; then
internalRuntimeDownloadArgs='/p:DotNetRuntimeSourceFeed=https://dotnetclimsrc.blob.core.windows.net/dotnet /p:DotNetRuntimeSourceFeedKey=$(dotnetclimsrc-read-sas-token-base64) --runtimesourcefeed https://dotnetclimsrc.blob.core.windows.net/dotnet --runtimesourcefeedkey $(dotnetclimsrc-read-sas-token-base64)'
fi
buildConfig=Release
# Check if AzDO substitutes in a build config from a variable, and use it if so.
if [ '$(_BuildConfig)' != '$''(_BuildConfig)' ]; then
@ -29,11 +58,6 @@ steps:
officialBuildArgs='/p:DotNetPublishUsingPipelines=true /p:OfficialBuildId=$(BUILD.BUILDNUMBER)'
fi
internalRuntimeDownloadArgs=
if [ '$(dotnetclimsrc-read-sas-token-base64)' != '$''(dotnetclimsrc-read-sas-token-base64)' ]; then
internalRuntimeDownloadArgs='--runtimesourcefeed https://dotnetclimsrc.blob.core.windows.net/dotnet --runtimesourcefeedkey $(dotnetclimsrc-read-sas-token-base64)'
fi
targetRidArgs=
if [ '${{ parameters.platform.targetRID }}' != '' ]; then
targetRidArgs='/p:TargetRid=${{ parameters.platform.targetRID }}'
@ -49,6 +73,7 @@ steps:
--restore --build --pack $publishArgs -bl \
$officialBuildArgs \
$internalRuntimeDownloadArgs \
$internalRestoreArgs \
$targetRidArgs \
/p:SourceBuildNonPortable=${{ parameters.platform.nonPortable }} \
/p:ArcadeBuildFromSource=true

42
eng/common/tools.ps1

@ -163,6 +163,9 @@ function InitializeDotNetCli([bool]$install, [bool]$createSdkLocationFile) {
# Disable telemetry on CI.
if ($ci) {
$env:DOTNET_CLI_TELEMETRY_OPTOUT=1
# In case of network error, try to log the current IP for reference
Try-LogClientIpAddress
}
# Source Build uses DotNetCoreSdkDir variable
@ -706,6 +709,8 @@ function MSBuild() {
Write-PipelineSetVariable -Name 'NUGET_PLUGIN_REQUEST_TIMEOUT_IN_SECONDS' -Value '20'
}
Enable-Nuget-EnhancedRetry
$toolsetBuildProject = InitializeToolset
$basePath = Split-Path -parent $toolsetBuildProject
$possiblePaths = @(
@ -714,6 +719,8 @@ function MSBuild() {
(Join-Path $basePath (Join-Path $buildTool.Framework 'Microsoft.DotNet.Arcade.Sdk.dll')),
(Join-Path $basePath (Join-Path netcoreapp2.1 'Microsoft.DotNet.ArcadeLogging.dll')),
(Join-Path $basePath (Join-Path netcoreapp2.1 'Microsoft.DotNet.Arcade.Sdk.dll'))
(Join-Path $basePath (Join-Path netcoreapp3.1 'Microsoft.DotNet.ArcadeLogging.dll')),
(Join-Path $basePath (Join-Path netcoreapp3.1 'Microsoft.DotNet.Arcade.Sdk.dll'))
)
$selectedPath = $null
foreach ($path in $possiblePaths) {
@ -750,6 +757,8 @@ function MSBuild-Core() {
}
}
Enable-Nuget-EnhancedRetry
$buildTool = InitializeBuildTool
$cmdArgs = "$($buildTool.Command) /m /nologo /clp:Summary /v:$verbosity /nr:$nodeReuse /p:ContinuousIntegrationBuild=$ci"
@ -872,3 +881,36 @@ if (!$disableConfigureToolsetImport) {
}
}
}
function Try-LogClientIpAddress()
{
Write-Host "Attempting to log this client's IP for Azure Package feed telemetry purposes"
try
{
$result = Invoke-WebRequest -Uri "http://co1.msedge.net/fdv2/diagnostics.aspx" -UseBasicParsing
$lines = $result.Content.Split([Environment]::NewLine)
$socketIp = $lines | Select-String -Pattern "^Socket IP:.*"
Write-Host $socketIp
$clientIp = $lines | Select-String -Pattern "^Client IP:.*"
Write-Host $clientIp
}
catch
{
Write-Host "Unable to get this machine's effective IP address for logging: $_"
}
}
#
# If $ci flag is set, turn on (and log that we did) special environment variables for improved Nuget client retry logic.
#
function Enable-Nuget-EnhancedRetry() {
if ($ci) {
Write-Host "Setting NUGET enhanced retry environment variables"
$env:NUGET_ENABLE_EXPERIMENTAL_HTTP_RETRY = 'true'
$env:NUGET_EXPERIMENTAL_MAX_NETWORK_TRY_COUNT = 6
$env:NUGET_EXPERIMENTAL_NETWORK_RETRY_DELAY_MILLISECONDS = 1000
Write-PipelineSetVariable -Name 'NUGET_ENABLE_EXPERIMENTAL_HTTP_RETRY' -Value 'true'
Write-PipelineSetVariable -Name 'NUGET_EXPERIMENTAL_MAX_NETWORK_TRY_COUNT' -Value '6'
Write-PipelineSetVariable -Name 'NUGET_EXPERIMENTAL_NETWORK_RETRY_DELAY_MILLISECONDS' -Value '1000'
}
}

16
eng/common/tools.sh

@ -399,6 +399,13 @@ function StopProcesses {
return 0
}
function TryLogClientIpAddress () {
echo 'Attempting to log this client''s IP for Azure Package feed telemetry purposes'
if command -v curl > /dev/null; then
curl -s 'http://co1.msedge.net/fdv2/diagnostics.aspx' | grep ' IP: ' || true
fi
}
function MSBuild {
local args=$@
if [[ "$pipelines_log" == true ]]; then
@ -410,6 +417,13 @@ function MSBuild {
export NUGET_PLUGIN_REQUEST_TIMEOUT_IN_SECONDS=20
Write-PipelineSetVariable -name "NUGET_PLUGIN_HANDSHAKE_TIMEOUT_IN_SECONDS" -value "20"
Write-PipelineSetVariable -name "NUGET_PLUGIN_REQUEST_TIMEOUT_IN_SECONDS" -value "20"
export NUGET_ENABLE_EXPERIMENTAL_HTTP_RETRY=true
export NUGET_EXPERIMENTAL_MAX_NETWORK_TRY_COUNT=6
export NUGET_EXPERIMENTAL_NETWORK_RETRY_DELAY_MILLISECONDS=1000
Write-PipelineSetVariable -name "NUGET_ENABLE_EXPERIMENTAL_HTTP_RETRY" -value "true"
Write-PipelineSetVariable -name "NUGET_EXPERIMENTAL_MAX_NETWORK_TRY_COUNT" -value "6"
Write-PipelineSetVariable -name "NUGET_EXPERIMENTAL_NETWORK_RETRY_DELAY_MILLISECONDS" -value "1000"
fi
local toolset_dir="${_InitializeToolset%/*}"
@ -420,6 +434,8 @@ function MSBuild {
possiblePaths+=( "$toolset_dir/$_InitializeBuildToolFramework/Microsoft.DotNet.Arcade.Sdk.dll" )
possiblePaths+=( "$toolset_dir/netcoreapp2.1/Microsoft.DotNet.ArcadeLogging.dll" )
possiblePaths+=( "$toolset_dir/netcoreapp2.1/Microsoft.DotNet.Arcade.Sdk.dll" )
possiblePaths+=( "$toolset_dir/netcoreapp3.1/Microsoft.DotNet.ArcadeLogging.dll" )
possiblePaths+=( "$toolset_dir/netcoreapp3.1/Microsoft.DotNet.Arcade.Sdk.dll" )
for path in "${possiblePaths[@]}"; do
if [[ -f $path ]]; then
selectedPath=$path

4
global.json

@ -3,7 +3,7 @@
"allowPrerelease": true
},
"tools": {
"dotnet": "6.0.100-preview.6.21355.2",
"dotnet": "6.0.100",
"runtimes": {
"dotnet": [
"3.1.14"
@ -14,6 +14,6 @@
}
},
"msbuild-sdks": {
"Microsoft.DotNet.Arcade.Sdk": "6.0.0-beta.21379.2"
"Microsoft.DotNet.Arcade.Sdk": "7.0.0-beta.21559.3"
}
}

Loading…
Cancel
Save