Browse Source

[main] Update dependencies from dotnet/arcade (#1141)

[main] Update dependencies from dotnet/arcade


 - Move build tooling to .NET 6 GA to avoid RC1 static web asset build issue.
sebros/macos
dotnet-maestro[bot] 5 years ago
committed by GitHub
parent
commit
995c669219
No known key found for this signature in database GPG Key ID: 4AEE18F83AFDEB23
  1. 20
      eng/Version.Details.xml
  2. 4
      eng/common/build.sh
  3. 74
      eng/common/cross/build-rootfs.sh
  4. 9
      eng/common/cross/toolchain.cmake
  5. 2
      eng/common/init-tools-native.sh
  6. 1
      eng/common/internal/Tools.csproj
  7. 1
      eng/common/msbuild.ps1
  8. 8
      eng/common/native/common-library.sh
  9. 102
      eng/common/native/init-compiler.sh
  10. 30
      eng/common/post-build/sourcelink-validation.ps1
  11. 95
      eng/common/post-build/symbols-validation.ps1
  12. 3
      eng/common/sdk-task.ps1
  13. 6
      eng/common/sdl/configure-sdl-tool.ps1
  14. 6
      eng/common/templates/job/execute-sdl.yml
  15. 2
      eng/common/templates/job/job.yml
  16. 5
      eng/common/templates/job/onelocbuild.yml
  17. 26
      eng/common/templates/job/publish-build-assets.yml
  18. 15
      eng/common/templates/job/source-build.yml
  19. 31
      eng/common/templates/job/source-index-stage1.yml
  20. 4
      eng/common/templates/jobs/jobs.yml
  21. 130
      eng/common/templates/phases/base.yml
  22. 52
      eng/common/templates/phases/publish-build-assets.yml
  23. 3
      eng/common/templates/steps/send-to-helix.yml
  24. 35
      eng/common/templates/steps/source-build.yml
  25. 42
      eng/common/tools.ps1
  26. 16
      eng/common/tools.sh
  27. 4
      global.json

20
eng/Version.Details.xml

@ -3,25 +3,25 @@
<ProductDependencies> <ProductDependencies>
</ProductDependencies> </ProductDependencies>
<ToolsetDependencies> <ToolsetDependencies>
<Dependency Name="Microsoft.DotNet.Arcade.Sdk" Version="6.0.0-beta.21379.2"> <Dependency Name="Microsoft.DotNet.Arcade.Sdk" Version="7.0.0-beta.21559.3">
<Uri>https://github.com/dotnet/arcade</Uri> <Uri>https://github.com/dotnet/arcade</Uri>
<Sha>62a8aafffd4c68ef887680f6837abdff906a662c</Sha> <Sha>fecf65bedcee9036b8ba9d8d7feef5413f294914</Sha>
</Dependency> </Dependency>
<Dependency Name="Microsoft.DotNet.Build.Tasks.Feed" Version="6.0.0-beta.21379.2"> <Dependency Name="Microsoft.DotNet.Build.Tasks.Feed" Version="7.0.0-beta.21559.3">
<Uri>https://github.com/dotnet/arcade</Uri> <Uri>https://github.com/dotnet/arcade</Uri>
<Sha>62a8aafffd4c68ef887680f6837abdff906a662c</Sha> <Sha>fecf65bedcee9036b8ba9d8d7feef5413f294914</Sha>
</Dependency> </Dependency>
<Dependency Name="Microsoft.DotNet.SignTool" Version="6.0.0-beta.21379.2"> <Dependency Name="Microsoft.DotNet.SignTool" Version="7.0.0-beta.21559.3">
<Uri>https://github.com/dotnet/arcade</Uri> <Uri>https://github.com/dotnet/arcade</Uri>
<Sha>62a8aafffd4c68ef887680f6837abdff906a662c</Sha> <Sha>fecf65bedcee9036b8ba9d8d7feef5413f294914</Sha>
</Dependency> </Dependency>
<Dependency Name="Microsoft.DotNet.Helix.Sdk" Version="6.0.0-beta.21379.2"> <Dependency Name="Microsoft.DotNet.Helix.Sdk" Version="7.0.0-beta.21559.3">
<Uri>https://github.com/dotnet/arcade</Uri> <Uri>https://github.com/dotnet/arcade</Uri>
<Sha>62a8aafffd4c68ef887680f6837abdff906a662c</Sha> <Sha>fecf65bedcee9036b8ba9d8d7feef5413f294914</Sha>
</Dependency> </Dependency>
<Dependency Name="Microsoft.DotNet.SwaggerGenerator.MSBuild" Version="6.0.0-beta.21379.2"> <Dependency Name="Microsoft.DotNet.SwaggerGenerator.MSBuild" Version="7.0.0-beta.21559.3">
<Uri>https://github.com/dotnet/arcade</Uri> <Uri>https://github.com/dotnet/arcade</Uri>
<Sha>62a8aafffd4c68ef887680f6837abdff906a662c</Sha> <Sha>fecf65bedcee9036b8ba9d8d7feef5413f294914</Sha>
</Dependency> </Dependency>
<Dependency Name="Microsoft.DotNet.Maestro.Client" Version="1.1.0-beta.19556.4"> <Dependency Name="Microsoft.DotNet.Maestro.Client" Version="1.1.0-beta.19556.4">
<Uri>https://github.com/dotnet/arcade-services</Uri> <Uri>https://github.com/dotnet/arcade-services</Uri>

4
eng/common/build.sh

@ -187,6 +187,10 @@ function InitializeCustomToolset {
} }
function Build { function Build {
if [[ "$ci" == true ]]; then
TryLogClientIpAddress
fi
InitializeToolset InitializeToolset
InitializeCustomToolset InitializeCustomToolset

74
eng/common/cross/build-rootfs.sh

@ -6,10 +6,10 @@ usage()
{ {
echo "Usage: $0 [BuildArch] [CodeName] [lldbx.y] [--skipunmount] --rootfsdir <directory>]" echo "Usage: $0 [BuildArch] [CodeName] [lldbx.y] [--skipunmount] --rootfsdir <directory>]"
echo "BuildArch can be: arm(default), armel, arm64, x86" echo "BuildArch can be: arm(default), armel, arm64, x86"
echo "CodeName - optional, Code name for Linux, can be: trusty, xenial(default), zesty, bionic, alpine, alpine3.9 or alpine3.13. If BuildArch is armel, LinuxCodeName is jessie(default) or tizen." echo "CodeName - optional, Code name for Linux, can be: xenial(default), zesty, bionic, alpine, alpine3.13 or alpine3.14. If BuildArch is armel, LinuxCodeName is jessie(default) or tizen."
echo " for FreeBSD can be: freebsd11 or freebsd12." echo " for FreeBSD can be: freebsd11, freebsd12, freebsd13"
echo " for illumos can be: illumos." echo " for illumos can be: illumos."
echo "lldbx.y - optional, LLDB version, can be: lldb3.9(default), lldb4.0, lldb5.0, lldb6.0 no-lldb. Ignored for alpine and FReeBSD" echo "lldbx.y - optional, LLDB version, can be: lldb3.9(default), lldb4.0, lldb5.0, lldb6.0 no-lldb. Ignored for alpine and FreeBSD"
echo "--skipunmount - optional, will skip the unmount of rootfs folder." echo "--skipunmount - optional, will skip the unmount of rootfs folder."
echo "--use-mirror - optional, use mirror URL to fetch resources, when available." echo "--use-mirror - optional, use mirror URL to fetch resources, when available."
exit 1 exit 1
@ -32,9 +32,9 @@ __UbuntuPackages="build-essential"
__AlpinePackages="alpine-base" __AlpinePackages="alpine-base"
__AlpinePackages+=" build-base" __AlpinePackages+=" build-base"
__AlpinePackages+=" linux-headers" __AlpinePackages+=" linux-headers"
__AlpinePackagesEdgeCommunity=" lldb-dev" __AlpinePackages+=" lldb-dev"
__AlpinePackagesEdgeMain+=" python3" __AlpinePackages+=" python3"
__AlpinePackagesEdgeMain+=" libedit" __AlpinePackages+=" libedit"
# symlinks fixer # symlinks fixer
__UbuntuPackages+=" symlinks" __UbuntuPackages+=" symlinks"
@ -60,13 +60,15 @@ __AlpinePackages+=" krb5-dev"
__AlpinePackages+=" openssl-dev" __AlpinePackages+=" openssl-dev"
__AlpinePackages+=" zlib-dev" __AlpinePackages+=" zlib-dev"
__FreeBSDBase="12.1-RELEASE" __FreeBSDBase="12.2-RELEASE"
__FreeBSDPkg="1.12.0" __FreeBSDPkg="1.12.0"
__FreeBSDABI="12"
__FreeBSDPackages="libunwind" __FreeBSDPackages="libunwind"
__FreeBSDPackages+=" icu" __FreeBSDPackages+=" icu"
__FreeBSDPackages+=" libinotify" __FreeBSDPackages+=" libinotify"
__FreeBSDPackages+=" lttng-ust" __FreeBSDPackages+=" lttng-ust"
__FreeBSDPackages+=" krb5" __FreeBSDPackages+=" krb5"
__FreeBSDPackages+=" terminfo-db"
__IllumosPackages="icu-64.2nb2" __IllumosPackages="icu-64.2nb2"
__IllumosPackages+=" mit-krb5-1.16.2nb4" __IllumosPackages+=" mit-krb5-1.16.2nb4"
@ -144,11 +146,6 @@ while :; do
no-lldb) no-lldb)
unset __LLDB_Package unset __LLDB_Package
;; ;;
trusty) # Ubuntu 14.04
if [ "$__CodeName" != "jessie" ]; then
__CodeName=trusty
fi
;;
xenial) # Ubuntu 16.04 xenial) # Ubuntu 16.04
if [ "$__CodeName" != "jessie" ]; then if [ "$__CodeName" != "jessie" ]; then
__CodeName=xenial __CodeName=xenial
@ -188,32 +185,34 @@ while :; do
__UbuntuRepo= __UbuntuRepo=
__Tizen=tizen __Tizen=tizen
;; ;;
alpine|alpine3.9) alpine|alpine3.13)
__CodeName=alpine __CodeName=alpine
__UbuntuRepo= __UbuntuRepo=
__AlpineVersion=3.9 __AlpineVersion=3.13
__AlpinePackagesEdgeMain+=" llvm11-libs" __AlpinePackages+=" llvm10-libs"
__AlpinePackagesEdgeMain+=" clang-libs"
;; ;;
alpine3.13) alpine3.14)
__CodeName=alpine __CodeName=alpine
__UbuntuRepo= __UbuntuRepo=
__AlpineVersion=3.13 __AlpineVersion=3.14
# Alpine 3.13 has all the packages we need in the 3.13 repository __AlpinePackages+=" llvm11-libs"
__AlpinePackages+=$__AlpinePackagesEdgeCommunity
__AlpinePackagesEdgeCommunity=
__AlpinePackages+=$__AlpinePackagesEdgeMain
__AlpinePackagesEdgeMain=
__AlpinePackages+=" llvm10-libs"
;; ;;
freebsd11) freebsd11)
__FreeBSDBase="11.3-RELEASE" __FreeBSDBase="11.3-RELEASE"
__FreeBSDABI="11"
;& ;&
freebsd12) freebsd12)
__CodeName=freebsd __CodeName=freebsd
__BuildArch=x64 __BuildArch=x64
__SkipUnmount=1 __SkipUnmount=1
;; ;;
freebsd13)
__CodeName=freebsd
__FreeBSDBase="13.0-RELEASE"
__FreeBSDABI="13"
__BuildArch=x64
__SkipUnmount=1
;;
illumos) illumos)
__CodeName=illumos __CodeName=illumos
__BuildArch=x64 __BuildArch=x64
@ -274,26 +273,12 @@ if [[ "$__CodeName" == "alpine" ]]; then
-U --allow-untrusted --root $__RootfsDir --arch $__AlpineArch --initdb \ -U --allow-untrusted --root $__RootfsDir --arch $__AlpineArch --initdb \
add $__AlpinePackages add $__AlpinePackages
if [[ -n "$__AlpinePackagesEdgeMain" ]]; then
$__ApkToolsDir/apk-tools-$__ApkToolsVersion/apk \
-X http://dl-cdn.alpinelinux.org/alpine/edge/main \
-U --allow-untrusted --root $__RootfsDir --arch $__AlpineArch --initdb \
add $__AlpinePackagesEdgeMain
fi
if [[ -n "$__AlpinePackagesEdgeCommunity" ]]; then
$__ApkToolsDir/apk-tools-$__ApkToolsVersion/apk \
-X http://dl-cdn.alpinelinux.org/alpine/edge/community \
-U --allow-untrusted --root $__RootfsDir --arch $__AlpineArch --initdb \
add $__AlpinePackagesEdgeCommunity
fi
rm -r $__ApkToolsDir rm -r $__ApkToolsDir
elif [[ "$__CodeName" == "freebsd" ]]; then elif [[ "$__CodeName" == "freebsd" ]]; then
mkdir -p $__RootfsDir/usr/local/etc mkdir -p $__RootfsDir/usr/local/etc
JOBS="$(getconf _NPROCESSORS_ONLN)"
wget -O - https://download.freebsd.org/ftp/releases/amd64/${__FreeBSDBase}/base.txz | tar -C $__RootfsDir -Jxf - ./lib ./usr/lib ./usr/libdata ./usr/include ./usr/share/keys ./etc ./bin/freebsd-version wget -O - https://download.freebsd.org/ftp/releases/amd64/${__FreeBSDBase}/base.txz | tar -C $__RootfsDir -Jxf - ./lib ./usr/lib ./usr/libdata ./usr/include ./usr/share/keys ./etc ./bin/freebsd-version
# For now, ask for 11 ABI even on 12. This can be revisited later. echo "ABI = \"FreeBSD:${__FreeBSDABI}:amd64\"; FINGERPRINTS = \"${__RootfsDir}/usr/share/keys\"; REPOS_DIR = [\"${__RootfsDir}/etc/pkg\"]; REPO_AUTOUPDATE = NO; RUN_SCRIPTS = NO;" > ${__RootfsDir}/usr/local/etc/pkg.conf
echo "ABI = \"FreeBSD:11:amd64\"; FINGERPRINTS = \"${__RootfsDir}/usr/share/keys\"; REPOS_DIR = [\"${__RootfsDir}/etc/pkg\"]; REPO_AUTOUPDATE = NO; RUN_SCRIPTS = NO;" > ${__RootfsDir}/usr/local/etc/pkg.conf
echo "FreeBSD: { url: "pkg+http://pkg.FreeBSD.org/\${ABI}/quarterly", mirror_type: \"srv\", signature_type: \"fingerprints\", fingerprints: \"${__RootfsDir}/usr/share/keys/pkg\", enabled: yes }" > ${__RootfsDir}/etc/pkg/FreeBSD.conf echo "FreeBSD: { url: "pkg+http://pkg.FreeBSD.org/\${ABI}/quarterly", mirror_type: \"srv\", signature_type: \"fingerprints\", fingerprints: \"${__RootfsDir}/usr/share/keys/pkg\", enabled: yes }" > ${__RootfsDir}/etc/pkg/FreeBSD.conf
mkdir -p $__RootfsDir/tmp mkdir -p $__RootfsDir/tmp
# get and build package manager # get and build package manager
@ -301,7 +286,7 @@ elif [[ "$__CodeName" == "freebsd" ]]; then
cd $__RootfsDir/tmp/pkg-${__FreeBSDPkg} cd $__RootfsDir/tmp/pkg-${__FreeBSDPkg}
# needed for install to succeed # needed for install to succeed
mkdir -p $__RootfsDir/host/etc mkdir -p $__RootfsDir/host/etc
./autogen.sh && ./configure --prefix=$__RootfsDir/host && make && make install ./autogen.sh && ./configure --prefix=$__RootfsDir/host && make -j "$JOBS" && make install
rm -rf $__RootfsDir/tmp/pkg-${__FreeBSDPkg} rm -rf $__RootfsDir/tmp/pkg-${__FreeBSDPkg}
# install packages we need. # install packages we need.
INSTALL_AS_USER=$(whoami) $__RootfsDir/host/sbin/pkg -r $__RootfsDir -C $__RootfsDir/usr/local/etc/pkg.conf update INSTALL_AS_USER=$(whoami) $__RootfsDir/host/sbin/pkg -r $__RootfsDir -C $__RootfsDir/usr/local/etc/pkg.conf update
@ -364,13 +349,6 @@ elif [[ -n $__CodeName ]]; then
umount $__RootfsDir/* || true umount $__RootfsDir/* || true
fi fi
if [[ "$__BuildArch" == "arm" && "$__CodeName" == "trusty" ]]; then
pushd $__RootfsDir
patch -p1 < $__CrossDir/$__BuildArch/trusty.patch
patch -p1 < $__CrossDir/$__BuildArch/trusty-lttng-2.4.patch
popd
fi
if [[ "$__BuildArch" == "armel" && "$__CodeName" == "jessie" ]]; then if [[ "$__BuildArch" == "armel" && "$__CodeName" == "jessie" ]]; then
pushd $__RootfsDir pushd $__RootfsDir
patch -p1 < $__CrossDir/$__BuildArch/armel.jessie.patch patch -p1 < $__CrossDir/$__BuildArch/armel.jessie.patch

9
eng/common/cross/toolchain.cmake

@ -44,7 +44,7 @@ elseif(TARGET_ARCH_NAME STREQUAL "x86")
set(TOOLCHAIN "i686-linux-gnu") set(TOOLCHAIN "i686-linux-gnu")
elseif (CMAKE_SYSTEM_NAME STREQUAL "FreeBSD") elseif (CMAKE_SYSTEM_NAME STREQUAL "FreeBSD")
set(CMAKE_SYSTEM_PROCESSOR "x86_64") set(CMAKE_SYSTEM_PROCESSOR "x86_64")
set(triple "x86_64-unknown-freebsd11") set(triple "x86_64-unknown-freebsd12")
elseif (ILLUMOS) elseif (ILLUMOS)
set(CMAKE_SYSTEM_PROCESSOR "x86_64") set(CMAKE_SYSTEM_PROCESSOR "x86_64")
set(TOOLCHAIN "x86_64-illumos") set(TOOLCHAIN "x86_64-illumos")
@ -91,6 +91,9 @@ elseif(CMAKE_SYSTEM_NAME STREQUAL "FreeBSD")
set(CMAKE_CXX_COMPILER_TARGET ${triple}) set(CMAKE_CXX_COMPILER_TARGET ${triple})
set(CMAKE_ASM_COMPILER_TARGET ${triple}) set(CMAKE_ASM_COMPILER_TARGET ${triple})
set(CMAKE_SYSROOT "${CROSS_ROOTFS}") set(CMAKE_SYSROOT "${CROSS_ROOTFS}")
set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -fuse-ld=lld")
set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} -fuse-ld=lld")
set(CMAKE_MODULE_LINKER_FLAGS "${CMAKE_MODULE_LINKER_FLAGS} -fuse-ld=lld")
elseif(ILLUMOS) elseif(ILLUMOS)
set(CMAKE_SYSROOT "${CROSS_ROOTFS}") set(CMAKE_SYSROOT "${CROSS_ROOTFS}")
@ -138,8 +141,8 @@ function(add_toolchain_linker_flag Flag)
if (NOT Config STREQUAL "") if (NOT Config STREQUAL "")
set(CONFIG_SUFFIX "_${Config}") set(CONFIG_SUFFIX "_${Config}")
endif() endif()
set("CMAKE_EXE_LINKER_FLAGS${CONFIG_SUFFIX}" "${CMAKE_EXE_LINKER_FLAGS${CONFIG_SUFFIX}} ${Flag}" PARENT_SCOPE) set("CMAKE_EXE_LINKER_FLAGS${CONFIG_SUFFIX}_INIT" "${CMAKE_EXE_LINKER_FLAGS${CONFIG_SUFFIX}_INIT} ${Flag}" PARENT_SCOPE)
set("CMAKE_SHARED_LINKER_FLAGS${CONFIG_SUFFIX}" "${CMAKE_SHARED_LINKER_FLAGS${CONFIG_SUFFIX}} ${Flag}" PARENT_SCOPE) set("CMAKE_SHARED_LINKER_FLAGS${CONFIG_SUFFIX}_INIT" "${CMAKE_SHARED_LINKER_FLAGS${CONFIG_SUFFIX}_INIT} ${Flag}" PARENT_SCOPE)
endfunction() endfunction()
if(CMAKE_SYSTEM_NAME STREQUAL "Linux") if(CMAKE_SYSTEM_NAME STREQUAL "Linux")

2
eng/common/init-tools-native.sh

@ -10,7 +10,7 @@ force=false
download_retries=5 download_retries=5
retry_wait_time_seconds=30 retry_wait_time_seconds=30
global_json_file="$(dirname "$(dirname "${scriptroot}")")/global.json" global_json_file="$(dirname "$(dirname "${scriptroot}")")/global.json"
declare -A native_assets declare -a native_assets
. $scriptroot/pipeline-logging-functions.sh . $scriptroot/pipeline-logging-functions.sh
. $scriptroot/native/common-library.sh . $scriptroot/native/common-library.sh

1
eng/common/internal/Tools.csproj

@ -1,5 +1,4 @@
<!-- Licensed to the .NET Foundation under one or more agreements. The .NET Foundation licenses this file to you under the MIT license. --> <!-- Licensed to the .NET Foundation under one or more agreements. The .NET Foundation licenses this file to you under the MIT license. -->
<?xml version="1.0" encoding="utf-8"?>
<Project Sdk="Microsoft.NET.Sdk"> <Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup> <PropertyGroup>
<TargetFramework>net472</TargetFramework> <TargetFramework>net472</TargetFramework>

1
eng/common/msbuild.ps1

@ -6,6 +6,7 @@ Param(
[switch] $ci, [switch] $ci,
[switch] $prepareMachine, [switch] $prepareMachine,
[switch] $excludePrereleaseVS, [switch] $excludePrereleaseVS,
[string] $msbuildEngine = $null,
[Parameter(ValueFromRemainingArguments=$true)][String[]]$extraArgs [Parameter(ValueFromRemainingArguments=$true)][String[]]$extraArgs
) )

8
eng/common/native/common-library.sh

@ -148,8 +148,12 @@ function NewScriptShim {
fi fi
if [[ ! -f $tool_file_path ]]; then if [[ ! -f $tool_file_path ]]; then
Write-PipelineTelemetryError -category 'NativeToolsBootstrap' "Specified tool file path:'$tool_file_path' does not exist" # try to see if the path is lower cased
return 1 tool_file_path="$(echo $tool_file_path | tr "[:upper:]" "[:lower:]")"
if [[ ! -f $tool_file_path ]]; then
Write-PipelineTelemetryError -category 'NativeToolsBootstrap' "Specified tool file path:'$tool_file_path' does not exist"
return 1
fi
fi fi
local shim_contents=$'#!/usr/bin/env bash\n' local shim_contents=$'#!/usr/bin/env bash\n'

102
eng/common/native/find-native-compiler.sh → eng/common/native/init-compiler.sh

@ -1,39 +1,34 @@
#!/usr/bin/env bash #!/usr/bin/env bash
# #
# This file locates the native compiler with the given name and version and sets the environment variables to locate it. # This file detects the C/C++ compiler and exports it to the CC/CXX environment variables
# #
source="${BASH_SOURCE[0]}" if [[ "$#" -lt 3 ]]; then
# resolve $SOURCE until the file is no longer a symlink
while [[ -h $source ]]; do
scriptroot="$( cd -P "$( dirname "$source" )" && pwd )"
source="$(readlink "$source")"
# if $source was a relative symlink, we need to resolve it relative to the path where the
# symlink file was located
[[ $source != /* ]] && source="$scriptroot/$source"
done
scriptroot="$( cd -P "$( dirname "$source" )" && pwd )"
if [ $# -lt 0 ]
then
echo "Usage..." echo "Usage..."
echo "find-native-compiler.sh <compiler> <compiler major version> <compiler minor version>" echo "init-compiler.sh <script directory> <Architecture> <compiler> <compiler major version> <compiler minor version>"
echo "Specify the script directory."
echo "Specify the target architecture."
echo "Specify the name of compiler (clang or gcc)." echo "Specify the name of compiler (clang or gcc)."
echo "Specify the major version of compiler." echo "Specify the major version of compiler."
echo "Specify the minor version of compiler." echo "Specify the minor version of compiler."
exit 1 exit 1
fi fi
. $scriptroot/../pipeline-logging-functions.sh nativescriptroot="$1"
build_arch="$2"
compiler="$1" compiler="$3"
cxxCompiler="$compiler++" cxxCompiler="$compiler++"
majorVersion="$2" majorVersion="$4"
minorVersion="$3" minorVersion="$5"
. "$nativescriptroot"/../pipeline-logging-functions.sh
# clear the existing CC and CXX from environment
CC=
CXX=
LDFLAGS=
if [ "$compiler" = "gcc" ]; then cxxCompiler="g++"; fi if [[ "$compiler" == "gcc" ]]; then cxxCompiler="g++"; fi
check_version_exists() { check_version_exists() {
desired_version=-1 desired_version=-1
@ -50,38 +45,38 @@ check_version_exists() {
echo "$desired_version" echo "$desired_version"
} }
if [ -z "$CLR_CC" ]; then if [[ -z "$CLR_CC" ]]; then
# Set default versions # Set default versions
if [ -z "$majorVersion" ]; then if [[ -z "$majorVersion" ]]; then
# note: gcc (all versions) and clang versions higher than 6 do not have minor version in file name, if it is zero. # note: gcc (all versions) and clang versions higher than 6 do not have minor version in file name, if it is zero.
if [ "$compiler" = "clang" ]; then versions=( 9 8 7 6.0 5.0 4.0 3.9 3.8 3.7 3.6 3.5 ) if [[ "$compiler" == "clang" ]]; then versions=( 13 12 11 10 9 8 7 6.0 5.0 4.0 3.9 3.8 3.7 3.6 3.5 )
elif [ "$compiler" = "gcc" ]; then versions=( 9 8 7 6 5 4.9 ); fi elif [[ "$compiler" == "gcc" ]]; then versions=( 12 11 10 9 8 7 6 5 4.9 ); fi
for version in "${versions[@]}"; do for version in "${versions[@]}"; do
parts=(${version//./ }) parts=(${version//./ })
desired_version="$(check_version_exists "${parts[0]}" "${parts[1]}")" desired_version="$(check_version_exists "${parts[0]}" "${parts[1]}")"
if [ "$desired_version" != "-1" ]; then majorVersion="${parts[0]}"; break; fi if [[ "$desired_version" != "-1" ]]; then majorVersion="${parts[0]}"; break; fi
done done
if [ -z "$majorVersion" ]; then if [[ -z "$majorVersion" ]]; then
if command -v "$compiler" > /dev/null; then if command -v "$compiler" > /dev/null; then
if [ "$(uname)" != "Darwin" ]; then if [[ "$(uname)" != "Darwin" ]]; then
Write-PipelineTelemetryError -category "Build" -type "warning" "Specific version of $compiler not found, falling back to use the one in PATH." Write-PipelineTelemetryError -category "Build" -type "warning" "Specific version of $compiler not found, falling back to use the one in PATH."
fi fi
export CC="$(command -v "$compiler")" CC="$(command -v "$compiler")"
export CXX="$(command -v "$cxxCompiler")" CXX="$(command -v "$cxxCompiler")"
else else
Write-PipelineTelemetryError -category "Build" "No usable version of $compiler found." Write-PipelineTelemetryError -category "Build" "No usable version of $compiler found."
exit 1 exit 1
fi fi
else else
if [ "$compiler" = "clang" ] && [ "$majorVersion" -lt 5 ]; then if [[ "$compiler" == "clang" && "$majorVersion" -lt 5 ]]; then
if [ "$build_arch" = "arm" ] || [ "$build_arch" = "armel" ]; then if [[ "$build_arch" == "arm" || "$build_arch" == "armel" ]]; then
if command -v "$compiler" > /dev/null; then if command -v "$compiler" > /dev/null; then
Write-PipelineTelemetryError -category "Build" -type "warning" "Found clang version $majorVersion which is not supported on arm/armel architectures, falling back to use clang from PATH." Write-PipelineTelemetryError -category "Build" -type "warning" "Found clang version $majorVersion which is not supported on arm/armel architectures, falling back to use clang from PATH."
export CC="$(command -v "$compiler")" CC="$(command -v "$compiler")"
export CXX="$(command -v "$cxxCompiler")" CXX="$(command -v "$cxxCompiler")"
else else
Write-PipelineTelemetryError -category "Build" "Found clang version $majorVersion which is not supported on arm/armel architectures, and there is no clang in PATH." Write-PipelineTelemetryError -category "Build" "Found clang version $majorVersion which is not supported on arm/armel architectures, and there is no clang in PATH."
exit 1 exit 1
@ -91,31 +86,40 @@ if [ -z "$CLR_CC" ]; then
fi fi
else else
desired_version="$(check_version_exists "$majorVersion" "$minorVersion")" desired_version="$(check_version_exists "$majorVersion" "$minorVersion")"
if [ "$desired_version" = "-1" ]; then if [[ "$desired_version" == "-1" ]]; then
Write-PipelineTelemetryError -category "Build" "Could not find specific version of $compiler: $majorVersion $minorVersion." Write-PipelineTelemetryError -category "Build" "Could not find specific version of $compiler: $majorVersion $minorVersion."
exit 1 exit 1
fi fi
fi fi
if [ -z "$CC" ]; then if [[ -z "$CC" ]]; then
export CC="$(command -v "$compiler$desired_version")" CC="$(command -v "$compiler$desired_version")"
export CXX="$(command -v "$cxxCompiler$desired_version")" CXX="$(command -v "$cxxCompiler$desired_version")"
if [ -z "$CXX" ]; then export CXX="$(command -v "$cxxCompiler")"; fi if [[ -z "$CXX" ]]; then CXX="$(command -v "$cxxCompiler")"; fi
fi fi
else else
if [ ! -f "$CLR_CC" ]; then if [[ ! -f "$CLR_CC" ]]; then
Write-PipelineTelemetryError -category "Build" "CLR_CC is set but path '$CLR_CC' does not exist" Write-PipelineTelemetryError -category "Build" "CLR_CC is set but path '$CLR_CC' does not exist"
exit 1 exit 1
fi fi
export CC="$CLR_CC" CC="$CLR_CC"
export CXX="$CLR_CXX" CXX="$CLR_CXX"
fi fi
if [ -z "$CC" ]; then if [[ -z "$CC" ]]; then
Write-PipelineTelemetryError -category "Build" "Unable to find $compiler." Write-PipelineTelemetryError -category "Build" "Unable to find $compiler."
exit 1 exit 1
fi fi
export CCC_CC="$CC" if [[ "$compiler" == "clang" ]]; then
export CCC_CXX="$CXX" if command -v "lld$desired_version" > /dev/null; then
export SCAN_BUILD_COMMAND="$(command -v "scan-build$desired_version")" # Only lld version >= 9 can be considered stable
if [[ "$majorVersion" -ge 9 ]]; then
LDFLAGS="-fuse-ld=lld"
fi
fi
fi
SCAN_BUILD_COMMAND="$(command -v "scan-build$desired_version")"
export CC CXX LDFLAGS SCAN_BUILD_COMMAND

30
eng/common/post-build/sourcelink-validation.ps1

@ -17,10 +17,16 @@ $global:RepoFiles = @{}
$MaxParallelJobs = 16 $MaxParallelJobs = 16
$MaxRetries = 5 $MaxRetries = 5
$RetryWaitTimeInSeconds = 30
# Wait time between check for system load # Wait time between check for system load
$SecondsBetweenLoadChecks = 10 $SecondsBetweenLoadChecks = 10
if (!$InputPath -or !(Test-Path $InputPath)){
Write-Host "No files to validate."
ExitWithExitCode 0
}
$ValidatePackage = { $ValidatePackage = {
param( param(
[string] $PackagePath # Full path to a Symbols.NuGet package [string] $PackagePath # Full path to a Symbols.NuGet package
@ -99,21 +105,25 @@ $ValidatePackage = {
$Status = 200 $Status = 200
$Cache = $using:RepoFiles $Cache = $using:RepoFiles
$totalRetries = 0 $attempts = 0
while ($totalRetries -lt $using:MaxRetries) { while ($attempts -lt $using:MaxRetries) {
if ( !($Cache.ContainsKey($FilePath)) ) { if ( !($Cache.ContainsKey($FilePath)) ) {
try { try {
$Uri = $Link -as [System.URI] $Uri = $Link -as [System.URI]
# Only GitHub links are valid if ($Link -match "submodules") {
if ($Uri.AbsoluteURI -ne $null -and ($Uri.Host -match 'github' -or $Uri.Host -match 'githubusercontent')) { # Skip submodule links until sourcelink properly handles submodules
$Status = 200
}
elseif ($Uri.AbsoluteURI -ne $null -and ($Uri.Host -match 'github' -or $Uri.Host -match 'githubusercontent')) {
# Only GitHub links are valid
$Status = (Invoke-WebRequest -Uri $Link -UseBasicParsing -Method HEAD -TimeoutSec 5).StatusCode $Status = (Invoke-WebRequest -Uri $Link -UseBasicParsing -Method HEAD -TimeoutSec 5).StatusCode
} }
else { else {
# If it's not a github link, we want to break out of the loop and not retry. # If it's not a github link, we want to break out of the loop and not retry.
$Status = 0 $Status = 0
$totalRetries = $using:MaxRetries $attempts = $using:MaxRetries
} }
} }
catch { catch {
@ -123,9 +133,15 @@ $ValidatePackage = {
} }
if ($Status -ne 200) { if ($Status -ne 200) {
$totalRetries++ $attempts++
if ($totalRetries -ge $using:MaxRetries) { if ($attempts -lt $using:MaxRetries)
{
$attemptsLeft = $using:MaxRetries - $attempts
Write-Warning "Download failed, $attemptsLeft attempts remaining, will retry in $using:RetryWaitTimeInSeconds seconds"
Start-Sleep -Seconds $using:RetryWaitTimeInSeconds
}
else {
if ($NumFailedLinks -eq 0) { if ($NumFailedLinks -eq 0) {
if ($FailedFiles.Value -eq 0) { if ($FailedFiles.Value -eq 0) {
Write-Host Write-Host

95
eng/common/post-build/symbols-validation.ps1

@ -4,9 +4,11 @@ param(
[Parameter(Mandatory = $true)][string] $DotnetSymbolVersion, # Version of dotnet symbol to use [Parameter(Mandatory = $true)][string] $DotnetSymbolVersion, # Version of dotnet symbol to use
[Parameter(Mandatory = $false)][switch] $CheckForWindowsPdbs, # If we should check for the existence of windows pdbs in addition to portable PDBs [Parameter(Mandatory = $false)][switch] $CheckForWindowsPdbs, # If we should check for the existence of windows pdbs in addition to portable PDBs
[Parameter(Mandatory = $false)][switch] $ContinueOnError, # If we should keep checking symbols after an error [Parameter(Mandatory = $false)][switch] $ContinueOnError, # If we should keep checking symbols after an error
[Parameter(Mandatory = $false)][switch] $Clean # Clean extracted symbols directory after checking symbols [Parameter(Mandatory = $false)][switch] $Clean, # Clean extracted symbols directory after checking symbols
[Parameter(Mandatory = $false)][string] $SymbolExclusionFile # Exclude the symbols in the file from publishing to symbol server
) )
. $PSScriptRoot\..\tools.ps1
# Maximum number of jobs to run in parallel # Maximum number of jobs to run in parallel
$MaxParallelJobs = 16 $MaxParallelJobs = 16
@ -25,14 +27,28 @@ if ($CheckForWindowsPdbs) {
$WindowsPdbVerificationParam = "--windows-pdbs" $WindowsPdbVerificationParam = "--windows-pdbs"
} }
$ExclusionSet = New-Object System.Collections.Generic.HashSet[string];
if (!$InputPath -or !(Test-Path $InputPath)){
Write-Host "No symbols to validate."
ExitWithExitCode 0
}
#Check if the path exists
if ($SymbolExclusionFile -and (Test-Path $SymbolExclusionFile)){
[string[]]$Exclusions = Get-Content "$SymbolExclusionFile"
$Exclusions | foreach { if($_ -and $_.Trim()){$ExclusionSet.Add($_)} }
}
else{
Write-Host "Symbol Exclusion file does not exists. No symbols to exclude."
}
$CountMissingSymbols = { $CountMissingSymbols = {
param( param(
[string] $PackagePath, # Path to a NuGet package [string] $PackagePath, # Path to a NuGet package
[string] $WindowsPdbVerificationParam # If we should check for the existence of windows pdbs in addition to portable PDBs [string] $WindowsPdbVerificationParam # If we should check for the existence of windows pdbs in addition to portable PDBs
) )
. $using:PSScriptRoot\..\tools.ps1
Add-Type -AssemblyName System.IO.Compression.FileSystem Add-Type -AssemblyName System.IO.Compression.FileSystem
Write-Host "Validating $PackagePath " Write-Host "Validating $PackagePath "
@ -118,17 +134,17 @@ $CountMissingSymbols = {
# Save the output and get diagnostic output # Save the output and get diagnostic output
$output = & $dotnetSymbolExe --symbols --modules $WindowsPdbVerificationParam $TargetServerParam $FullPath -o $SymbolsPath --diagnostics | Out-String $output = & $dotnetSymbolExe --symbols --modules $WindowsPdbVerificationParam $TargetServerParam $FullPath -o $SymbolsPath --diagnostics | Out-String
if (Test-Path $PdbPath) { if ((Test-Path $PdbPath) -and (Test-path $SymbolPath)) {
return 'PDB' return 'Module and PDB for Module'
} }
elseif (Test-Path $NGenPdb) { elseif ((Test-Path $NGenPdb) -and (Test-Path $PdbPath) -and (Test-Path $SymbolPath)) {
return 'NGen PDB' return 'Dll, PDB and NGen PDB'
} }
elseif (Test-Path $SODbg) { elseif ((Test-Path $SODbg) -and (Test-Path $SymbolPath)) {
return 'DBG for SO' return 'So and DBG for SO'
} }
elseif (Test-Path $DylibDwarf) { elseif ((Test-Path $DylibDwarf) -and (Test-Path $SymbolPath)) {
return 'Dwarf for Dylib' return 'Dylib and Dwarf for Dylib'
} }
elseif (Test-Path $SymbolPath) { elseif (Test-Path $SymbolPath) {
return 'Module' return 'Module'
@ -142,37 +158,44 @@ $CountMissingSymbols = {
return $null return $null
} }
$FileGuid = New-Guid $FileRelativePath = $FileName.Replace("$ExtractPath\", "")
$ExpandedSymbolsPath = Join-Path -Path $SymbolsPath -ChildPath $FileGuid if (($($using:ExclusionSet) -ne $null) -and ($($using:ExclusionSet).Contains($FileRelativePath) -or ($($using:ExclusionSet).Contains($FileRelativePath.Replace("\", "/"))))){
Write-Host "Skipping $FileName from symbol validation"
$SymbolsOnMSDL = & $FirstMatchingSymbolDescriptionOrDefault `
-FullPath $FileName `
-TargetServerParam '--microsoft-symbol-server' `
-SymbolsPath "$ExpandedSymbolsPath-msdl" `
-WindowsPdbVerificationParam $WindowsPdbVerificationParam
$SymbolsOnSymWeb = & $FirstMatchingSymbolDescriptionOrDefault `
-FullPath $FileName `
-TargetServerParam '--internal-server' `
-SymbolsPath "$ExpandedSymbolsPath-symweb" `
-WindowsPdbVerificationParam $WindowsPdbVerificationParam
Write-Host -NoNewLine "`t Checking file " $FileName "... "
if ($SymbolsOnMSDL -ne $null -and $SymbolsOnSymWeb -ne $null) {
Write-Host "Symbols found on MSDL ($SymbolsOnMSDL) and SymWeb ($SymbolsOnSymWeb)"
} }
else {
$MissingSymbols++
if ($SymbolsOnMSDL -eq $null -and $SymbolsOnSymWeb -eq $null) { else {
Write-Host 'No symbols found on MSDL or SymWeb!' $FileGuid = New-Guid
$ExpandedSymbolsPath = Join-Path -Path $SymbolsPath -ChildPath $FileGuid
$SymbolsOnMSDL = & $FirstMatchingSymbolDescriptionOrDefault `
-FullPath $FileName `
-TargetServerParam '--microsoft-symbol-server' `
-SymbolsPath "$ExpandedSymbolsPath-msdl" `
-WindowsPdbVerificationParam $WindowsPdbVerificationParam
$SymbolsOnSymWeb = & $FirstMatchingSymbolDescriptionOrDefault `
-FullPath $FileName `
-TargetServerParam '--internal-server' `
-SymbolsPath "$ExpandedSymbolsPath-symweb" `
-WindowsPdbVerificationParam $WindowsPdbVerificationParam
Write-Host -NoNewLine "`t Checking file " $FileName "... "
if ($SymbolsOnMSDL -ne $null -and $SymbolsOnSymWeb -ne $null) {
Write-Host "Symbols found on MSDL ($SymbolsOnMSDL) and SymWeb ($SymbolsOnSymWeb)"
} }
else { else {
if ($SymbolsOnMSDL -eq $null) { $MissingSymbols++
Write-Host 'No symbols found on MSDL!'
if ($SymbolsOnMSDL -eq $null -and $SymbolsOnSymWeb -eq $null) {
Write-Host 'No symbols found on MSDL or SymWeb!'
} }
else { else {
Write-Host 'No symbols found on SymWeb!' if ($SymbolsOnMSDL -eq $null) {
Write-Host 'No symbols found on MSDL!'
}
else {
Write-Host 'No symbols found on SymWeb!'
}
} }
} }
} }

3
eng/common/sdk-task.ps1

@ -83,6 +83,9 @@ try {
} }
if ($restore) { if ($restore) {
if ($ci) {
Try-LogClientIpAddress
}
Build 'Restore' Build 'Restore'
} }

6
eng/common/sdl/configure-sdl-tool.ps1

@ -69,13 +69,13 @@ try {
# For some tools, add default and automatic args. # For some tools, add default and automatic args.
if ($tool.Name -eq 'credscan') { if ($tool.Name -eq 'credscan') {
if ($targetDirectory) { if ($targetDirectory) {
$tool.Args += "TargetDirectory < $TargetDirectory" $tool.Args += "`"TargetDirectory < $TargetDirectory`""
} }
$tool.Args += "OutputType < pre" $tool.Args += "`"OutputType < pre`""
$tool.Args += $CrScanAdditionalRunConfigParams $tool.Args += $CrScanAdditionalRunConfigParams
} elseif ($tool.Name -eq 'policheck') { } elseif ($tool.Name -eq 'policheck') {
if ($targetDirectory) { if ($targetDirectory) {
$tool.Args += "Target < $TargetDirectory" $tool.Args += "`"Target < $TargetDirectory`""
} }
$tool.Args += $PoliCheckAdditionalRunConfigParams $tool.Args += $PoliCheckAdditionalRunConfigParams
} }

6
eng/common/templates/job/execute-sdl.yml

@ -60,11 +60,7 @@ jobs:
- name: GuardianPackagesConfigFile - name: GuardianPackagesConfigFile
value: $(Build.SourcesDirectory)\eng\common\sdl\packages.config value: $(Build.SourcesDirectory)\eng\common\sdl\packages.config
pool: pool:
# To extract archives (.tar.gz, .zip), we need access to "tar", added in Windows 10/2019. vmImage: windows-2019
${{ if eq(parameters.extractArchiveArtifacts, 'false') }}:
name: Hosted VS2017
${{ if ne(parameters.extractArchiveArtifacts, 'false') }}:
vmImage: windows-2019
steps: steps:
- checkout: self - checkout: self
clean: true clean: true

2
eng/common/templates/job/job.yml

@ -103,7 +103,7 @@ jobs:
- ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}:
- ${{ if eq(parameters.enableMicrobuild, 'true') }}: - ${{ if eq(parameters.enableMicrobuild, 'true') }}:
- task: MicroBuildSigningPlugin@2 - task: MicroBuildSigningPlugin@3
displayName: Install MicroBuild plugin displayName: Install MicroBuild plugin
inputs: inputs:
signType: $(_SignType) signType: $(_SignType)

5
eng/common/templates/job/onelocbuild.yml

@ -4,7 +4,7 @@ parameters:
# Optional: A defined YAML pool - https://docs.microsoft.com/en-us/azure/devops/pipelines/yaml-schema?view=vsts&tabs=schema#pool # Optional: A defined YAML pool - https://docs.microsoft.com/en-us/azure/devops/pipelines/yaml-schema?view=vsts&tabs=schema#pool
pool: pool:
vmImage: vs2017-win2016 vmImage: 'windows-2019'
CeapexPat: $(dn-bot-ceapex-package-r) # PAT for the loc AzDO instance https://dev.azure.com/ceapex CeapexPat: $(dn-bot-ceapex-package-r) # PAT for the loc AzDO instance https://dev.azure.com/ceapex
GithubPat: $(BotAccount-dotnet-bot-repo-PAT) GithubPat: $(BotAccount-dotnet-bot-repo-PAT)
@ -12,6 +12,7 @@ parameters:
SourcesDirectory: $(Build.SourcesDirectory) SourcesDirectory: $(Build.SourcesDirectory)
CreatePr: true CreatePr: true
AutoCompletePr: false AutoCompletePr: false
ReusePr: true
UseLfLineEndings: true UseLfLineEndings: true
UseCheckedInLocProjectJson: false UseCheckedInLocProjectJson: false
LanguageSet: VS_Main_Languages LanguageSet: VS_Main_Languages
@ -64,6 +65,8 @@ jobs:
${{ if eq(parameters.CreatePr, true) }}: ${{ if eq(parameters.CreatePr, true) }}:
isAutoCompletePrSelected: ${{ parameters.AutoCompletePr }} isAutoCompletePrSelected: ${{ parameters.AutoCompletePr }}
isUseLfLineEndingsSelected: ${{ parameters.UseLfLineEndings }} isUseLfLineEndingsSelected: ${{ parameters.UseLfLineEndings }}
${{ if eq(parameters.RepoType, 'gitHub') }}:
isShouldReusePrSelected: ${{ parameters.ReusePr }}
packageSourceAuth: patAuth packageSourceAuth: patAuth
patVariable: ${{ parameters.CeapexPat }} patVariable: ${{ parameters.CeapexPat }}
${{ if eq(parameters.RepoType, 'gitHub') }}: ${{ if eq(parameters.RepoType, 'gitHub') }}:

26
eng/common/templates/job/publish-build-assets.yml

@ -94,7 +94,31 @@ jobs:
PathtoPublish: '$(Build.StagingDirectory)/ReleaseConfigs.txt' PathtoPublish: '$(Build.StagingDirectory)/ReleaseConfigs.txt'
PublishLocation: Container PublishLocation: Container
ArtifactName: ReleaseConfigs ArtifactName: ReleaseConfigs
- task: powershell@2
displayName: Check if SymbolPublishingExclusionsFile.txt exists
inputs:
targetType: inline
script: |
$symbolExclusionfile = "$(Build.SourcesDirectory)/eng/SymbolPublishingExclusionsFile.txt"
if(Test-Path -Path $symbolExclusionfile)
{
Write-Host "SymbolExclusionFile exists"
Write-Host "##vso[task.setvariable variable=SymbolExclusionFile]true"
}
else{
Write-Host "Symbols Exclusion file does not exists"
Write-Host "##vso[task.setvariable variable=SymbolExclusionFile]false"
}
- task: PublishBuildArtifacts@1
displayName: Publish SymbolPublishingExclusionsFile Artifact
condition: eq(variables['SymbolExclusionFile'], 'true')
inputs:
PathtoPublish: '$(Build.SourcesDirectory)/eng/SymbolPublishingExclusionsFile.txt'
PublishLocation: Container
ArtifactName: ReleaseConfigs
- ${{ if eq(parameters.enablePublishBuildArtifacts, 'true') }}: - ${{ if eq(parameters.enablePublishBuildArtifacts, 'true') }}:
- template: /eng/common/templates/steps/publish-logs.yml - template: /eng/common/templates/steps/publish-logs.yml
parameters: parameters:

15
eng/common/templates/job/source-build.yml

@ -31,11 +31,6 @@ parameters:
# container and pool. # container and pool.
platform: {} platform: {}
# The default VM host AzDO pool. This should be capable of running Docker containers: almost all
# source-build builds run in Docker, including the default managed platform.
defaultContainerHostPool:
vmImage: ubuntu-20.04
jobs: jobs:
- job: ${{ parameters.jobNamePrefix }}_${{ parameters.platform.name }} - job: ${{ parameters.jobNamePrefix }}_${{ parameters.platform.name }}
displayName: Source-Build (${{ parameters.platform.name }}) displayName: Source-Build (${{ parameters.platform.name }})
@ -47,7 +42,15 @@ jobs:
container: ${{ parameters.platform.container }} container: ${{ parameters.platform.container }}
${{ if eq(parameters.platform.pool, '') }}: ${{ if eq(parameters.platform.pool, '') }}:
pool: ${{ parameters.defaultContainerHostPool }} # The default VM host AzDO pool. This should be capable of running Docker containers: almost all
# source-build builds run in Docker, including the default managed platform.
pool:
${{ if eq(variables['System.TeamProject'], 'public') }}:
name: NetCore1ESPool-Public
demands: ImageOverride -equals Build.Ubuntu.1804.Amd64.Open
${{ if eq(variables['System.TeamProject'], 'internal') }}:
name: NetCore1ESPool-Internal
demands: ImageOverride -equals Build.Ubuntu.1804.Amd64
${{ if ne(parameters.platform.pool, '') }}: ${{ if ne(parameters.platform.pool, '') }}:
pool: ${{ parameters.platform.pool }} pool: ${{ parameters.platform.pool }}

31
eng/common/templates/job/source-index-stage1.yml

@ -5,8 +5,6 @@ parameters:
sourceIndexBuildCommand: powershell -NoLogo -NoProfile -ExecutionPolicy Bypass -Command "eng/common/build.ps1 -restore -build -binarylog -ci" sourceIndexBuildCommand: powershell -NoLogo -NoProfile -ExecutionPolicy Bypass -Command "eng/common/build.ps1 -restore -build -binarylog -ci"
preSteps: [] preSteps: []
binlogPath: artifacts/log/Debug/Build.binlog binlogPath: artifacts/log/Debug/Build.binlog
pool:
vmImage: vs2017-win2016
condition: '' condition: ''
dependsOn: '' dependsOn: ''
@ -24,7 +22,13 @@ jobs:
- ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}:
- group: source-dot-net stage1 variables - group: source-dot-net stage1 variables
pool: ${{ parameters.pool }} pool:
${{ if eq(variables['System.TeamProject'], 'public') }}:
name: NetCore1ESPool-Public
demands: ImageOverride -equals Build.Server.Amd64.VS2019.Open
${{ if eq(variables['System.TeamProject'], 'internal') }}:
name: NetCore1ESPool-Internal
demands: ImageOverride -equals Build.Server.Amd64.VS2019
steps: steps:
- ${{ each preStep in parameters.preSteps }}: - ${{ each preStep in parameters.preSteps }}:
- ${{ preStep }} - ${{ preStep }}
@ -34,29 +38,24 @@ jobs:
inputs: inputs:
packageType: sdk packageType: sdk
version: 3.1.x version: 3.1.x
installationPath: $(Agent.TempDirectory)/dotnet
- task: UseDotNet@2 workingDirectory: $(Agent.TempDirectory)
displayName: Use .NET Core sdk
inputs:
useGlobalJson: true
- script: | - script: |
dotnet tool install BinLogToSln --version $(SourceIndexPackageVersion) --add-source $(SourceIndexPackageSource) --tool-path .source-index/tools $(Agent.TempDirectory)/dotnet/dotnet tool install BinLogToSln --version $(SourceIndexPackageVersion) --add-source $(SourceIndexPackageSource) --tool-path $(Agent.TempDirectory)/.source-index/tools
dotnet tool install UploadIndexStage1 --version $(SourceIndexPackageVersion) --add-source $(SourceIndexPackageSource) --tool-path .source-index/tools $(Agent.TempDirectory)/dotnet/dotnet tool install UploadIndexStage1 --version $(SourceIndexPackageVersion) --add-source $(SourceIndexPackageSource) --tool-path $(Agent.TempDirectory)/.source-index/tools
echo ##vso[task.prependpath]$(Build.SourcesDirectory)/.source-index/tools
displayName: Download Tools displayName: Download Tools
# Set working directory to temp directory so 'dotnet' doesn't try to use global.json and use the repo's sdk.
workingDirectory: $(Agent.TempDirectory)
- script: ${{ parameters.sourceIndexBuildCommand }} - script: ${{ parameters.sourceIndexBuildCommand }}
displayName: Build Repository displayName: Build Repository
- script: BinLogToSln -i $(BinlogPath) -r $(Build.SourcesDirectory) -n $(Build.Repository.Name) -o .source-index/stage1output - script: $(Agent.TempDirectory)/.source-index/tools/BinLogToSln -i $(BinlogPath) -r $(Build.SourcesDirectory) -n $(Build.Repository.Name) -o .source-index/stage1output
displayName: Process Binlog into indexable sln displayName: Process Binlog into indexable sln
env:
DOTNET_ROLL_FORWARD_ON_NO_CANDIDATE_FX: 2
- ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}:
- script: UploadIndexStage1 -i .source-index/stage1output -n $(Build.Repository.Name) - script: $(Agent.TempDirectory)/.source-index/tools/UploadIndexStage1 -i .source-index/stage1output -n $(Build.Repository.Name)
displayName: Upload stage1 artifacts to source index displayName: Upload stage1 artifacts to source index
env: env:
BLOB_CONTAINER_URL: $(source-dot-net-stage1-blob-container-url) BLOB_CONTAINER_URL: $(source-dot-net-stage1-blob-container-url)
DOTNET_ROLL_FORWARD_ON_NO_CANDIDATE_FX: 2

4
eng/common/templates/jobs/jobs.yml

@ -83,7 +83,7 @@ jobs:
- ${{ if eq(parameters.enableSourceBuild, true) }}: - ${{ if eq(parameters.enableSourceBuild, true) }}:
- Source_Build_Complete - Source_Build_Complete
pool: pool:
vmImage: vs2017-win2016 vmImage: 'windows-2019'
runAsPublic: ${{ parameters.runAsPublic }} runAsPublic: ${{ parameters.runAsPublic }}
publishUsingPipelines: ${{ parameters.enablePublishUsingPipelines }} publishUsingPipelines: ${{ parameters.enablePublishUsingPipelines }}
enablePublishBuildArtifacts: ${{ parameters.enablePublishBuildArtifacts }} enablePublishBuildArtifacts: ${{ parameters.enablePublishBuildArtifacts }}
@ -96,4 +96,4 @@ jobs:
dependsOn: dependsOn:
- Asset_Registry_Publish - Asset_Registry_Publish
pool: pool:
vmImage: vs2017-win2016 vmImage: 'windows-2019'

130
eng/common/templates/phases/base.yml

@ -1,130 +0,0 @@
parameters:
# Optional: Clean sources before building
clean: true
# Optional: Git fetch depth
fetchDepth: ''
# Optional: name of the phase (not specifying phase name may cause name collisions)
name: ''
# Optional: display name of the phase
displayName: ''
# Optional: condition for the job to run
condition: ''
# Optional: dependencies of the phase
dependsOn: ''
# Required: A defined YAML queue
queue: {}
# Required: build steps
steps: []
# Optional: variables
variables: {}
# Optional: should run as a public build even in the internal project
# if 'true', the build won't run any of the internal only steps, even if it is running in non-public projects.
runAsPublic: false
## Telemetry variables
# Optional: enable sending telemetry
# if 'true', these "variables" must be specified in the variables object or as part of the queue matrix
# _HelixBuildConfig - differentiate between Debug, Release, other
# _HelixSource - Example: build/product
# _HelixType - Example: official/dotnet/arcade/$(Build.SourceBranch)
enableTelemetry: false
# Optional: Enable installing Microbuild plugin
# if 'true', these "variables" must be specified in the variables object or as part of the queue matrix
# _TeamName - the name of your team
# _SignType - 'test' or 'real'
enableMicrobuild: false
# Internal resources (telemetry, microbuild) can only be accessed from non-public projects,
# and some (Microbuild) should only be applied to non-PR cases for internal builds.
phases:
- phase: ${{ parameters.name }}
${{ if ne(parameters.displayName, '') }}:
displayName: ${{ parameters.displayName }}
${{ if ne(parameters.condition, '') }}:
condition: ${{ parameters.condition }}
${{ if ne(parameters.dependsOn, '') }}:
dependsOn: ${{ parameters.dependsOn }}
queue: ${{ parameters.queue }}
${{ if ne(parameters.variables, '') }}:
variables:
${{ insert }}: ${{ parameters.variables }}
steps:
- checkout: self
clean: ${{ parameters.clean }}
${{ if ne(parameters.fetchDepth, '') }}:
fetchDepth: ${{ parameters.fetchDepth }}
- ${{ if eq(parameters.enableTelemetry, 'true') }}:
- template: /eng/common/templates/steps/telemetry-start.yml
parameters:
buildConfig: $(_HelixBuildConfig)
helixSource: $(_HelixSource)
helixType: $(_HelixType)
runAsPublic: ${{ parameters.runAsPublic }}
- ${{ if eq(parameters.enableMicrobuild, 'true') }}:
# Internal only resource, and Microbuild signing shouldn't be applied to PRs.
- ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}:
- task: MicroBuildSigningPlugin@2
displayName: Install MicroBuild plugin
inputs:
signType: $(_SignType)
zipSources: false
feedSource: https://dnceng.pkgs.visualstudio.com/_packaging/MicroBuildToolset/nuget/v3/index.json
env:
TeamName: $(_TeamName)
continueOnError: false
condition: and(succeeded(), in(variables['_SignType'], 'real', 'test'), eq(variables['Agent.Os'], 'Windows_NT'))
# Run provided build steps
- ${{ parameters.steps }}
- ${{ if eq(parameters.enableMicrobuild, 'true') }}:
# Internal only resources
- ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}:
- task: MicroBuildCleanup@1
displayName: Execute Microbuild cleanup tasks
condition: and(always(), in(variables['_SignType'], 'real', 'test'), eq(variables['Agent.Os'], 'Windows_NT'))
env:
TeamName: $(_TeamName)
- ${{ if eq(parameters.enableTelemetry, 'true') }}:
- template: /eng/common/templates/steps/telemetry-end.yml
parameters:
helixSource: $(_HelixSource)
helixType: $(_HelixType)
- ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}:
- task: CopyFiles@2
displayName: Gather Asset Manifests
inputs:
SourceFolder: '$(Build.SourcesDirectory)/artifacts/log/$(_BuildConfig)/AssetManifest'
TargetFolder: '$(Build.StagingDirectory)/AssetManifests'
continueOnError: false
condition: and(succeeded(), eq(variables['_DotNetPublishToBlobFeed'], 'true'))
- task: PublishBuildArtifacts@1
displayName: Push Asset Manifests
inputs:
PathtoPublish: '$(Build.StagingDirectory)/AssetManifests'
PublishLocation: Container
ArtifactName: AssetManifests
continueOnError: false
condition: and(succeeded(), eq(variables['_DotNetPublishToBlobFeed'], 'true'))

52
eng/common/templates/phases/publish-build-assets.yml

@ -1,52 +0,0 @@
parameters:
dependsOn: ''
queue: {}
configuration: 'Debug'
condition: succeeded()
continueOnError: false
runAsPublic: false
publishUsingPipelines: false
phases:
- phase: Asset_Registry_Publish
displayName: Publish to Build Asset Registry
dependsOn: ${{ parameters.dependsOn }}
queue: ${{ parameters.queue }}
variables:
_BuildConfig: ${{ parameters.configuration }}
steps:
- ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}:
- task: DownloadBuildArtifacts@0
displayName: Download artifact
inputs:
artifactName: AssetManifests
downloadPath: '$(Build.StagingDirectory)/Download'
checkDownloadedFiles: true
condition: ${{ parameters.condition }}
continueOnError: ${{ parameters.continueOnError }}
- task: AzureKeyVault@1
inputs:
azureSubscription: 'DotNet-Engineering-Services_KeyVault'
KeyVaultName: EngKeyVault
SecretsFilter: 'MaestroAccessToken'
condition: ${{ parameters.condition }}
continueOnError: ${{ parameters.continueOnError }}
- task: PowerShell@2
displayName: Publish Build Assets
inputs:
filePath: eng\common\sdk-task.ps1
arguments: -task PublishBuildAssets -restore -msbuildEngine dotnet
/p:ManifestsPath='$(Build.StagingDirectory)/Download/AssetManifests'
/p:BuildAssetRegistryToken=$(MaestroAccessToken)
/p:MaestroApiEndpoint=https://maestro-prod.westus2.cloudapp.azure.com
/p:PublishUsingPipelines=${{ parameters.publishUsingPipelines }}
/p:Configuration=$(_BuildConfig)
condition: ${{ parameters.condition }}
continueOnError: ${{ parameters.continueOnError }}
- task: PublishBuildArtifacts@1
displayName: Publish Logs to VSTS
inputs:
PathtoPublish: '$(Build.SourcesDirectory)/artifacts/log/$(_BuildConfig)'
PublishLocation: Container
ArtifactName: $(Agent.Os)_Asset_Registry_Publish
continueOnError: true
condition: always()

3
eng/common/templates/steps/send-to-helix.yml

@ -20,7 +20,6 @@ parameters:
IncludeDotNetCli: false # optional -- true will download a version of the .NET CLI onto the Helix machine as a correlation payload; requires DotNetCliPackageType and DotNetCliVersion IncludeDotNetCli: false # optional -- true will download a version of the .NET CLI onto the Helix machine as a correlation payload; requires DotNetCliPackageType and DotNetCliVersion
DotNetCliPackageType: '' # optional -- either 'sdk', 'runtime' or 'aspnetcore-runtime'; determines whether the sdk or runtime will be sent to Helix; see https://raw.githubusercontent.com/dotnet/core/main/release-notes/releases-index.json DotNetCliPackageType: '' # optional -- either 'sdk', 'runtime' or 'aspnetcore-runtime'; determines whether the sdk or runtime will be sent to Helix; see https://raw.githubusercontent.com/dotnet/core/main/release-notes/releases-index.json
DotNetCliVersion: '' # optional -- version of the CLI to send to Helix; based on this: https://raw.githubusercontent.com/dotnet/core/main/release-notes/releases-index.json DotNetCliVersion: '' # optional -- version of the CLI to send to Helix; based on this: https://raw.githubusercontent.com/dotnet/core/main/release-notes/releases-index.json
EnableXUnitReporter: false # optional -- true enables XUnit result reporting to Mission Control
WaitForWorkItemCompletion: true # optional -- true will make the task wait until work items have been completed and fail the build if work items fail. False is "fire and forget." WaitForWorkItemCompletion: true # optional -- true will make the task wait until work items have been completed and fail the build if work items fail. False is "fire and forget."
IsExternal: false # [DEPRECATED] -- doesn't do anything, jobs are external if HelixAccessToken is empty and Creator is set IsExternal: false # [DEPRECATED] -- doesn't do anything, jobs are external if HelixAccessToken is empty and Creator is set
HelixBaseUri: 'https://helix.dot.net/' # optional -- sets the Helix API base URI (allows targeting int) HelixBaseUri: 'https://helix.dot.net/' # optional -- sets the Helix API base URI (allows targeting int)
@ -54,7 +53,6 @@ steps:
IncludeDotNetCli: ${{ parameters.IncludeDotNetCli }} IncludeDotNetCli: ${{ parameters.IncludeDotNetCli }}
DotNetCliPackageType: ${{ parameters.DotNetCliPackageType }} DotNetCliPackageType: ${{ parameters.DotNetCliPackageType }}
DotNetCliVersion: ${{ parameters.DotNetCliVersion }} DotNetCliVersion: ${{ parameters.DotNetCliVersion }}
EnableXUnitReporter: ${{ parameters.EnableXUnitReporter }}
WaitForWorkItemCompletion: ${{ parameters.WaitForWorkItemCompletion }} WaitForWorkItemCompletion: ${{ parameters.WaitForWorkItemCompletion }}
HelixBaseUri: ${{ parameters.HelixBaseUri }} HelixBaseUri: ${{ parameters.HelixBaseUri }}
Creator: ${{ parameters.Creator }} Creator: ${{ parameters.Creator }}
@ -85,7 +83,6 @@ steps:
IncludeDotNetCli: ${{ parameters.IncludeDotNetCli }} IncludeDotNetCli: ${{ parameters.IncludeDotNetCli }}
DotNetCliPackageType: ${{ parameters.DotNetCliPackageType }} DotNetCliPackageType: ${{ parameters.DotNetCliPackageType }}
DotNetCliVersion: ${{ parameters.DotNetCliVersion }} DotNetCliVersion: ${{ parameters.DotNetCliVersion }}
EnableXUnitReporter: ${{ parameters.EnableXUnitReporter }}
WaitForWorkItemCompletion: ${{ parameters.WaitForWorkItemCompletion }} WaitForWorkItemCompletion: ${{ parameters.WaitForWorkItemCompletion }}
HelixBaseUri: ${{ parameters.HelixBaseUri }} HelixBaseUri: ${{ parameters.HelixBaseUri }}
Creator: ${{ parameters.Creator }} Creator: ${{ parameters.Creator }}

35
eng/common/templates/steps/source-build.yml

@ -18,6 +18,35 @@ steps:
set -x set -x
df -h df -h
# If building on the internal project, the artifact feeds variable may be available (usually only if needed)
# In that case, call the feed setup script to add internal feeds corresponding to public ones.
# In addition, add an msbuild argument to copy the WIP from the repo to the target build location.
# This is because SetupNuGetSources.sh will alter the current NuGet.config file, and we need to preserve those
# changes.
$internalRestoreArgs=
if [ '$(dn-bot-dnceng-artifact-feeds-rw)' != '$''(dn-bot-dnceng-artifact-feeds-rw)' ]; then
# Temporarily work around https://github.com/dotnet/arcade/issues/7709
chmod +x $(Build.SourcesDirectory)/eng/common/SetupNugetSources.sh
$(Build.SourcesDirectory)/eng/common/SetupNugetSources.sh $(Build.SourcesDirectory)/NuGet.config $(dn-bot-dnceng-artifact-feeds-rw)
internalRestoreArgs='/p:CopyWipIntoInnerSourceBuildRepo=true'
# The 'Copy WIP' feature of source build uses git stash to apply changes from the original repo.
# This only works if there is a username/email configured, which won't be the case in most CI runs.
git config --get user.email
if [ $? -ne 0 ]; then
git config user.email dn-bot@microsoft.com
git config user.name dn-bot
fi
fi
# If building on the internal project, the internal storage variable may be available (usually only if needed)
# In that case, add variables to allow the download of internal runtimes if the specified versions are not found
# in the default public locations.
internalRuntimeDownloadArgs=
if [ '$(dotnetclimsrc-read-sas-token-base64)' != '$''(dotnetclimsrc-read-sas-token-base64)' ]; then
internalRuntimeDownloadArgs='/p:DotNetRuntimeSourceFeed=https://dotnetclimsrc.blob.core.windows.net/dotnet /p:DotNetRuntimeSourceFeedKey=$(dotnetclimsrc-read-sas-token-base64) --runtimesourcefeed https://dotnetclimsrc.blob.core.windows.net/dotnet --runtimesourcefeedkey $(dotnetclimsrc-read-sas-token-base64)'
fi
buildConfig=Release buildConfig=Release
# Check if AzDO substitutes in a build config from a variable, and use it if so. # Check if AzDO substitutes in a build config from a variable, and use it if so.
if [ '$(_BuildConfig)' != '$''(_BuildConfig)' ]; then if [ '$(_BuildConfig)' != '$''(_BuildConfig)' ]; then
@ -29,11 +58,6 @@ steps:
officialBuildArgs='/p:DotNetPublishUsingPipelines=true /p:OfficialBuildId=$(BUILD.BUILDNUMBER)' officialBuildArgs='/p:DotNetPublishUsingPipelines=true /p:OfficialBuildId=$(BUILD.BUILDNUMBER)'
fi fi
internalRuntimeDownloadArgs=
if [ '$(dotnetclimsrc-read-sas-token-base64)' != '$''(dotnetclimsrc-read-sas-token-base64)' ]; then
internalRuntimeDownloadArgs='--runtimesourcefeed https://dotnetclimsrc.blob.core.windows.net/dotnet --runtimesourcefeedkey $(dotnetclimsrc-read-sas-token-base64)'
fi
targetRidArgs= targetRidArgs=
if [ '${{ parameters.platform.targetRID }}' != '' ]; then if [ '${{ parameters.platform.targetRID }}' != '' ]; then
targetRidArgs='/p:TargetRid=${{ parameters.platform.targetRID }}' targetRidArgs='/p:TargetRid=${{ parameters.platform.targetRID }}'
@ -49,6 +73,7 @@ steps:
--restore --build --pack $publishArgs -bl \ --restore --build --pack $publishArgs -bl \
$officialBuildArgs \ $officialBuildArgs \
$internalRuntimeDownloadArgs \ $internalRuntimeDownloadArgs \
$internalRestoreArgs \
$targetRidArgs \ $targetRidArgs \
/p:SourceBuildNonPortable=${{ parameters.platform.nonPortable }} \ /p:SourceBuildNonPortable=${{ parameters.platform.nonPortable }} \
/p:ArcadeBuildFromSource=true /p:ArcadeBuildFromSource=true

42
eng/common/tools.ps1

@ -163,6 +163,9 @@ function InitializeDotNetCli([bool]$install, [bool]$createSdkLocationFile) {
# Disable telemetry on CI. # Disable telemetry on CI.
if ($ci) { if ($ci) {
$env:DOTNET_CLI_TELEMETRY_OPTOUT=1 $env:DOTNET_CLI_TELEMETRY_OPTOUT=1
# In case of network error, try to log the current IP for reference
Try-LogClientIpAddress
} }
# Source Build uses DotNetCoreSdkDir variable # Source Build uses DotNetCoreSdkDir variable
@ -706,6 +709,8 @@ function MSBuild() {
Write-PipelineSetVariable -Name 'NUGET_PLUGIN_REQUEST_TIMEOUT_IN_SECONDS' -Value '20' Write-PipelineSetVariable -Name 'NUGET_PLUGIN_REQUEST_TIMEOUT_IN_SECONDS' -Value '20'
} }
Enable-Nuget-EnhancedRetry
$toolsetBuildProject = InitializeToolset $toolsetBuildProject = InitializeToolset
$basePath = Split-Path -parent $toolsetBuildProject $basePath = Split-Path -parent $toolsetBuildProject
$possiblePaths = @( $possiblePaths = @(
@ -714,6 +719,8 @@ function MSBuild() {
(Join-Path $basePath (Join-Path $buildTool.Framework 'Microsoft.DotNet.Arcade.Sdk.dll')), (Join-Path $basePath (Join-Path $buildTool.Framework 'Microsoft.DotNet.Arcade.Sdk.dll')),
(Join-Path $basePath (Join-Path netcoreapp2.1 'Microsoft.DotNet.ArcadeLogging.dll')), (Join-Path $basePath (Join-Path netcoreapp2.1 'Microsoft.DotNet.ArcadeLogging.dll')),
(Join-Path $basePath (Join-Path netcoreapp2.1 'Microsoft.DotNet.Arcade.Sdk.dll')) (Join-Path $basePath (Join-Path netcoreapp2.1 'Microsoft.DotNet.Arcade.Sdk.dll'))
(Join-Path $basePath (Join-Path netcoreapp3.1 'Microsoft.DotNet.ArcadeLogging.dll')),
(Join-Path $basePath (Join-Path netcoreapp3.1 'Microsoft.DotNet.Arcade.Sdk.dll'))
) )
$selectedPath = $null $selectedPath = $null
foreach ($path in $possiblePaths) { foreach ($path in $possiblePaths) {
@ -750,6 +757,8 @@ function MSBuild-Core() {
} }
} }
Enable-Nuget-EnhancedRetry
$buildTool = InitializeBuildTool $buildTool = InitializeBuildTool
$cmdArgs = "$($buildTool.Command) /m /nologo /clp:Summary /v:$verbosity /nr:$nodeReuse /p:ContinuousIntegrationBuild=$ci" $cmdArgs = "$($buildTool.Command) /m /nologo /clp:Summary /v:$verbosity /nr:$nodeReuse /p:ContinuousIntegrationBuild=$ci"
@ -872,3 +881,36 @@ if (!$disableConfigureToolsetImport) {
} }
} }
} }
function Try-LogClientIpAddress()
{
Write-Host "Attempting to log this client's IP for Azure Package feed telemetry purposes"
try
{
$result = Invoke-WebRequest -Uri "http://co1.msedge.net/fdv2/diagnostics.aspx" -UseBasicParsing
$lines = $result.Content.Split([Environment]::NewLine)
$socketIp = $lines | Select-String -Pattern "^Socket IP:.*"
Write-Host $socketIp
$clientIp = $lines | Select-String -Pattern "^Client IP:.*"
Write-Host $clientIp
}
catch
{
Write-Host "Unable to get this machine's effective IP address for logging: $_"
}
}
#
# If $ci flag is set, turn on (and log that we did) special environment variables for improved Nuget client retry logic.
#
function Enable-Nuget-EnhancedRetry() {
if ($ci) {
Write-Host "Setting NUGET enhanced retry environment variables"
$env:NUGET_ENABLE_EXPERIMENTAL_HTTP_RETRY = 'true'
$env:NUGET_EXPERIMENTAL_MAX_NETWORK_TRY_COUNT = 6
$env:NUGET_EXPERIMENTAL_NETWORK_RETRY_DELAY_MILLISECONDS = 1000
Write-PipelineSetVariable -Name 'NUGET_ENABLE_EXPERIMENTAL_HTTP_RETRY' -Value 'true'
Write-PipelineSetVariable -Name 'NUGET_EXPERIMENTAL_MAX_NETWORK_TRY_COUNT' -Value '6'
Write-PipelineSetVariable -Name 'NUGET_EXPERIMENTAL_NETWORK_RETRY_DELAY_MILLISECONDS' -Value '1000'
}
}

16
eng/common/tools.sh

@ -399,6 +399,13 @@ function StopProcesses {
return 0 return 0
} }
function TryLogClientIpAddress () {
echo 'Attempting to log this client''s IP for Azure Package feed telemetry purposes'
if command -v curl > /dev/null; then
curl -s 'http://co1.msedge.net/fdv2/diagnostics.aspx' | grep ' IP: ' || true
fi
}
function MSBuild { function MSBuild {
local args=$@ local args=$@
if [[ "$pipelines_log" == true ]]; then if [[ "$pipelines_log" == true ]]; then
@ -410,6 +417,13 @@ function MSBuild {
export NUGET_PLUGIN_REQUEST_TIMEOUT_IN_SECONDS=20 export NUGET_PLUGIN_REQUEST_TIMEOUT_IN_SECONDS=20
Write-PipelineSetVariable -name "NUGET_PLUGIN_HANDSHAKE_TIMEOUT_IN_SECONDS" -value "20" Write-PipelineSetVariable -name "NUGET_PLUGIN_HANDSHAKE_TIMEOUT_IN_SECONDS" -value "20"
Write-PipelineSetVariable -name "NUGET_PLUGIN_REQUEST_TIMEOUT_IN_SECONDS" -value "20" Write-PipelineSetVariable -name "NUGET_PLUGIN_REQUEST_TIMEOUT_IN_SECONDS" -value "20"
export NUGET_ENABLE_EXPERIMENTAL_HTTP_RETRY=true
export NUGET_EXPERIMENTAL_MAX_NETWORK_TRY_COUNT=6
export NUGET_EXPERIMENTAL_NETWORK_RETRY_DELAY_MILLISECONDS=1000
Write-PipelineSetVariable -name "NUGET_ENABLE_EXPERIMENTAL_HTTP_RETRY" -value "true"
Write-PipelineSetVariable -name "NUGET_EXPERIMENTAL_MAX_NETWORK_TRY_COUNT" -value "6"
Write-PipelineSetVariable -name "NUGET_EXPERIMENTAL_NETWORK_RETRY_DELAY_MILLISECONDS" -value "1000"
fi fi
local toolset_dir="${_InitializeToolset%/*}" local toolset_dir="${_InitializeToolset%/*}"
@ -420,6 +434,8 @@ function MSBuild {
possiblePaths+=( "$toolset_dir/$_InitializeBuildToolFramework/Microsoft.DotNet.Arcade.Sdk.dll" ) possiblePaths+=( "$toolset_dir/$_InitializeBuildToolFramework/Microsoft.DotNet.Arcade.Sdk.dll" )
possiblePaths+=( "$toolset_dir/netcoreapp2.1/Microsoft.DotNet.ArcadeLogging.dll" ) possiblePaths+=( "$toolset_dir/netcoreapp2.1/Microsoft.DotNet.ArcadeLogging.dll" )
possiblePaths+=( "$toolset_dir/netcoreapp2.1/Microsoft.DotNet.Arcade.Sdk.dll" ) possiblePaths+=( "$toolset_dir/netcoreapp2.1/Microsoft.DotNet.Arcade.Sdk.dll" )
possiblePaths+=( "$toolset_dir/netcoreapp3.1/Microsoft.DotNet.ArcadeLogging.dll" )
possiblePaths+=( "$toolset_dir/netcoreapp3.1/Microsoft.DotNet.Arcade.Sdk.dll" )
for path in "${possiblePaths[@]}"; do for path in "${possiblePaths[@]}"; do
if [[ -f $path ]]; then if [[ -f $path ]]; then
selectedPath=$path selectedPath=$path

4
global.json

@ -3,7 +3,7 @@
"allowPrerelease": true "allowPrerelease": true
}, },
"tools": { "tools": {
"dotnet": "6.0.100-preview.6.21355.2", "dotnet": "6.0.100",
"runtimes": { "runtimes": {
"dotnet": [ "dotnet": [
"3.1.14" "3.1.14"
@ -14,6 +14,6 @@
} }
}, },
"msbuild-sdks": { "msbuild-sdks": {
"Microsoft.DotNet.Arcade.Sdk": "6.0.0-beta.21379.2" "Microsoft.DotNet.Arcade.Sdk": "7.0.0-beta.21559.3"
} }
} }

Loading…
Cancel
Save