Browse Source

Merge 5714d520fe into 75465614e6

pull/1598/merge
dotnet-maestro[bot] 3 years ago
committed by GitHub
parent
commit
5326443a9a
No known key found for this signature in database GPG Key ID: 4AEE18F83AFDEB23
  1. 20
      eng/Version.Details.xml
  2. 2
      eng/common/SetupNugetSources.ps1
  3. 2
      eng/common/SetupNugetSources.sh
  4. 2
      eng/common/build.sh
  5. 2
      eng/common/cross/build-rootfs.sh
  6. 9
      eng/common/cross/riscv64/tizen/tizen.patch
  7. 21
      eng/common/cross/tizen-build-rootfs.sh
  8. 17
      eng/common/cross/tizen-fetch.sh
  9. 15
      eng/common/cross/toolchain.cmake
  10. 4
      eng/common/dotnet-install.sh
  11. BIN
      eng/common/loc/P22DotNetHtmlLocalization.lss
  12. 2
      eng/common/native/init-compiler.sh
  13. 130
      eng/common/native/init-distro-rid.sh
  14. 80
      eng/common/native/init-os-and-arch.sh
  15. 2
      eng/common/post-build/check-channel-consistency.ps1
  16. 2
      eng/common/sdk-task.ps1
  17. 42
      eng/common/sdl/configure-sdl-tool.ps1
  18. 4
      eng/common/sdl/execute-all-sdl-tools.ps1
  19. 20
      eng/common/sdl/extract-artifact-packages.ps1
  20. 75
      eng/common/sdl/trim-assets-version.ps1
  21. 5
      eng/common/templates/job/execute-sdl.yml
  22. 14
      eng/common/templates/job/job.yml
  23. 2
      eng/common/templates/steps/execute-sdl.yml
  24. 15
      eng/common/templates/steps/source-build.yml
  25. 16
      eng/common/tools.ps1
  26. 13
      eng/common/tools.sh
  27. 4
      global.json

20
eng/Version.Details.xml

@ -3,25 +3,25 @@
<ProductDependencies>
</ProductDependencies>
<ToolsetDependencies>
<Dependency Name="Microsoft.DotNet.Arcade.Sdk" Version="8.0.0-beta.23265.1">
<Dependency Name="Microsoft.DotNet.Arcade.Sdk" Version="9.0.0-beta.23567.1">
<Uri>https://github.com/dotnet/arcade</Uri>
<Sha>2d8d59065b5e090584a8e90c4371fc06ed60bdc5</Sha>
<Sha>b6801170e6ca8bf12bdff1f10d410c5e01660681</Sha>
</Dependency>
<Dependency Name="Microsoft.DotNet.Build.Tasks.Feed" Version="8.0.0-beta.23265.1">
<Dependency Name="Microsoft.DotNet.Build.Tasks.Feed" Version="9.0.0-beta.23567.1">
<Uri>https://github.com/dotnet/arcade</Uri>
<Sha>2d8d59065b5e090584a8e90c4371fc06ed60bdc5</Sha>
<Sha>b6801170e6ca8bf12bdff1f10d410c5e01660681</Sha>
</Dependency>
<Dependency Name="Microsoft.DotNet.SignTool" Version="8.0.0-beta.23265.1">
<Dependency Name="Microsoft.DotNet.SignTool" Version="9.0.0-beta.23567.1">
<Uri>https://github.com/dotnet/arcade</Uri>
<Sha>2d8d59065b5e090584a8e90c4371fc06ed60bdc5</Sha>
<Sha>b6801170e6ca8bf12bdff1f10d410c5e01660681</Sha>
</Dependency>
<Dependency Name="Microsoft.DotNet.Helix.Sdk" Version="8.0.0-beta.23265.1">
<Dependency Name="Microsoft.DotNet.Helix.Sdk" Version="9.0.0-beta.23567.1">
<Uri>https://github.com/dotnet/arcade</Uri>
<Sha>2d8d59065b5e090584a8e90c4371fc06ed60bdc5</Sha>
<Sha>b6801170e6ca8bf12bdff1f10d410c5e01660681</Sha>
</Dependency>
<Dependency Name="Microsoft.DotNet.SwaggerGenerator.MSBuild" Version="8.0.0-beta.23265.1">
<Dependency Name="Microsoft.DotNet.SwaggerGenerator.MSBuild" Version="9.0.0-beta.23567.1">
<Uri>https://github.com/dotnet/arcade</Uri>
<Sha>2d8d59065b5e090584a8e90c4371fc06ed60bdc5</Sha>
<Sha>b6801170e6ca8bf12bdff1f10d410c5e01660681</Sha>
</Dependency>
<Dependency Name="Microsoft.DotNet.Maestro.Client" Version="1.1.0-beta.19556.4">
<Uri>https://github.com/dotnet/arcade-services</Uri>

2
eng/common/SetupNugetSources.ps1

@ -153,7 +153,7 @@ if ($dotnet31Source -ne $null) {
AddPackageSource -Sources $sources -SourceName "dotnet3.1-internal-transport" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/_packaging/dotnet3.1-internal-transport/nuget/v2" -Creds $creds -Username $userName -Password $Password
}
$dotnetVersions = @('5','6','7')
$dotnetVersions = @('5','6','7','8')
foreach ($dotnetVersion in $dotnetVersions) {
$feedPrefix = "dotnet" + $dotnetVersion;

2
eng/common/SetupNugetSources.sh

@ -105,7 +105,7 @@ if [ "$?" == "0" ]; then
PackageSources+=('dotnet3.1-internal-transport')
fi
DotNetVersions=('5' '6' '7')
DotNetVersions=('5' '6' '7' '8')
for DotNetVersion in ${DotNetVersions[@]} ; do
FeedPrefix="dotnet${DotNetVersion}";

2
eng/common/build.sh

@ -105,7 +105,7 @@ while [[ $# > 0 ]]; do
-binarylog|-bl)
binary_log=true
;;
-excludeCIBinarylog|-nobl)
-excludecibinarylog|-nobl)
exclude_ci_binary_log=true
;;
-pipelineslog|-pl)

2
eng/common/cross/build-rootfs.sh

@ -487,7 +487,7 @@ if [[ "$__CodeName" == "alpine" ]]; then
-X "http://dl-cdn.alpinelinux.org/alpine/$version/main" \
-X "http://dl-cdn.alpinelinux.org/alpine/$version/community" \
-U $__ApkSignatureArg --root "$__RootfsDir" --arch "$__AlpineArch" \
search 'llvm*-libs' | sort | tail -1 | sed 's/-[^-]*//2g')"
search 'llvm*-libs' | grep -E '^llvm' | sort | tail -1 | sed 's/-[^-]*//2g')"
fi
# install all packages in one go

9
eng/common/cross/riscv64/tizen/tizen.patch

@ -0,0 +1,9 @@
diff -u -r a/usr/lib/libc.so b/usr/lib/libc.so
--- a/usr/lib64/libc.so 2016-12-30 23:00:08.284951863 +0900
+++ b/usr/lib64/libc.so 2016-12-30 23:00:32.140951815 +0900
@@ -2,4 +2,4 @@
Use the shared library, but some functions are only in
the static library, so try that secondarily. */
OUTPUT_FORMAT(elf64-littleriscv)
-GROUP ( /lib64/libc.so.6 /usr/lib64/libc_nonshared.a AS_NEEDED ( /lib64/ld-linux-riscv64-lp64d.so.1 ) )
+GROUP ( libc.so.6 libc_nonshared.a AS_NEEDED ( ld-linux-riscv64-lp64d.so.1 ) )

21
eng/common/cross/tizen-build-rootfs.sh

@ -22,6 +22,10 @@ case "$ARCH" in
TIZEN_ARCH="x86_64"
LINK_ARCH="x86"
;;
riscv64)
TIZEN_ARCH="riscv64"
LINK_ARCH="riscv"
;;
*)
echo "Unsupported architecture for tizen: $ARCH"
exit 1
@ -58,4 +62,21 @@ rm -rf $TIZEN_TMP_DIR
echo ">>Start configuring Tizen rootfs"
ln -sfn asm-${LINK_ARCH} ./usr/include/asm
patch -p1 < $__TIZEN_CROSSDIR/tizen.patch
if [[ "$TIZEN_ARCH" == "riscv64" ]]; then
echo "Fixing broken symlinks in $PWD"
rm ./usr/lib64/libresolv.so
ln -s ../../lib64/libresolv.so.2 ./usr/lib64/libresolv.so
rm ./usr/lib64/libpthread.so
ln -s ../../lib64/libpthread.so.0 ./usr/lib64/libpthread.so
rm ./usr/lib64/libdl.so
ln -s ../../lib64/libdl.so.2 ./usr/lib64/libdl.so
rm ./usr/lib64/libutil.so
ln -s ../../lib64/libutil.so.1 ./usr/lib64/libutil.so
rm ./usr/lib64/libm.so
ln -s ../../lib64/libm.so.6 ./usr/lib64/libm.so
rm ./usr/lib64/librt.so
ln -s ../../lib64/librt.so.1 ./usr/lib64/librt.so
rm ./lib/ld-linux-riscv64-lp64d.so.1
ln -s ../lib64/ld-linux-riscv64-lp64d.so.1 ./lib/ld-linux-riscv64-lp64d.so.1
fi
echo "<<Finish configuring Tizen rootfs"

17
eng/common/cross/tizen-fetch.sh

@ -156,17 +156,28 @@ fetch_tizen_pkgs()
done
}
if [ "$TIZEN_ARCH" == "riscv64" ]; then
BASE="Tizen-Base-RISCV"
UNIFIED="Tizen-Unified-RISCV"
else
BASE="Tizen-Base"
UNIFIED="Tizen-Unified"
fi
Inform "Initialize ${TIZEN_ARCH} base"
fetch_tizen_pkgs_init standard Tizen-Base
fetch_tizen_pkgs_init standard $BASE
Inform "fetch common packages"
fetch_tizen_pkgs ${TIZEN_ARCH} gcc gcc-devel-static glibc glibc-devel libicu libicu-devel libatomic linux-glibc-devel keyutils keyutils-devel libkeyutils
Inform "fetch coreclr packages"
fetch_tizen_pkgs ${TIZEN_ARCH} lldb lldb-devel libgcc libstdc++ libstdc++-devel libunwind libunwind-devel lttng-ust-devel lttng-ust userspace-rcu-devel userspace-rcu
fetch_tizen_pkgs ${TIZEN_ARCH} libgcc libstdc++ libstdc++-devel libunwind libunwind-devel lttng-ust-devel lttng-ust userspace-rcu-devel userspace-rcu
if [ "$TIZEN_ARCH" != "riscv64" ]; then
fetch_tizen_pkgs ${TIZEN_ARCH} lldb lldb-devel
fi
Inform "fetch corefx packages"
fetch_tizen_pkgs ${TIZEN_ARCH} libcom_err libcom_err-devel zlib zlib-devel libopenssl11 libopenssl1.1-devel krb5 krb5-devel
Inform "Initialize standard unified"
fetch_tizen_pkgs_init standard Tizen-Unified
fetch_tizen_pkgs_init standard $UNIFIED
Inform "fetch corefx packages"
fetch_tizen_pkgs ${TIZEN_ARCH} gssdp gssdp-devel tizen-release

15
eng/common/cross/toolchain.cmake

@ -80,6 +80,9 @@ elseif(TARGET_ARCH_NAME STREQUAL "riscv64")
set(TOOLCHAIN "riscv64-alpine-linux-musl")
else()
set(TOOLCHAIN "riscv64-linux-gnu")
if(TIZEN)
set(TIZEN_TOOLCHAIN "riscv64-tizen-linux-gnu/13.1.0")
endif()
endif()
elseif(TARGET_ARCH_NAME STREQUAL "s390x")
set(CMAKE_SYSTEM_PROCESSOR s390x)
@ -144,6 +147,10 @@ if(TIZEN)
include_directories(SYSTEM ${CROSS_ROOTFS}/usr/lib64/gcc/${TIZEN_TOOLCHAIN}/include/c++/)
include_directories(SYSTEM ${CROSS_ROOTFS}/usr/lib64/gcc/${TIZEN_TOOLCHAIN}/include/c++/x86_64-tizen-linux-gnu)
endif()
if(TARGET_ARCH_NAME STREQUAL "riscv64")
include_directories(SYSTEM ${CROSS_ROOTFS}/usr/lib64/gcc/${TIZEN_TOOLCHAIN}/include/c++/)
include_directories(SYSTEM ${CROSS_ROOTFS}/usr/lib64/gcc/${TIZEN_TOOLCHAIN}/include/c++/riscv64-tizen-linux-gnu)
endif()
endif()
if(ANDROID)
@ -207,6 +214,7 @@ elseif(ILLUMOS)
set(CMAKE_CXX_STANDARD_LIBRARIES "${CMAKE_CXX_STANDARD_LIBRARIES} -lssp")
elseif(HAIKU)
set(CMAKE_SYSROOT "${CROSS_ROOTFS}")
set(CMAKE_PROGRAM_PATH "${CMAKE_PROGRAM_PATH};${CROSS_ROOTFS}/cross-tools-x86_64/bin")
set(TOOLSET_PREFIX ${TOOLCHAIN}-)
function(locate_toolchain_exec exec var)
@ -217,7 +225,6 @@ elseif(HAIKU)
endif()
find_program(EXEC_LOCATION_${exec}
PATHS "${CROSS_ROOTFS}/cross-tools-x86_64/bin"
NAMES
"${TOOLSET_PREFIX}${exec}${CLR_CMAKE_COMPILER_FILE_NAME_VERSION}"
"${TOOLSET_PREFIX}${exec}")
@ -270,7 +277,7 @@ if(TARGET_ARCH_NAME MATCHES "^(arm|armel)$")
add_toolchain_linker_flag("-L${CROSS_ROOTFS}/usr/lib")
add_toolchain_linker_flag("-L${CROSS_ROOTFS}/usr/lib/gcc/${TIZEN_TOOLCHAIN}")
endif()
elseif(TARGET_ARCH_NAME MATCHES "^(arm64|x64)$")
elseif(TARGET_ARCH_NAME MATCHES "^(arm64|x64|riscv64)$")
if(TIZEN)
add_toolchain_linker_flag("-B${CROSS_ROOTFS}/usr/lib64/gcc/${TIZEN_TOOLCHAIN}")
add_toolchain_linker_flag("-L${CROSS_ROOTFS}/lib64")
@ -281,6 +288,8 @@ elseif(TARGET_ARCH_NAME MATCHES "^(arm64|x64)$")
add_toolchain_linker_flag("-Wl,--rpath-link=${CROSS_ROOTFS}/usr/lib64")
add_toolchain_linker_flag("-Wl,--rpath-link=${CROSS_ROOTFS}/usr/lib64/gcc/${TIZEN_TOOLCHAIN}")
endif()
elseif(TARGET_ARCH_NAME STREQUAL "s390x")
add_toolchain_linker_flag("--target=${TOOLCHAIN}")
elseif(TARGET_ARCH_NAME STREQUAL "x86")
if(EXISTS ${CROSS_ROOTFS}/usr/lib/gcc/i586-alpine-linux-musl)
add_toolchain_linker_flag("--target=${TOOLCHAIN}")
@ -328,6 +337,8 @@ if(TARGET_ARCH_NAME MATCHES "^(arm|armel)$")
if(TARGET_ARCH_NAME STREQUAL "armel")
add_compile_options(-mfloat-abi=softfp)
endif()
elseif(TARGET_ARCH_NAME STREQUAL "s390x")
add_compile_options("--target=${TOOLCHAIN}")
elseif(TARGET_ARCH_NAME STREQUAL "x86")
if(EXISTS ${CROSS_ROOTFS}/usr/lib/gcc/i586-alpine-linux-musl)
add_compile_options(--target=${TOOLCHAIN})

4
eng/common/dotnet-install.sh

@ -54,6 +54,10 @@ cpuname=$(uname -m)
case $cpuname in
arm64|aarch64)
buildarch=arm64
if [ "$(getconf LONG_BIT)" -lt 64 ]; then
# This is 32-bit OS running on 64-bit CPU (for example Raspberry Pi OS)
buildarch=arm
fi
;;
loongarch64)
buildarch=loongarch64

BIN
eng/common/loc/P22DotNetHtmlLocalization.lss

Binary file not shown.

2
eng/common/native/init-compiler.sh

@ -63,7 +63,7 @@ if [ -z "$CLR_CC" ]; then
# Set default versions
if [ -z "$majorVersion" ]; then
# note: gcc (all versions) and clang versions higher than 6 do not have minor version in file name, if it is zero.
if [ "$compiler" = "clang" ]; then versions="16 15 14 13 12 11 10 9 8 7 6.0 5.0 4.0 3.9 3.8 3.7 3.6 3.5"
if [ "$compiler" = "clang" ]; then versions="17 16 15 14 13 12 11 10 9 8 7 6.0 5.0 4.0 3.9 3.8 3.7 3.6 3.5"
elif [ "$compiler" = "gcc" ]; then versions="13 12 11 10 9 8 7 6 5 4.9"; fi
for version in $versions; do

130
eng/common/native/init-distro-rid.sh

@ -0,0 +1,130 @@
#!/usr/bin/env bash
# getNonPortableDistroRid
#
# Input:
# targetOs: (str)
# targetArch: (str)
# rootfsDir: (str)
#
# Return:
# non-portable rid
getNonPortableDistroRid()
{
local targetOs="$1"
local targetArch="$2"
local rootfsDir="$3"
local nonPortableRid=""
if [ "$targetOs" = "linux" ]; then
if [ -e "${rootfsDir}/etc/os-release" ]; then
source "${rootfsDir}/etc/os-release"
if [[ "${ID}" == "rhel" || "${ID}" == "rocky" || "${ID}" == "alpine" ]]; then
# remove the last version digit
VERSION_ID="${VERSION_ID%.*}"
fi
if [[ "${VERSION_ID:-}" =~ ^([[:digit:]]|\.)+$ ]]; then
nonPortableRid="${ID}.${VERSION_ID}-${targetArch}"
else
# Rolling release distros either do not set VERSION_ID, set it as blank or
# set it to non-version looking string (such as TEMPLATE_VERSION_ID on ArchLinux);
# so omit it here to be consistent with everything else.
nonPortableRid="${ID}-${targetArch}"
fi
elif [ -e "${rootfsDir}/android_platform" ]; then
source "$rootfsDir"/android_platform
nonPortableRid="$RID"
fi
fi
if [ "$targetOs" = "freebsd" ]; then
# $rootfsDir can be empty. freebsd-version is shell script and it should always work.
__freebsd_major_version=$($rootfsDir/bin/freebsd-version | { read v; echo "${v%%.*}"; })
nonPortableRid="freebsd.$__freebsd_major_version-${targetArch}"
elif command -v getprop && getprop ro.product.system.model 2>&1 | grep -qi android; then
__android_sdk_version=$(getprop ro.build.version.sdk)
nonPortableRid="android.$__android_sdk_version-${targetArch}"
elif [ "$targetOs" = "illumos" ]; then
__uname_version=$(uname -v)
case "$__uname_version" in
omnios-*)
__omnios_major_version=$(echo "${__uname_version:8:2}")
nonPortableRid=omnios."$__omnios_major_version"-"$targetArch"
;;
joyent_*)
__smartos_major_version=$(echo "${__uname_version:7:4}")
nonPortableRid=smartos."$__smartos_major_version"-"$targetArch"
;;
illumos_*)
nonPortableRid=openindiana-"$targetArch"
;;
esac
elif [ "$targetOs" = "solaris" ]; then
__uname_version=$(uname -v)
__solaris_major_version=$(echo "${__uname_version%.*}")
nonPortableRid=solaris."$__solaris_major_version"-"$targetArch"
elif [ "$targetOs" = "haiku" ]; then
__uname_release=$(uname -r)
nonPortableRid=haiku.r"$__uname_release"-"$targetArch"
fi
echo "$(echo $nonPortableRid | tr '[:upper:]' '[:lower:]')"
}
# initDistroRidGlobal
#
# Input:
# os: (str)
# arch: (str)
# rootfsDir?: (nullable:string)
#
# Return:
# None
#
# Notes:
#
# It is important to note that the function does not return anything, but it
# exports the following variables on success:
#
# __DistroRid : Non-portable rid of the target platform.
# __PortableTargetOS : OS-part of the portable rid that corresponds to the target platform.
#
initDistroRidGlobal()
{
local targetOs="$1"
local targetArch="$2"
local rootfsDir=""
if [ "$#" -ge 3 ]; then
rootfsDir="$3"
fi
if [ -n "${rootfsDir}" ]; then
# We may have a cross build. Check for the existence of the rootfsDir
if [ ! -e "${rootfsDir}" ]; then
echo "Error rootfsDir has been passed, but the location is not valid."
exit 1
fi
fi
__DistroRid=$(getNonPortableDistroRid "${targetOs}" "${targetArch}" "${rootfsDir}")
if [ -z "${__PortableTargetOS:-}" ]; then
__PortableTargetOS="$targetOs"
STRINGS="$(command -v strings || true)"
if [ -z "$STRINGS" ]; then
STRINGS="$(command -v llvm-strings || true)"
fi
# Check for musl-based distros (e.g Alpine Linux, Void Linux).
if "${rootfsDir}/usr/bin/ldd" --version 2>&1 | grep -q musl ||
( [ -n "$STRINGS" ] && "$STRINGS" "${rootfsDir}/usr/bin/ldd" 2>&1 | grep -q musl ); then
__PortableTargetOS="linux-musl"
fi
fi
export __DistroRid __PortableTargetOS
}

80
eng/common/native/init-os-and-arch.sh

@ -0,0 +1,80 @@
#!/usr/bin/env bash
# Use uname to determine what the OS is.
OSName=$(uname -s | tr '[:upper:]' '[:lower:]')
if command -v getprop && getprop ro.product.system.model 2>&1 | grep -qi android; then
OSName="android"
fi
case "$OSName" in
freebsd|linux|netbsd|openbsd|sunos|android|haiku)
os="$OSName" ;;
darwin)
os=osx ;;
*)
echo "Unsupported OS $OSName detected!"
exit 1 ;;
esac
# On Solaris, `uname -m` is discouraged, see https://docs.oracle.com/cd/E36784_01/html/E36870/uname-1.html
# and `uname -p` returns processor type (e.g. i386 on amd64).
# The appropriate tool to determine CPU is isainfo(1) https://docs.oracle.com/cd/E36784_01/html/E36870/isainfo-1.html.
if [ "$os" = "sunos" ]; then
if uname -o 2>&1 | grep -q illumos; then
os="illumos"
else
os="solaris"
fi
CPUName=$(isainfo -n)
else
# For the rest of the operating systems, use uname(1) to determine what the CPU is.
CPUName=$(uname -m)
fi
case "$CPUName" in
arm64|aarch64)
arch=arm64
;;
loongarch64)
arch=loongarch64
;;
riscv64)
arch=riscv64
;;
amd64|x86_64)
arch=x64
;;
armv7l|armv8l)
if (NAME=""; . /etc/os-release; test "$NAME" = "Tizen"); then
arch=armel
else
arch=arm
fi
;;
armv6l)
arch=armv6
;;
i[3-6]86)
echo "Unsupported CPU $CPUName detected, build might not succeed!"
arch=x86
;;
s390x)
arch=s390x
;;
ppc64le)
arch=ppc64le
;;
*)
echo "Unknown CPU $CPUName detected!"
exit 1
;;
esac

2
eng/common/post-build/check-channel-consistency.ps1

@ -7,7 +7,7 @@ try {
. $PSScriptRoot\post-build-utils.ps1
if ($PromoteToChannels -eq "") {
Write-PipelineTaskError -Type 'warning' -Message "This build won't publish assets as it's not configured to any Maestro channel. If that wasn't intended use Darc to configure a default channel using add-default-channel for this branch or to promote it to a channel using add-build-to-channel. See https://github.com/dotnet/arcade/blob/master/Documentation/Darc.md#assigning-an-individual-build-to-a-channel for more info."
Write-PipelineTaskError -Type 'warning' -Message "This build won't publish assets as it's not configured to any Maestro channel. If that wasn't intended use Darc to configure a default channel using add-default-channel for this branch or to promote it to a channel using add-build-to-channel. See https://github.com/dotnet/arcade/blob/main/Documentation/Darc.md#assigning-an-individual-build-to-a-channel for more info."
ExitWithExitCode 0
}

2
eng/common/sdk-task.ps1

@ -64,7 +64,7 @@ try {
$GlobalJson.tools | Add-Member -Name "vs" -Value (ConvertFrom-Json "{ `"version`": `"16.5`" }") -MemberType NoteProperty
}
if( -not ($GlobalJson.tools.PSObject.Properties.Name -match "xcopy-msbuild" )) {
$GlobalJson.tools | Add-Member -Name "xcopy-msbuild" -Value "17.4.1" -MemberType NoteProperty
$GlobalJson.tools | Add-Member -Name "xcopy-msbuild" -Value "17.8.1-2" -MemberType NoteProperty
}
if ($GlobalJson.tools."xcopy-msbuild".Trim() -ine "none") {
$xcopyMSBuildToolsFolder = InitializeXCopyMSBuild $GlobalJson.tools."xcopy-msbuild" -install $true

42
eng/common/sdl/configure-sdl-tool.ps1

@ -17,7 +17,9 @@ Param(
# Optional: Additional params to add to any tool using PoliCheck.
[string[]] $PoliCheckAdditionalRunConfigParams,
# Optional: Additional params to add to any tool using CodeQL/Semmle.
[string[]] $CodeQLAdditionalRunConfigParams
[string[]] $CodeQLAdditionalRunConfigParams,
# Optional: Additional params to add to any tool using Binskim.
[string[]] $BinskimAdditionalRunConfigParams
)
$ErrorActionPreference = 'Stop'
@ -69,22 +71,34 @@ try {
$gdnConfigFile = Join-Path $gdnConfigPath "$toolConfigName-configure.gdnconfig"
# For some tools, add default and automatic args.
if ($tool.Name -eq 'credscan') {
if ($targetDirectory) {
$tool.Args += "`"TargetDirectory < $TargetDirectory`""
switch -Exact ($tool.Name) {
'credscan' {
if ($targetDirectory) {
$tool.Args += "`"TargetDirectory < $TargetDirectory`""
}
$tool.Args += "`"OutputType < pre`""
$tool.Args += $CrScanAdditionalRunConfigParams
}
$tool.Args += "`"OutputType < pre`""
$tool.Args += $CrScanAdditionalRunConfigParams
} elseif ($tool.Name -eq 'policheck') {
if ($targetDirectory) {
$tool.Args += "`"Target < $TargetDirectory`""
'policheck' {
if ($targetDirectory) {
$tool.Args += "`"Target < $TargetDirectory`""
}
$tool.Args += $PoliCheckAdditionalRunConfigParams
}
$tool.Args += $PoliCheckAdditionalRunConfigParams
} elseif ($tool.Name -eq 'semmle' -or $tool.Name -eq 'codeql') {
if ($targetDirectory) {
$tool.Args += "`"SourceCodeDirectory < $TargetDirectory`""
{$_ -in 'semmle', 'codeql'} {
if ($targetDirectory) {
$tool.Args += "`"SourceCodeDirectory < $TargetDirectory`""
}
$tool.Args += $CodeQLAdditionalRunConfigParams
}
'binskim' {
if ($targetDirectory) {
# Binskim crashes due to specific PDBs. GitHub issue: https://github.com/microsoft/binskim/issues/924.
# We are excluding all `_.pdb` files from the scan.
$tool.Args += "`"Target < $TargetDirectory\**;-:file|$TargetDirectory\**\_.pdb`""
}
$tool.Args += $BinskimAdditionalRunConfigParams
}
$tool.Args += $CodeQLAdditionalRunConfigParams
}
# Create variable pointing to the args array directly so we can use splat syntax later.

4
eng/common/sdl/execute-all-sdl-tools.ps1

@ -35,6 +35,7 @@ Param(
[string[]] $CrScanAdditionalRunConfigParams, # Optional: Additional Params to custom build a CredScan run config in the format @("xyz:abc","sdf:1")
[string[]] $PoliCheckAdditionalRunConfigParams, # Optional: Additional Params to custom build a Policheck run config in the format @("xyz:abc","sdf:1")
[string[]] $CodeQLAdditionalRunConfigParams, # Optional: Additional Params to custom build a Semmle/CodeQL run config in the format @("xyz < abc","sdf < 1")
[string[]] $BinskimAdditionalRunConfigParams, # Optional: Additional Params to custom build a Binskim run config in the format @("xyz < abc","sdf < 1")
[bool] $BreakOnFailure=$False # Optional: Fail the build if there were errors during the run
)
@ -107,7 +108,8 @@ try {
-GuardianLoggerLevel $GuardianLoggerLevel `
-CrScanAdditionalRunConfigParams $CrScanAdditionalRunConfigParams `
-PoliCheckAdditionalRunConfigParams $PoliCheckAdditionalRunConfigParams `
-CodeQLAdditionalRunConfigParams $CodeQLAdditionalRunConfigParams
-CodeQLAdditionalRunConfigParams $CodeQLAdditionalRunConfigParams `
-BinskimAdditionalRunConfigParams $BinskimAdditionalRunConfigParams
if ($BreakOnFailure) {
Exit-IfNZEC "Sdl"
}

20
eng/common/sdl/extract-artifact-packages.ps1

@ -35,31 +35,33 @@ try {
param(
[string] $PackagePath # Full path to a NuGet package
)
if (!(Test-Path $PackagePath)) {
Write-PipelineTelemetryError -Category 'Build' -Message "Input file does not exist: $PackagePath"
ExitWithExitCode 1
}
$RelevantExtensions = @('.dll', '.exe', '.pdb')
Write-Host -NoNewLine 'Extracting ' ([System.IO.Path]::GetFileName($PackagePath)) '...'
$PackageId = [System.IO.Path]::GetFileNameWithoutExtension($PackagePath)
$ExtractPath = Join-Path -Path $using:ExtractPath -ChildPath $PackageId
Add-Type -AssemblyName System.IO.Compression.FileSystem
[System.IO.Directory]::CreateDirectory($ExtractPath);
try {
$zip = [System.IO.Compression.ZipFile]::OpenRead($PackagePath)
$zip.Entries |
Where-Object {$RelevantExtensions -contains [System.IO.Path]::GetExtension($_.Name)} |
ForEach-Object {
$TargetFile = Join-Path -Path $ExtractPath -ChildPath $_.Name
[System.IO.Compression.ZipFileExtensions]::ExtractToFile($_, $TargetFile, $true)
$TargetPath = Join-Path -Path $ExtractPath -ChildPath (Split-Path -Path $_.FullName)
[System.IO.Directory]::CreateDirectory($TargetPath);
$TargetFile = Join-Path -Path $ExtractPath -ChildPath $_.FullName
[System.IO.Compression.ZipFileExtensions]::ExtractToFile($_, $TargetFile)
}
}
catch {

75
eng/common/sdl/trim-assets-version.ps1

@ -0,0 +1,75 @@
<#
.SYNOPSIS
Install and run the 'Microsoft.DotNet.VersionTools.Cli' tool with the 'trim-artifacts-version' command to trim the version from the NuGet assets file name.
.PARAMETER InputPath
Full path to directory where artifact packages are stored
.PARAMETER Recursive
Search for NuGet packages recursively
#>
Param(
[string] $InputPath,
[bool] $Recursive = $true
)
$CliToolName = "Microsoft.DotNet.VersionTools.Cli"
function Install-VersionTools-Cli {
param(
[Parameter(Mandatory=$true)][string]$Version
)
Write-Host "Installing the package '$CliToolName' with a version of '$version' ..."
$feed = "https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-eng/nuget/v3/index.json"
$argumentList = @("tool", "install", "--local", "$CliToolName", "--add-source $feed", "--no-cache", "--version $Version", "--create-manifest-if-needed")
Start-Process "$dotnet" -Verbose -ArgumentList $argumentList -NoNewWindow -Wait
}
# -------------------------------------------------------------------
if (!(Test-Path $InputPath)) {
Write-Host "Input Path '$InputPath' does not exist"
ExitWithExitCode 1
}
$ErrorActionPreference = 'Stop'
Set-StrictMode -Version 2.0
$disableConfigureToolsetImport = $true
$global:LASTEXITCODE = 0
# `tools.ps1` checks $ci to perform some actions. Since the SDL
# scripts don't necessarily execute in the same agent that run the
# build.ps1/sh script this variable isn't automatically set.
$ci = $true
. $PSScriptRoot\..\tools.ps1
try {
$dotnetRoot = InitializeDotNetCli -install:$true
$dotnet = "$dotnetRoot\dotnet.exe"
$toolsetVersion = Read-ArcadeSdkVersion
Install-VersionTools-Cli -Version $toolsetVersion
$cliToolFound = (& "$dotnet" tool list --local | Where-Object {$_.Split(' ')[0] -eq $CliToolName})
if ($null -eq $cliToolFound) {
Write-PipelineTelemetryError -Force -Category 'Sdl' -Message "The '$CliToolName' tool is not installed."
ExitWithExitCode 1
}
Exec-BlockVerbosely {
& "$dotnet" $CliToolName trim-assets-version `
--assets-path $InputPath `
--recursive $Recursive
Exit-IfNZEC "Sdl"
}
}
catch {
Write-Host $_
Write-PipelineTelemetryError -Force -Category 'Sdl' -Message $_
ExitWithExitCode 1
}

5
eng/common/templates/job/execute-sdl.yml

@ -105,6 +105,11 @@ jobs:
downloadPath: $(Build.ArtifactStagingDirectory)\artifacts
checkDownloadedFiles: true
- powershell: eng/common/sdl/trim-assets-version.ps1
-InputPath $(Build.ArtifactStagingDirectory)\artifacts
displayName: Trim the version from the NuGet packages
continueOnError: ${{ parameters.sdlContinueOnError }}
- powershell: eng/common/sdl/extract-artifact-packages.ps1
-InputPath $(Build.ArtifactStagingDirectory)\artifacts\BlobArtifacts
-ExtractPath $(Build.ArtifactStagingDirectory)\artifacts\BlobArtifacts

14
eng/common/templates/job/job.yml

@ -75,6 +75,10 @@ jobs:
- ${{ if eq(parameters.enableRichCodeNavigation, 'true') }}:
- name: EnableRichCodeNavigation
value: 'true'
# Retry signature validation up to three times, waiting 2 seconds between attempts.
# See https://learn.microsoft.com/en-us/nuget/reference/errors-and-warnings/nu3028#retry-untrusted-root-failures
- name: NUGET_EXPERIMENTAL_CHAIN_BUILD_RETRY_POLICY
value: 3,2000
- ${{ each variable in parameters.variables }}:
# handle name-value variable syntax
# example:
@ -83,7 +87,7 @@ jobs:
- ${{ if ne(variable.name, '') }}:
- name: ${{ variable.name }}
value: ${{ variable.value }}
# handle variable groups
- ${{ if ne(variable.group, '') }}:
- group: ${{ variable.group }}
@ -150,7 +154,7 @@ jobs:
displayName: RichCodeNav Upload
inputs:
languages: ${{ coalesce(parameters.richCodeNavigationLanguage, 'csharp') }}
environment: ${{ coalesce(parameters.richCodeNavigationEnvironment, 'production') }}
environment: ${{ coalesce(parameters.richCodeNavigationEnvironment, 'internal') }}
richNavLogOutputDirectory: $(Build.SourcesDirectory)/artifacts/bin
uploadRichNavArtifacts: ${{ coalesce(parameters.richCodeNavigationUploadArtifacts, false) }}
continueOnError: true
@ -169,7 +173,7 @@ jobs:
- ${{ if eq(parameters.enableMicrobuild, 'true') }}:
- ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}:
- task: MicroBuildCleanup@1
displayName: Execute Microbuild cleanup tasks
displayName: Execute Microbuild cleanup tasks
condition: and(always(), in(variables['_SignType'], 'real', 'test'), eq(variables['Agent.Os'], 'Windows_NT'))
continueOnError: ${{ parameters.continueOnError }}
env:
@ -219,7 +223,7 @@ jobs:
displayName: Publish XUnit Test Results
inputs:
testResultsFormat: 'xUnit'
testResultsFiles: '*.xml'
testResultsFiles: '*.xml'
searchFolder: '$(Build.SourcesDirectory)/artifacts/TestResults/$(_BuildConfig)'
testRunTitle: ${{ coalesce(parameters.testRunTitle, parameters.name, '$(System.JobName)') }}-xunit
mergeTestResults: ${{ parameters.mergeTestResults }}
@ -230,7 +234,7 @@ jobs:
displayName: Publish TRX Test Results
inputs:
testResultsFormat: 'VSTest'
testResultsFiles: '*.trx'
testResultsFiles: '*.trx'
searchFolder: '$(Build.SourcesDirectory)/artifacts/TestResults/$(_BuildConfig)'
testRunTitle: ${{ coalesce(parameters.testRunTitle, parameters.name, '$(System.JobName)') }}-trx
mergeTestResults: ${{ parameters.mergeTestResults }}

2
eng/common/templates/steps/execute-sdl.yml

@ -33,7 +33,7 @@ steps:
- ${{ if ne(parameters.overrideParameters, '') }}:
- powershell: ${{ parameters.executeAllSdlToolsScript }} ${{ parameters.overrideParameters }}
displayName: Execute SDL
displayName: Execute SDL (Overridden)
continueOnError: ${{ parameters.sdlContinueOnError }}
condition: ${{ parameters.condition }}

15
eng/common/templates/steps/source-build.yml

@ -68,6 +68,11 @@ steps:
runtimeOsArgs='/p:RuntimeOS=${{ parameters.platform.runtimeOS }}'
fi
baseOsArgs=
if [ '${{ parameters.platform.baseOS }}' != '' ]; then
baseOsArgs='/p:BaseOS=${{ parameters.platform.baseOS }}'
fi
publishArgs=
if [ '${{ parameters.platform.skipPublishValidation }}' != 'true' ]; then
publishArgs='--publish'
@ -86,6 +91,7 @@ steps:
$internalRestoreArgs \
$targetRidArgs \
$runtimeOsArgs \
$baseOsArgs \
/p:SourceBuildNonPortable=${{ parameters.platform.nonPortable }} \
/p:ArcadeBuildFromSource=true \
/p:AssetManifestFileName=$assetManifestFileName
@ -112,3 +118,12 @@ steps:
artifactName: BuildLogs_SourceBuild_${{ parameters.platform.name }}_Attempt$(System.JobAttempt)
continueOnError: true
condition: succeededOrFailed()
# Manually inject component detection so that we can ignore the source build upstream cache, which contains
# a nupkg cache of input packages (a local feed).
# This path must match the upstream cache path in property 'CurrentRepoSourceBuiltNupkgCacheDir'
# in src\Microsoft.DotNet.Arcade.Sdk\tools\SourceBuild\SourceBuildArcade.targets
- task: ComponentGovernanceComponentDetection@0
displayName: Component Detection (Exclude upstream cache)
inputs:
ignoreDirectories: '$(Build.SourcesDirectory)/artifacts/source-build/self/src/artifacts/obj/source-built-upstream-cache'

16
eng/common/tools.ps1

@ -158,7 +158,7 @@ function InitializeDotNetCli([bool]$install, [bool]$createSdkLocationFile) {
$env:DOTNET_MULTILEVEL_LOOKUP=0
# Disable first run since we do not need all ASP.NET packages restored.
$env:DOTNET_SKIP_FIRST_TIME_EXPERIENCE=1
$env:DOTNET_NOLOGO=1
# Disable telemetry on CI.
if ($ci) {
@ -228,7 +228,7 @@ function InitializeDotNetCli([bool]$install, [bool]$createSdkLocationFile) {
Write-PipelinePrependPath -Path $dotnetRoot
Write-PipelineSetVariable -Name 'DOTNET_MULTILEVEL_LOOKUP' -Value '0'
Write-PipelineSetVariable -Name 'DOTNET_SKIP_FIRST_TIME_EXPERIENCE' -Value '1'
Write-PipelineSetVariable -Name 'DOTNET_NOLOGO' -Value '1'
return $global:_DotNetInstallDir = $dotnetRoot
}
@ -379,13 +379,13 @@ function InitializeVisualStudioMSBuild([bool]$install, [object]$vsRequirements =
}
# Minimum VS version to require.
$vsMinVersionReqdStr = '16.8'
$vsMinVersionReqdStr = '17.7'
$vsMinVersionReqd = [Version]::new($vsMinVersionReqdStr)
# If the version of msbuild is going to be xcopied,
# use this version. Version matches a package here:
# https://dev.azure.com/dnceng/public/_packaging?_a=package&feed=dotnet-eng&package=RoslynTools.MSBuild&protocolType=NuGet&version=17.4.1&view=overview
$defaultXCopyMSBuildVersion = '17.4.1'
# https://dev.azure.com/dnceng/public/_artifacts/feed/dotnet-eng/NuGet/RoslynTools.MSBuild/versions/17.8.1-2
$defaultXCopyMSBuildVersion = '17.8.1-2'
if (!$vsRequirements) {
if (Get-Member -InputObject $GlobalJson.tools -Name 'vs') {
@ -671,6 +671,10 @@ function InitializeNativeTools() {
}
}
function Read-ArcadeSdkVersion() {
return $GlobalJson.'msbuild-sdks'.'Microsoft.DotNet.Arcade.Sdk'
}
function InitializeToolset() {
if (Test-Path variable:global:_ToolsetBuildProj) {
return $global:_ToolsetBuildProj
@ -678,7 +682,7 @@ function InitializeToolset() {
$nugetCache = GetNuGetPackageCachePath
$toolsetVersion = $GlobalJson.'msbuild-sdks'.'Microsoft.DotNet.Arcade.Sdk'
$toolsetVersion = Read-ArcadeSdkVersion
$toolsetLocationFile = Join-Path $ToolsetDir "$toolsetVersion.txt"
if (Test-Path $toolsetLocationFile) {

13
eng/common/tools.sh

@ -112,7 +112,7 @@ function InitializeDotNetCli {
export DOTNET_MULTILEVEL_LOOKUP=0
# Disable first run since we want to control all package sources
export DOTNET_SKIP_FIRST_TIME_EXPERIENCE=1
export DOTNET_NOLOGO=1
# Disable telemetry on CI
if [[ $ci == true ]]; then
@ -165,7 +165,7 @@ function InitializeDotNetCli {
Write-PipelinePrependPath -path "$dotnet_root"
Write-PipelineSetVariable -name "DOTNET_MULTILEVEL_LOOKUP" -value "0"
Write-PipelineSetVariable -name "DOTNET_SKIP_FIRST_TIME_EXPERIENCE" -value "1"
Write-PipelineSetVariable -name "DOTNET_NOLOGO" -value "1"
# return value
_InitializeDotNetCli="$dotnet_root"
@ -310,7 +310,7 @@ function GetDotNetInstallScript {
curl "$install_script_url" -sSL --retry 10 --create-dirs -o "$install_script" || {
if command -v openssl &> /dev/null; then
echo "Curl failed; dumping some information about dotnet.microsoft.com for later investigation"
echo | openssl s_client -showcerts -servername dotnet.microsoft.com -connect dotnet.microsoft.com:443
echo | openssl s_client -showcerts -servername dotnet.microsoft.com -connect dotnet.microsoft.com:443 || true
fi
echo "Will now retry the same URL with verbose logging."
with_retries curl "$install_script_url" -sSL --verbose --retry 10 --create-dirs -o "$install_script" || {
@ -341,7 +341,12 @@ function InitializeBuildTool {
# return values
_InitializeBuildTool="$_InitializeDotNetCli/dotnet"
_InitializeBuildToolCommand="msbuild"
_InitializeBuildToolFramework="net8.0"
# use override if it exists - commonly set by source-build
if [[ "${_OverrideArcadeInitializeBuildToolFramework:-x}" == "x" ]]; then
_InitializeBuildToolFramework="net8.0"
else
_InitializeBuildToolFramework="${_OverrideArcadeInitializeBuildToolFramework}"
fi
}
# Set RestoreNoCache as a workaround for https://github.com/NuGet/Home/issues/3116

4
global.json

@ -3,7 +3,7 @@
"allowPrerelease": true
},
"tools": {
"dotnet": "8.0.100-preview.3.23178.7",
"dotnet": "8.0.100",
"runtimes": {
"dotnet": [
"6.0.0"
@ -14,6 +14,6 @@
}
},
"msbuild-sdks": {
"Microsoft.DotNet.Arcade.Sdk": "8.0.0-beta.23265.1"
"Microsoft.DotNet.Arcade.Sdk": "9.0.0-beta.23567.1"
}
}

Loading…
Cancel
Save