Update dependencies from https://github.com/dotnet/arcade build 20231117.1

Microsoft.DotNet.Arcade.Sdk , Microsoft.DotNet.Build.Tasks.Feed , Microsoft.DotNet.Helix.Sdk , Microsoft.DotNet.SignTool , Microsoft.DotNet.SwaggerGenerator.MSBuild
 From Version 8.0.0-beta.23265.1 -> To Version 9.0.0-beta.23567.1

eng/Version.Details.xml

@@ -3,25 +3,25 @@
   <ProductDependencies>
   </ProductDependencies>
   <ToolsetDependencies>
-    <Dependency Name="Microsoft.DotNet.Arcade.Sdk" Version="8.0.0-beta.23265.1">
+    <Dependency Name="Microsoft.DotNet.Arcade.Sdk" Version="9.0.0-beta.23567.1">
       <Uri>https://github.com/dotnet/arcade</Uri>
-      <Sha>2d8d59065b5e090584a8e90c4371fc06ed60bdc5</Sha>
+      <Sha>b6801170e6ca8bf12bdff1f10d410c5e01660681</Sha>
     </Dependency>
-    <Dependency Name="Microsoft.DotNet.Build.Tasks.Feed" Version="8.0.0-beta.23265.1">
+    <Dependency Name="Microsoft.DotNet.Build.Tasks.Feed" Version="9.0.0-beta.23567.1">
       <Uri>https://github.com/dotnet/arcade</Uri>
-      <Sha>2d8d59065b5e090584a8e90c4371fc06ed60bdc5</Sha>
+      <Sha>b6801170e6ca8bf12bdff1f10d410c5e01660681</Sha>
     </Dependency>
-    <Dependency Name="Microsoft.DotNet.SignTool" Version="8.0.0-beta.23265.1">
+    <Dependency Name="Microsoft.DotNet.SignTool" Version="9.0.0-beta.23567.1">
       <Uri>https://github.com/dotnet/arcade</Uri>
-      <Sha>2d8d59065b5e090584a8e90c4371fc06ed60bdc5</Sha>
+      <Sha>b6801170e6ca8bf12bdff1f10d410c5e01660681</Sha>
     </Dependency>
-    <Dependency Name="Microsoft.DotNet.Helix.Sdk" Version="8.0.0-beta.23265.1">
+    <Dependency Name="Microsoft.DotNet.Helix.Sdk" Version="9.0.0-beta.23567.1">
       <Uri>https://github.com/dotnet/arcade</Uri>
-      <Sha>2d8d59065b5e090584a8e90c4371fc06ed60bdc5</Sha>
+      <Sha>b6801170e6ca8bf12bdff1f10d410c5e01660681</Sha>
     </Dependency>
-    <Dependency Name="Microsoft.DotNet.SwaggerGenerator.MSBuild" Version="8.0.0-beta.23265.1">
+    <Dependency Name="Microsoft.DotNet.SwaggerGenerator.MSBuild" Version="9.0.0-beta.23567.1">
       <Uri>https://github.com/dotnet/arcade</Uri>
-      <Sha>2d8d59065b5e090584a8e90c4371fc06ed60bdc5</Sha>
+      <Sha>b6801170e6ca8bf12bdff1f10d410c5e01660681</Sha>
     </Dependency>
     <Dependency Name="Microsoft.DotNet.Maestro.Client" Version="1.1.0-beta.19556.4">
       <Uri>https://github.com/dotnet/arcade-services</Uri>

eng/common/SetupNugetSources.ps1

@@ -153,7 +153,7 @@ if ($dotnet31Source -ne $null) {
     AddPackageSource -Sources $sources -SourceName "dotnet3.1-internal-transport" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/_packaging/dotnet3.1-internal-transport/nuget/v2" -Creds $creds -Username $userName -Password $Password
 }
 
-$dotnetVersions = @('5','6','7')
+$dotnetVersions = @('5','6','7','8')
 
 foreach ($dotnetVersion in $dotnetVersions) {
     $feedPrefix = "dotnet" + $dotnetVersion;

eng/common/SetupNugetSources.sh

@@ -105,7 +105,7 @@ if [ "$?" == "0" ]; then
     PackageSources+=('dotnet3.1-internal-transport')
 fi
 
-DotNetVersions=('5' '6' '7')
+DotNetVersions=('5' '6' '7' '8')
 
 for DotNetVersion in ${DotNetVersions[@]} ; do
     FeedPrefix="dotnet${DotNetVersion}";

eng/common/build.sh

@@ -105,7 +105,7 @@ while [[ $# > 0 ]]; do
     -binarylog|-bl)
       binary_log=true
       ;;
-    -excludeCIBinarylog|-nobl)
+    -excludecibinarylog|-nobl)
       exclude_ci_binary_log=true
       ;;
     -pipelineslog|-pl)

eng/common/cross/build-rootfs.sh

@@ -487,7 +487,7 @@ if [[ "$__CodeName" == "alpine" ]]; then
             -X "http://dl-cdn.alpinelinux.org/alpine/$version/main" \
             -X "http://dl-cdn.alpinelinux.org/alpine/$version/community" \
             -U $__ApkSignatureArg --root "$__RootfsDir" --arch "$__AlpineArch" \
-            search 'llvm*-libs' | sort | tail -1 | sed 's/-[^-]*//2g')"
+            search 'llvm*-libs' | grep -E '^llvm' | sort | tail -1 | sed 's/-[^-]*//2g')"
     fi
 
     # install all packages in one go

eng/common/cross/riscv64/tizen/tizen.patch

@@ -0,0 +1,9 @@
+diff -u -r a/usr/lib/libc.so b/usr/lib/libc.so
+--- a/usr/lib64/libc.so	2016-12-30 23:00:08.284951863 +0900
++++ b/usr/lib64/libc.so	2016-12-30 23:00:32.140951815 +0900
+@@ -2,4 +2,4 @@
+    Use the shared library, but some functions are only in
+    the static library, so try that secondarily.  */
+ OUTPUT_FORMAT(elf64-littleriscv)
+-GROUP ( /lib64/libc.so.6 /usr/lib64/libc_nonshared.a  AS_NEEDED ( /lib64/ld-linux-riscv64-lp64d.so.1 ) )
++GROUP ( libc.so.6 libc_nonshared.a  AS_NEEDED ( ld-linux-riscv64-lp64d.so.1 ) )

eng/common/cross/tizen-build-rootfs.sh

@@ -22,6 +22,10 @@ case "$ARCH" in
         TIZEN_ARCH="x86_64"
         LINK_ARCH="x86"
         ;;
+    riscv64)
+        TIZEN_ARCH="riscv64"
+        LINK_ARCH="riscv"
+        ;;
     *)
         echo "Unsupported architecture for tizen: $ARCH"
         exit 1
@@ -58,4 +62,21 @@ rm -rf $TIZEN_TMP_DIR
 echo ">>Start configuring Tizen rootfs"
 ln -sfn asm-${LINK_ARCH} ./usr/include/asm
 patch -p1 < $__TIZEN_CROSSDIR/tizen.patch
+if [[ "$TIZEN_ARCH" == "riscv64" ]]; then
+    echo "Fixing broken symlinks in $PWD"
+    rm ./usr/lib64/libresolv.so
+    ln -s ../../lib64/libresolv.so.2 ./usr/lib64/libresolv.so
+    rm ./usr/lib64/libpthread.so
+    ln -s ../../lib64/libpthread.so.0 ./usr/lib64/libpthread.so
+    rm ./usr/lib64/libdl.so
+    ln -s ../../lib64/libdl.so.2 ./usr/lib64/libdl.so
+    rm ./usr/lib64/libutil.so
+    ln -s ../../lib64/libutil.so.1 ./usr/lib64/libutil.so
+    rm ./usr/lib64/libm.so
+    ln -s ../../lib64/libm.so.6 ./usr/lib64/libm.so
+    rm ./usr/lib64/librt.so
+    ln -s ../../lib64/librt.so.1 ./usr/lib64/librt.so
+    rm ./lib/ld-linux-riscv64-lp64d.so.1
+    ln -s ../lib64/ld-linux-riscv64-lp64d.so.1 ./lib/ld-linux-riscv64-lp64d.so.1
+fi
 echo "<<Finish configuring Tizen rootfs"

eng/common/cross/tizen-fetch.sh

@@ -156,17 +156,28 @@ fetch_tizen_pkgs()
     done
 }
 
+if [ "$TIZEN_ARCH" == "riscv64" ]; then
+    BASE="Tizen-Base-RISCV"
+    UNIFIED="Tizen-Unified-RISCV"
+else
+    BASE="Tizen-Base"
+    UNIFIED="Tizen-Unified"
+fi
+
 Inform "Initialize ${TIZEN_ARCH} base"
-fetch_tizen_pkgs_init standard Tizen-Base
+fetch_tizen_pkgs_init standard $BASE
 Inform "fetch common packages"
 fetch_tizen_pkgs ${TIZEN_ARCH} gcc gcc-devel-static glibc glibc-devel libicu libicu-devel libatomic linux-glibc-devel keyutils keyutils-devel libkeyutils
 Inform "fetch coreclr packages"
-fetch_tizen_pkgs ${TIZEN_ARCH} lldb lldb-devel libgcc libstdc++ libstdc++-devel libunwind libunwind-devel lttng-ust-devel lttng-ust userspace-rcu-devel userspace-rcu
+fetch_tizen_pkgs ${TIZEN_ARCH} libgcc libstdc++ libstdc++-devel libunwind libunwind-devel lttng-ust-devel lttng-ust userspace-rcu-devel userspace-rcu
+if [ "$TIZEN_ARCH" != "riscv64" ]; then
+    fetch_tizen_pkgs ${TIZEN_ARCH} lldb lldb-devel
+fi
 Inform "fetch corefx packages"
 fetch_tizen_pkgs ${TIZEN_ARCH} libcom_err libcom_err-devel zlib zlib-devel libopenssl11 libopenssl1.1-devel krb5 krb5-devel
 
 Inform "Initialize standard unified"
-fetch_tizen_pkgs_init standard Tizen-Unified
+fetch_tizen_pkgs_init standard $UNIFIED
 Inform "fetch corefx packages"
 fetch_tizen_pkgs ${TIZEN_ARCH} gssdp gssdp-devel tizen-release
 

eng/common/cross/toolchain.cmake

@@ -80,6 +80,9 @@ elseif(TARGET_ARCH_NAME STREQUAL "riscv64")
     set(TOOLCHAIN "riscv64-alpine-linux-musl")
   else()
     set(TOOLCHAIN "riscv64-linux-gnu")
+    if(TIZEN)
+      set(TIZEN_TOOLCHAIN "riscv64-tizen-linux-gnu/13.1.0")
+    endif()
   endif()
 elseif(TARGET_ARCH_NAME STREQUAL "s390x")
   set(CMAKE_SYSTEM_PROCESSOR s390x)
@@ -144,6 +147,10 @@ if(TIZEN)
     include_directories(SYSTEM ${CROSS_ROOTFS}/usr/lib64/gcc/${TIZEN_TOOLCHAIN}/include/c++/)
     include_directories(SYSTEM ${CROSS_ROOTFS}/usr/lib64/gcc/${TIZEN_TOOLCHAIN}/include/c++/x86_64-tizen-linux-gnu)
   endif()
+  if(TARGET_ARCH_NAME STREQUAL "riscv64")
+    include_directories(SYSTEM ${CROSS_ROOTFS}/usr/lib64/gcc/${TIZEN_TOOLCHAIN}/include/c++/)
+    include_directories(SYSTEM ${CROSS_ROOTFS}/usr/lib64/gcc/${TIZEN_TOOLCHAIN}/include/c++/riscv64-tizen-linux-gnu)
+  endif()
 endif()
 
 if(ANDROID)
@@ -207,6 +214,7 @@ elseif(ILLUMOS)
     set(CMAKE_CXX_STANDARD_LIBRARIES "${CMAKE_CXX_STANDARD_LIBRARIES} -lssp")
 elseif(HAIKU)
     set(CMAKE_SYSROOT "${CROSS_ROOTFS}")
+    set(CMAKE_PROGRAM_PATH "${CMAKE_PROGRAM_PATH};${CROSS_ROOTFS}/cross-tools-x86_64/bin")
 
     set(TOOLSET_PREFIX ${TOOLCHAIN}-)
     function(locate_toolchain_exec exec var)
@@ -217,7 +225,6 @@ elseif(HAIKU)
         endif()
 
         find_program(EXEC_LOCATION_${exec}
-            PATHS "${CROSS_ROOTFS}/cross-tools-x86_64/bin"
             NAMES
             "${TOOLSET_PREFIX}${exec}${CLR_CMAKE_COMPILER_FILE_NAME_VERSION}"
             "${TOOLSET_PREFIX}${exec}")
@@ -270,7 +277,7 @@ if(TARGET_ARCH_NAME MATCHES "^(arm|armel)$")
     add_toolchain_linker_flag("-L${CROSS_ROOTFS}/usr/lib")
     add_toolchain_linker_flag("-L${CROSS_ROOTFS}/usr/lib/gcc/${TIZEN_TOOLCHAIN}")
   endif()
-elseif(TARGET_ARCH_NAME MATCHES "^(arm64|x64)$")
+elseif(TARGET_ARCH_NAME MATCHES "^(arm64|x64|riscv64)$")
   if(TIZEN)
     add_toolchain_linker_flag("-B${CROSS_ROOTFS}/usr/lib64/gcc/${TIZEN_TOOLCHAIN}")
     add_toolchain_linker_flag("-L${CROSS_ROOTFS}/lib64")
@@ -281,6 +288,8 @@ elseif(TARGET_ARCH_NAME MATCHES "^(arm64|x64)$")
     add_toolchain_linker_flag("-Wl,--rpath-link=${CROSS_ROOTFS}/usr/lib64")
     add_toolchain_linker_flag("-Wl,--rpath-link=${CROSS_ROOTFS}/usr/lib64/gcc/${TIZEN_TOOLCHAIN}")
   endif()
+elseif(TARGET_ARCH_NAME STREQUAL "s390x")
+  add_toolchain_linker_flag("--target=${TOOLCHAIN}")
 elseif(TARGET_ARCH_NAME STREQUAL "x86")
   if(EXISTS ${CROSS_ROOTFS}/usr/lib/gcc/i586-alpine-linux-musl)
     add_toolchain_linker_flag("--target=${TOOLCHAIN}")
@@ -328,6 +337,8 @@ if(TARGET_ARCH_NAME MATCHES "^(arm|armel)$")
   if(TARGET_ARCH_NAME STREQUAL "armel")
     add_compile_options(-mfloat-abi=softfp)
   endif()
+elseif(TARGET_ARCH_NAME STREQUAL "s390x")
+  add_compile_options("--target=${TOOLCHAIN}")
 elseif(TARGET_ARCH_NAME STREQUAL "x86")
   if(EXISTS ${CROSS_ROOTFS}/usr/lib/gcc/i586-alpine-linux-musl)
     add_compile_options(--target=${TOOLCHAIN})

eng/common/dotnet-install.sh

@@ -54,6 +54,10 @@ cpuname=$(uname -m)
 case $cpuname in
   arm64|aarch64)
     buildarch=arm64
+    if [ "$(getconf LONG_BIT)" -lt 64 ]; then
+        # This is 32-bit OS running on 64-bit CPU (for example Raspberry Pi OS)
+        buildarch=arm
+    fi
     ;;
   loongarch64)
     buildarch=loongarch64

eng/common/native/init-compiler.sh

@@ -63,7 +63,7 @@ if [ -z "$CLR_CC" ]; then
     # Set default versions
     if [ -z "$majorVersion" ]; then
         # note: gcc (all versions) and clang versions higher than 6 do not have minor version in file name, if it is zero.
-        if [ "$compiler" = "clang" ]; then versions="16 15 14 13 12 11 10 9 8 7 6.0 5.0 4.0 3.9 3.8 3.7 3.6 3.5"
+        if [ "$compiler" = "clang" ]; then versions="17 16 15 14 13 12 11 10 9 8 7 6.0 5.0 4.0 3.9 3.8 3.7 3.6 3.5"
         elif [ "$compiler" = "gcc" ]; then versions="13 12 11 10 9 8 7 6 5 4.9"; fi
 
         for version in $versions; do

eng/common/native/init-distro-rid.sh

@@ -0,0 +1,130 @@
+#!/usr/bin/env bash
+
+# getNonPortableDistroRid
+#
+# Input:
+#   targetOs: (str)
+#   targetArch: (str)
+#   rootfsDir: (str)
+#
+# Return:
+#   non-portable rid
+getNonPortableDistroRid()
+{
+    local targetOs="$1"
+    local targetArch="$2"
+    local rootfsDir="$3"
+    local nonPortableRid=""
+
+    if [ "$targetOs" = "linux" ]; then
+        if [ -e "${rootfsDir}/etc/os-release" ]; then
+            source "${rootfsDir}/etc/os-release"
+
+            if [[ "${ID}" == "rhel" || "${ID}" == "rocky" || "${ID}" == "alpine" ]]; then
+                # remove the last version digit
+                VERSION_ID="${VERSION_ID%.*}"
+            fi
+
+            if [[ "${VERSION_ID:-}" =~ ^([[:digit:]]|\.)+$ ]]; then
+                nonPortableRid="${ID}.${VERSION_ID}-${targetArch}"
+            else
+                # Rolling release distros either do not set VERSION_ID, set it as blank or
+                # set it to non-version looking string (such as TEMPLATE_VERSION_ID on ArchLinux);
+                # so omit it here to be consistent with everything else.
+                nonPortableRid="${ID}-${targetArch}"
+            fi
+
+        elif [ -e "${rootfsDir}/android_platform" ]; then
+            source "$rootfsDir"/android_platform
+            nonPortableRid="$RID"
+        fi
+    fi
+
+    if [ "$targetOs" = "freebsd" ]; then
+        # $rootfsDir can be empty. freebsd-version is shell script and it should always work.
+        __freebsd_major_version=$($rootfsDir/bin/freebsd-version | { read v; echo "${v%%.*}"; })
+        nonPortableRid="freebsd.$__freebsd_major_version-${targetArch}"
+    elif command -v getprop && getprop ro.product.system.model 2>&1 | grep -qi android; then
+        __android_sdk_version=$(getprop ro.build.version.sdk)
+        nonPortableRid="android.$__android_sdk_version-${targetArch}"
+    elif [ "$targetOs" = "illumos" ]; then
+        __uname_version=$(uname -v)
+        case "$__uname_version" in
+            omnios-*)
+                __omnios_major_version=$(echo "${__uname_version:8:2}")
+                nonPortableRid=omnios."$__omnios_major_version"-"$targetArch"
+            ;;
+            joyent_*)
+                __smartos_major_version=$(echo "${__uname_version:7:4}")
+                nonPortableRid=smartos."$__smartos_major_version"-"$targetArch"
+            ;;
+            illumos_*)
+                nonPortableRid=openindiana-"$targetArch"
+            ;;
+        esac
+    elif [ "$targetOs" = "solaris" ]; then
+        __uname_version=$(uname -v)
+        __solaris_major_version=$(echo "${__uname_version%.*}")
+        nonPortableRid=solaris."$__solaris_major_version"-"$targetArch"
+    elif [ "$targetOs" = "haiku" ]; then
+        __uname_release=$(uname -r)
+        nonPortableRid=haiku.r"$__uname_release"-"$targetArch"
+    fi
+
+    echo "$(echo $nonPortableRid | tr '[:upper:]' '[:lower:]')"
+}
+
+# initDistroRidGlobal
+#
+# Input:
+#   os: (str)
+#   arch: (str)
+#   rootfsDir?: (nullable:string)
+#
+# Return:
+#   None
+#
+# Notes:
+#
+# It is important to note that the function does not return anything, but it
+# exports the following variables on success:
+#
+#   __DistroRid   : Non-portable rid of the target platform.
+#   __PortableTargetOS  : OS-part of the portable rid that corresponds to the target platform.
+#
+initDistroRidGlobal()
+{
+    local targetOs="$1"
+    local targetArch="$2"
+    local rootfsDir=""
+    if [ "$#" -ge 3 ]; then
+        rootfsDir="$3"
+    fi
+
+    if [ -n "${rootfsDir}" ]; then
+        # We may have a cross build. Check for the existence of the rootfsDir
+        if [ ! -e "${rootfsDir}" ]; then
+            echo "Error rootfsDir has been passed, but the location is not valid."
+            exit 1
+        fi
+    fi
+
+    __DistroRid=$(getNonPortableDistroRid "${targetOs}" "${targetArch}" "${rootfsDir}")
+
+    if [ -z "${__PortableTargetOS:-}" ]; then
+        __PortableTargetOS="$targetOs"
+
+        STRINGS="$(command -v strings || true)"
+        if [ -z "$STRINGS" ]; then
+            STRINGS="$(command -v llvm-strings || true)"
+        fi
+
+        # Check for musl-based distros (e.g Alpine Linux, Void Linux).
+        if "${rootfsDir}/usr/bin/ldd" --version 2>&1 | grep -q musl ||
+                ( [ -n "$STRINGS" ] && "$STRINGS" "${rootfsDir}/usr/bin/ldd" 2>&1 | grep -q musl ); then
+            __PortableTargetOS="linux-musl"
+        fi
+    fi
+
+    export __DistroRid __PortableTargetOS
+}

eng/common/native/init-os-and-arch.sh

@@ -0,0 +1,80 @@
+#!/usr/bin/env bash
+
+# Use uname to determine what the OS is.
+OSName=$(uname -s | tr '[:upper:]' '[:lower:]')
+
+if command -v getprop && getprop ro.product.system.model 2>&1 | grep -qi android; then
+    OSName="android"
+fi
+
+case "$OSName" in
+freebsd|linux|netbsd|openbsd|sunos|android|haiku)
+    os="$OSName" ;;
+darwin)
+    os=osx ;;
+*)
+    echo "Unsupported OS $OSName detected!"
+    exit 1 ;;
+esac
+
+# On Solaris, `uname -m` is discouraged, see https://docs.oracle.com/cd/E36784_01/html/E36870/uname-1.html
+# and `uname -p` returns processor type (e.g. i386 on amd64).
+# The appropriate tool to determine CPU is isainfo(1) https://docs.oracle.com/cd/E36784_01/html/E36870/isainfo-1.html.
+if [ "$os" = "sunos" ]; then
+    if uname -o 2>&1 | grep -q illumos; then
+        os="illumos"
+    else
+        os="solaris"
+    fi
+    CPUName=$(isainfo -n)
+else
+    # For the rest of the operating systems, use uname(1) to determine what the CPU is.
+    CPUName=$(uname -m)
+fi
+
+case "$CPUName" in
+    arm64|aarch64)
+        arch=arm64
+        ;;
+
+    loongarch64)
+        arch=loongarch64
+        ;;
+
+    riscv64)
+        arch=riscv64
+        ;;
+
+    amd64|x86_64)
+        arch=x64
+        ;;
+
+    armv7l|armv8l)
+        if (NAME=""; . /etc/os-release; test "$NAME" = "Tizen"); then
+            arch=armel
+        else
+            arch=arm
+        fi
+        ;;
+
+    armv6l)
+        arch=armv6
+        ;;
+
+    i[3-6]86)
+        echo "Unsupported CPU $CPUName detected, build might not succeed!"
+        arch=x86
+        ;;
+
+    s390x)
+        arch=s390x
+        ;;
+
+    ppc64le)
+        arch=ppc64le
+        ;;
+    *)
+        echo "Unknown CPU $CPUName detected!"
+        exit 1
+        ;;
+esac

eng/common/post-build/check-channel-consistency.ps1

@@ -7,7 +7,7 @@ try {
   . $PSScriptRoot\post-build-utils.ps1
 
   if ($PromoteToChannels -eq "") {
-    Write-PipelineTaskError -Type 'warning' -Message "This build won't publish assets as it's not configured to any Maestro channel. If that wasn't intended use Darc to configure a default channel using add-default-channel for this branch or to promote it to a channel using add-build-to-channel. See https://github.com/dotnet/arcade/blob/master/Documentation/Darc.md#assigning-an-individual-build-to-a-channel for more info."
+    Write-PipelineTaskError -Type 'warning' -Message "This build won't publish assets as it's not configured to any Maestro channel. If that wasn't intended use Darc to configure a default channel using add-default-channel for this branch or to promote it to a channel using add-build-to-channel. See https://github.com/dotnet/arcade/blob/main/Documentation/Darc.md#assigning-an-individual-build-to-a-channel for more info."
     ExitWithExitCode 0
   }
 

eng/common/sdk-task.ps1

@@ -64,7 +64,7 @@ try {
       $GlobalJson.tools | Add-Member -Name "vs" -Value (ConvertFrom-Json "{ `"version`": `"16.5`" }") -MemberType NoteProperty
     }
     if( -not ($GlobalJson.tools.PSObject.Properties.Name -match "xcopy-msbuild" )) {
-      $GlobalJson.tools | Add-Member -Name "xcopy-msbuild" -Value "17.4.1" -MemberType NoteProperty
+      $GlobalJson.tools | Add-Member -Name "xcopy-msbuild" -Value "17.8.1-2" -MemberType NoteProperty
     }
     if ($GlobalJson.tools."xcopy-msbuild".Trim() -ine "none") {
         $xcopyMSBuildToolsFolder = InitializeXCopyMSBuild $GlobalJson.tools."xcopy-msbuild" -install $true

eng/common/sdl/configure-sdl-tool.ps1

@@ -17,7 +17,9 @@ Param(
   # Optional: Additional params to add to any tool using PoliCheck.
   [string[]] $PoliCheckAdditionalRunConfigParams,
   # Optional: Additional params to add to any tool using CodeQL/Semmle.
-  [string[]] $CodeQLAdditionalRunConfigParams
+  [string[]] $CodeQLAdditionalRunConfigParams,
+  # Optional: Additional params to add to any tool using Binskim.
+  [string[]] $BinskimAdditionalRunConfigParams
 )
 
 $ErrorActionPreference = 'Stop'
@@ -69,22 +71,34 @@ try {
     $gdnConfigFile = Join-Path $gdnConfigPath "$toolConfigName-configure.gdnconfig"
 
     # For some tools, add default and automatic args.
-    if ($tool.Name -eq 'credscan') {
-      if ($targetDirectory) {
-        $tool.Args += "`"TargetDirectory < $TargetDirectory`""
+    switch -Exact ($tool.Name) {
+      'credscan' {
+        if ($targetDirectory) {
+          $tool.Args += "`"TargetDirectory < $TargetDirectory`""
+        }
+        $tool.Args += "`"OutputType < pre`""
+        $tool.Args += $CrScanAdditionalRunConfigParams
       }
-      $tool.Args += "`"OutputType < pre`""
-      $tool.Args += $CrScanAdditionalRunConfigParams
-    } elseif ($tool.Name -eq 'policheck') {
-      if ($targetDirectory) {
-        $tool.Args += "`"Target < $TargetDirectory`""
+      'policheck' {
+        if ($targetDirectory) {
+          $tool.Args += "`"Target < $TargetDirectory`""
+        }
+        $tool.Args += $PoliCheckAdditionalRunConfigParams
       }
-      $tool.Args += $PoliCheckAdditionalRunConfigParams
-    } elseif ($tool.Name -eq 'semmle' -or $tool.Name -eq 'codeql') {
-      if ($targetDirectory) {
-        $tool.Args += "`"SourceCodeDirectory < $TargetDirectory`""
+      {$_ -in 'semmle', 'codeql'} {
+        if ($targetDirectory) {
+          $tool.Args += "`"SourceCodeDirectory < $TargetDirectory`""
+        }
+        $tool.Args += $CodeQLAdditionalRunConfigParams
+      }
+      'binskim' {
+        if ($targetDirectory) {
+          # Binskim crashes due to specific PDBs. GitHub issue: https://github.com/microsoft/binskim/issues/924.
+          # We are excluding all `_.pdb` files from the scan.
+          $tool.Args += "`"Target < $TargetDirectory\**;-:file|$TargetDirectory\**\_.pdb`""
+        }
+        $tool.Args += $BinskimAdditionalRunConfigParams
       }
-      $tool.Args += $CodeQLAdditionalRunConfigParams
     }
 
     # Create variable pointing to the args array directly so we can use splat syntax later.

eng/common/sdl/execute-all-sdl-tools.ps1

@@ -35,6 +35,7 @@ Param(
   [string[]] $CrScanAdditionalRunConfigParams,                                                   # Optional: Additional Params to custom build a CredScan run config in the format @("xyz:abc","sdf:1")
   [string[]] $PoliCheckAdditionalRunConfigParams,                                                # Optional: Additional Params to custom build a Policheck run config in the format @("xyz:abc","sdf:1")
   [string[]] $CodeQLAdditionalRunConfigParams,                                                   # Optional: Additional Params to custom build a Semmle/CodeQL run config in the format @("xyz < abc","sdf < 1")
+  [string[]] $BinskimAdditionalRunConfigParams,                                                  # Optional: Additional Params to custom build a Binskim run config in the format @("xyz < abc","sdf < 1")
   [bool] $BreakOnFailure=$False                                                                  # Optional: Fail the build if there were errors during the run
 )
 
@@ -107,7 +108,8 @@ try {
           -GuardianLoggerLevel $GuardianLoggerLevel `
           -CrScanAdditionalRunConfigParams $CrScanAdditionalRunConfigParams `
           -PoliCheckAdditionalRunConfigParams $PoliCheckAdditionalRunConfigParams `
-          -CodeQLAdditionalRunConfigParams $CodeQLAdditionalRunConfigParams
+          -CodeQLAdditionalRunConfigParams $CodeQLAdditionalRunConfigParams `
+          -BinskimAdditionalRunConfigParams $BinskimAdditionalRunConfigParams
         if ($BreakOnFailure) {
           Exit-IfNZEC "Sdl"
         }

eng/common/sdl/extract-artifact-packages.ps1

@@ -35,31 +35,33 @@ try {
     param( 
       [string] $PackagePath                                 # Full path to a NuGet package
     )
-    
+
     if (!(Test-Path $PackagePath)) {
       Write-PipelineTelemetryError -Category 'Build' -Message "Input file does not exist: $PackagePath"
       ExitWithExitCode 1
     }
-    
+
     $RelevantExtensions = @('.dll', '.exe', '.pdb')
     Write-Host -NoNewLine 'Extracting ' ([System.IO.Path]::GetFileName($PackagePath)) '...'
-  
+
     $PackageId = [System.IO.Path]::GetFileNameWithoutExtension($PackagePath)
     $ExtractPath = Join-Path -Path $using:ExtractPath -ChildPath $PackageId
-  
+
     Add-Type -AssemblyName System.IO.Compression.FileSystem
-  
+
     [System.IO.Directory]::CreateDirectory($ExtractPath);
-  
+
     try {
       $zip = [System.IO.Compression.ZipFile]::OpenRead($PackagePath)
   
       $zip.Entries | 
       Where-Object {$RelevantExtensions -contains [System.IO.Path]::GetExtension($_.Name)} |
         ForEach-Object {
-            $TargetFile = Join-Path -Path $ExtractPath -ChildPath $_.Name
-  
-            [System.IO.Compression.ZipFileExtensions]::ExtractToFile($_, $TargetFile, $true)
+            $TargetPath = Join-Path -Path $ExtractPath -ChildPath (Split-Path -Path $_.FullName)
+            [System.IO.Directory]::CreateDirectory($TargetPath);
+
+            $TargetFile = Join-Path -Path $ExtractPath -ChildPath $_.FullName
+            [System.IO.Compression.ZipFileExtensions]::ExtractToFile($_, $TargetFile)
           }
     }
     catch {

eng/common/sdl/trim-assets-version.ps1

@@ -0,0 +1,75 @@
+<#
+.SYNOPSIS
+Install and run the 'Microsoft.DotNet.VersionTools.Cli' tool with the 'trim-artifacts-version' command to trim the version from the NuGet assets file name.
+
+.PARAMETER InputPath
+Full path to directory where artifact packages are stored
+
+.PARAMETER Recursive
+Search for NuGet packages recursively
+
+#>
+
+Param(
+  [string] $InputPath,
+  [bool] $Recursive = $true
+)
+
+$CliToolName = "Microsoft.DotNet.VersionTools.Cli"
+
+function Install-VersionTools-Cli {
+  param(
+      [Parameter(Mandatory=$true)][string]$Version
+  )
+
+  Write-Host "Installing the package '$CliToolName' with a version of '$version' ..."
+  $feed = "https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-eng/nuget/v3/index.json"
+
+  $argumentList = @("tool", "install", "--local", "$CliToolName", "--add-source $feed", "--no-cache", "--version $Version", "--create-manifest-if-needed")
+  Start-Process "$dotnet" -Verbose -ArgumentList $argumentList -NoNewWindow -Wait
+}
+
+# -------------------------------------------------------------------
+
+if (!(Test-Path $InputPath)) {
+  Write-Host "Input Path '$InputPath' does not exist"
+  ExitWithExitCode 1
+}
+
+$ErrorActionPreference = 'Stop'
+Set-StrictMode -Version 2.0
+
+$disableConfigureToolsetImport = $true
+$global:LASTEXITCODE = 0
+
+# `tools.ps1` checks $ci to perform some actions. Since the SDL
+# scripts don't necessarily execute in the same agent that run the
+# build.ps1/sh script this variable isn't automatically set.
+$ci = $true
+. $PSScriptRoot\..\tools.ps1
+
+try {
+  $dotnetRoot = InitializeDotNetCli -install:$true
+  $dotnet = "$dotnetRoot\dotnet.exe"
+
+  $toolsetVersion = Read-ArcadeSdkVersion
+  Install-VersionTools-Cli -Version $toolsetVersion
+
+  $cliToolFound = (& "$dotnet" tool list --local | Where-Object {$_.Split(' ')[0] -eq $CliToolName})
+  if ($null -eq $cliToolFound) {
+    Write-PipelineTelemetryError -Force -Category 'Sdl' -Message "The '$CliToolName' tool is not installed."
+    ExitWithExitCode 1
+  }
+
+  Exec-BlockVerbosely {
+    & "$dotnet" $CliToolName trim-assets-version `
+      --assets-path $InputPath `
+      --recursive $Recursive
+    Exit-IfNZEC "Sdl"
+  }
+}
+catch {
+  Write-Host $_
+  Write-PipelineTelemetryError -Force -Category 'Sdl' -Message $_
+  ExitWithExitCode 1
+}

eng/common/templates/job/execute-sdl.yml

@@ -105,6 +105,11 @@ jobs:
         downloadPath: $(Build.ArtifactStagingDirectory)\artifacts
         checkDownloadedFiles: true
 
+  - powershell: eng/common/sdl/trim-assets-version.ps1
+      -InputPath $(Build.ArtifactStagingDirectory)\artifacts
+    displayName: Trim the version from the NuGet packages
+    continueOnError: ${{ parameters.sdlContinueOnError }}
+
   - powershell: eng/common/sdl/extract-artifact-packages.ps1
       -InputPath $(Build.ArtifactStagingDirectory)\artifacts\BlobArtifacts
       -ExtractPath $(Build.ArtifactStagingDirectory)\artifacts\BlobArtifacts

eng/common/templates/job/job.yml

@@ -75,6 +75,10 @@ jobs:
   - ${{ if eq(parameters.enableRichCodeNavigation, 'true') }}:
     - name: EnableRichCodeNavigation
       value: 'true'
+  # Retry signature validation up to three times, waiting 2 seconds between attempts.
+  # See https://learn.microsoft.com/en-us/nuget/reference/errors-and-warnings/nu3028#retry-untrusted-root-failures
+  - name: NUGET_EXPERIMENTAL_CHAIN_BUILD_RETRY_POLICY
+    value: 3,2000
   - ${{ each variable in parameters.variables }}:
     # handle name-value variable syntax
     # example:
@@ -83,7 +87,7 @@ jobs:
     - ${{ if ne(variable.name, '') }}:
       - name: ${{ variable.name }}
         value: ${{ variable.value }}
-    
+
     # handle variable groups
     - ${{ if ne(variable.group, '') }}:
       - group: ${{ variable.group }}
@@ -150,7 +154,7 @@ jobs:
       displayName: RichCodeNav Upload
       inputs:
         languages: ${{ coalesce(parameters.richCodeNavigationLanguage, 'csharp') }}
-        environment: ${{ coalesce(parameters.richCodeNavigationEnvironment, 'production') }}
+        environment: ${{ coalesce(parameters.richCodeNavigationEnvironment, 'internal') }}
         richNavLogOutputDirectory: $(Build.SourcesDirectory)/artifacts/bin
         uploadRichNavArtifacts: ${{ coalesce(parameters.richCodeNavigationUploadArtifacts, false) }}
       continueOnError: true
@@ -169,7 +173,7 @@ jobs:
   - ${{ if eq(parameters.enableMicrobuild, 'true') }}:
     - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}:
       - task: MicroBuildCleanup@1
-        displayName: Execute Microbuild cleanup tasks  
+        displayName: Execute Microbuild cleanup tasks
         condition: and(always(), in(variables['_SignType'], 'real', 'test'), eq(variables['Agent.Os'], 'Windows_NT'))
         continueOnError: ${{ parameters.continueOnError }}
         env:
@@ -219,7 +223,7 @@ jobs:
       displayName: Publish XUnit Test Results
       inputs:
         testResultsFormat: 'xUnit'
-        testResultsFiles: '*.xml' 
+        testResultsFiles: '*.xml'
         searchFolder: '$(Build.SourcesDirectory)/artifacts/TestResults/$(_BuildConfig)'
         testRunTitle: ${{ coalesce(parameters.testRunTitle, parameters.name, '$(System.JobName)') }}-xunit
         mergeTestResults: ${{ parameters.mergeTestResults }}
@@ -230,7 +234,7 @@ jobs:
       displayName: Publish TRX Test Results
       inputs:
         testResultsFormat: 'VSTest'
-        testResultsFiles: '*.trx' 
+        testResultsFiles: '*.trx'
         searchFolder: '$(Build.SourcesDirectory)/artifacts/TestResults/$(_BuildConfig)'
         testRunTitle: ${{ coalesce(parameters.testRunTitle, parameters.name, '$(System.JobName)') }}-trx
         mergeTestResults: ${{ parameters.mergeTestResults }}

eng/common/templates/steps/execute-sdl.yml

@@ -33,7 +33,7 @@ steps:
 
 - ${{ if ne(parameters.overrideParameters, '') }}:
   - powershell: ${{ parameters.executeAllSdlToolsScript }} ${{ parameters.overrideParameters }}
-    displayName: Execute SDL
+    displayName: Execute SDL (Overridden)
     continueOnError: ${{ parameters.sdlContinueOnError }}
     condition: ${{ parameters.condition }}
 

eng/common/templates/steps/source-build.yml

@@ -68,6 +68,11 @@ steps:
       runtimeOsArgs='/p:RuntimeOS=${{ parameters.platform.runtimeOS }}'
     fi
 
+    baseOsArgs=
+    if [ '${{ parameters.platform.baseOS }}' != '' ]; then
+      baseOsArgs='/p:BaseOS=${{ parameters.platform.baseOS }}'
+    fi
+
     publishArgs=
     if [ '${{ parameters.platform.skipPublishValidation }}' != 'true' ]; then
       publishArgs='--publish'
@@ -86,6 +91,7 @@ steps:
       $internalRestoreArgs \
       $targetRidArgs \
       $runtimeOsArgs \
+      $baseOsArgs \
       /p:SourceBuildNonPortable=${{ parameters.platform.nonPortable }} \
       /p:ArcadeBuildFromSource=true \
       /p:AssetManifestFileName=$assetManifestFileName
@@ -112,3 +118,12 @@ steps:
     artifactName: BuildLogs_SourceBuild_${{ parameters.platform.name }}_Attempt$(System.JobAttempt)
   continueOnError: true
   condition: succeededOrFailed()
+
+# Manually inject component detection so that we can ignore the source build upstream cache, which contains
+# a nupkg cache of input packages (a local feed).
+# This path must match the upstream cache path in property 'CurrentRepoSourceBuiltNupkgCacheDir'
+# in src\Microsoft.DotNet.Arcade.Sdk\tools\SourceBuild\SourceBuildArcade.targets
+- task: ComponentGovernanceComponentDetection@0
+  displayName: Component Detection (Exclude upstream cache)
+  inputs:
+    ignoreDirectories: '$(Build.SourcesDirectory)/artifacts/source-build/self/src/artifacts/obj/source-built-upstream-cache'

eng/common/tools.ps1

@@ -158,7 +158,7 @@ function InitializeDotNetCli([bool]$install, [bool]$createSdkLocationFile) {
   $env:DOTNET_MULTILEVEL_LOOKUP=0
 
   # Disable first run since we do not need all ASP.NET packages restored.
-  $env:DOTNET_SKIP_FIRST_TIME_EXPERIENCE=1
+  $env:DOTNET_NOLOGO=1
 
   # Disable telemetry on CI.
   if ($ci) {
@@ -228,7 +228,7 @@ function InitializeDotNetCli([bool]$install, [bool]$createSdkLocationFile) {
   Write-PipelinePrependPath -Path $dotnetRoot
 
   Write-PipelineSetVariable -Name 'DOTNET_MULTILEVEL_LOOKUP' -Value '0'
-  Write-PipelineSetVariable -Name 'DOTNET_SKIP_FIRST_TIME_EXPERIENCE' -Value '1'
+  Write-PipelineSetVariable -Name 'DOTNET_NOLOGO' -Value '1'
 
   return $global:_DotNetInstallDir = $dotnetRoot
 }
@@ -379,13 +379,13 @@ function InitializeVisualStudioMSBuild([bool]$install, [object]$vsRequirements =
   }
 
   # Minimum VS version to require.
-  $vsMinVersionReqdStr = '16.8'
+  $vsMinVersionReqdStr = '17.7'
   $vsMinVersionReqd = [Version]::new($vsMinVersionReqdStr)
 
   # If the version of msbuild is going to be xcopied,
   # use this version. Version matches a package here:
-  # https://dev.azure.com/dnceng/public/_packaging?_a=package&feed=dotnet-eng&package=RoslynTools.MSBuild&protocolType=NuGet&version=17.4.1&view=overview
-  $defaultXCopyMSBuildVersion = '17.4.1'
+  # https://dev.azure.com/dnceng/public/_artifacts/feed/dotnet-eng/NuGet/RoslynTools.MSBuild/versions/17.8.1-2
+  $defaultXCopyMSBuildVersion = '17.8.1-2'
 
   if (!$vsRequirements) {
     if (Get-Member -InputObject $GlobalJson.tools -Name 'vs') {
@@ -671,6 +671,10 @@ function InitializeNativeTools() {
   }
 }
 
+function Read-ArcadeSdkVersion() {
+  return $GlobalJson.'msbuild-sdks'.'Microsoft.DotNet.Arcade.Sdk'
+}
+
 function InitializeToolset() {
   if (Test-Path variable:global:_ToolsetBuildProj) {
     return $global:_ToolsetBuildProj
@@ -678,7 +682,7 @@ function InitializeToolset() {
 
   $nugetCache = GetNuGetPackageCachePath
 
-  $toolsetVersion = $GlobalJson.'msbuild-sdks'.'Microsoft.DotNet.Arcade.Sdk'
+  $toolsetVersion = Read-ArcadeSdkVersion
   $toolsetLocationFile = Join-Path $ToolsetDir "$toolsetVersion.txt"
 
   if (Test-Path $toolsetLocationFile) {

eng/common/tools.sh

@@ -112,7 +112,7 @@ function InitializeDotNetCli {
   export DOTNET_MULTILEVEL_LOOKUP=0
 
   # Disable first run since we want to control all package sources
-  export DOTNET_SKIP_FIRST_TIME_EXPERIENCE=1
+  export DOTNET_NOLOGO=1
 
   # Disable telemetry on CI
   if [[ $ci == true ]]; then
@@ -165,7 +165,7 @@ function InitializeDotNetCli {
   Write-PipelinePrependPath -path "$dotnet_root"
 
   Write-PipelineSetVariable -name "DOTNET_MULTILEVEL_LOOKUP" -value "0"
-  Write-PipelineSetVariable -name "DOTNET_SKIP_FIRST_TIME_EXPERIENCE" -value "1"
+  Write-PipelineSetVariable -name "DOTNET_NOLOGO" -value "1"
 
   # return value
   _InitializeDotNetCli="$dotnet_root"
@@ -310,7 +310,7 @@ function GetDotNetInstallScript {
       curl "$install_script_url" -sSL --retry 10 --create-dirs -o "$install_script" || {
         if command -v openssl &> /dev/null; then
           echo "Curl failed; dumping some information about dotnet.microsoft.com for later investigation"
-          echo | openssl s_client -showcerts -servername dotnet.microsoft.com  -connect dotnet.microsoft.com:443
+          echo | openssl s_client -showcerts -servername dotnet.microsoft.com  -connect dotnet.microsoft.com:443 || true
         fi
         echo "Will now retry the same URL with verbose logging."
         with_retries curl "$install_script_url" -sSL --verbose --retry 10 --create-dirs -o "$install_script" || {
@@ -341,7 +341,12 @@ function InitializeBuildTool {
   # return values
   _InitializeBuildTool="$_InitializeDotNetCli/dotnet"
   _InitializeBuildToolCommand="msbuild"
-  _InitializeBuildToolFramework="net8.0"
+  # use override if it exists - commonly set by source-build
+  if [[ "${_OverrideArcadeInitializeBuildToolFramework:-x}" == "x" ]]; then
+    _InitializeBuildToolFramework="net8.0"
+  else
+    _InitializeBuildToolFramework="${_OverrideArcadeInitializeBuildToolFramework}"
+  fi
 }
 
 # Set RestoreNoCache as a workaround for https://github.com/NuGet/Home/issues/3116

global.json

@@ -3,7 +3,7 @@
     "allowPrerelease": true
   },
   "tools": {
-    "dotnet": "8.0.100-preview.3.23178.7",
+    "dotnet": "8.0.100",
     "runtimes": {
       "dotnet": [
         "6.0.0"
@@ -14,6 +14,6 @@
     }
   },
   "msbuild-sdks": {
-    "Microsoft.DotNet.Arcade.Sdk": "8.0.0-beta.23265.1"
+    "Microsoft.DotNet.Arcade.Sdk": "9.0.0-beta.23567.1"
   }
 }