diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index 4139151b..10e46191 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -3,25 +3,25 @@ - + https://github.com/dotnet/arcade - 590a102630c7efc7ca6f652f7c6c47dee4c4086c + 243cc92161ad44c2a07464425892daee19121c99 - + https://github.com/dotnet/arcade - 590a102630c7efc7ca6f652f7c6c47dee4c4086c + 243cc92161ad44c2a07464425892daee19121c99 - + https://github.com/dotnet/arcade - 590a102630c7efc7ca6f652f7c6c47dee4c4086c + 243cc92161ad44c2a07464425892daee19121c99 - + https://github.com/dotnet/arcade - 590a102630c7efc7ca6f652f7c6c47dee4c4086c + 243cc92161ad44c2a07464425892daee19121c99 - + https://github.com/dotnet/arcade - 590a102630c7efc7ca6f652f7c6c47dee4c4086c + 243cc92161ad44c2a07464425892daee19121c99 https://github.com/dotnet/arcade-services diff --git a/eng/common/build.ps1 b/eng/common/build.ps1 index 813d440d..67ee6d28 100644 --- a/eng/common/build.ps1 +++ b/eng/common/build.ps1 @@ -20,6 +20,7 @@ Param( [switch] $publish, [switch] $clean, [switch][Alias('bl')]$binaryLog, + [switch][Alias('nobl')]$excludeCIBinarylog, [switch] $ci, [switch] $prepareMachine, [switch] $help, @@ -58,6 +59,7 @@ function Print-Usage() { Write-Host "Advanced settings:" Write-Host " -projects Semi-colon delimited list of sln/proj's to build. Globbing is supported (*.sln)" Write-Host " -ci Set when running on CI server" + Write-Host " -excludeCIBinarylog Don't output binary log (short: -nobl)" Write-Host " -prepareMachine Prepare machine for CI run, clean up processes after build" Write-Host " -warnAsError Sets warnaserror msbuild parameter ('true' or 'false')" Write-Host " -msbuildEngine Msbuild engine to use to run build ('dotnet', 'vs', or unspecified)." @@ -134,7 +136,9 @@ try { } if ($ci) { - $binaryLog = $true + if (-not $excludeCIBinarylog) { + $binaryLog = $true + } $nodeReuse = $false } diff --git a/eng/common/build.sh b/eng/common/build.sh index 36f9aa04..6d7c5a1f 100755 --- a/eng/common/build.sh +++ b/eng/common/build.sh @@ -32,6 +32,7 @@ usage() echo "Advanced settings:" echo " --projects Project or solution file(s) to build" echo " --ci Set when running on CI server" + echo " --excludeCIBinarylog Don't output binary log (short: -nobl)" echo " --prepareMachine Prepare machine for CI run, clean up processes after build" echo " --nodeReuse Sets nodereuse msbuild parameter ('true' or 'false')" echo " --warnAsError Sets warnaserror msbuild parameter ('true' or 'false')" @@ -68,6 +69,7 @@ clean=false warn_as_error=true node_reuse=true binary_log=false +exclude_ci_binary_log=false pipelines_log=false projects='' @@ -98,6 +100,9 @@ while [[ $# > 0 ]]; do -binarylog|-bl) binary_log=true ;; + -excludeCIBinarylog|-nobl) + exclude_ci_binary_log=true + ;; -pipelineslog|-pl) pipelines_log=true ;; @@ -156,8 +161,10 @@ done if [[ "$ci" == true ]]; then pipelines_log=true - binary_log=true node_reuse=false + if [[ "$exclude_ci_binary_log" == false ]]; then + binary_log=true + fi fi . "$scriptroot/tools.sh" diff --git a/eng/common/cross/armel/tizen-build-rootfs.sh b/eng/common/cross/armel/tizen-build-rootfs.sh index 87c48e78..25a0efa0 100755 --- a/eng/common/cross/armel/tizen-build-rootfs.sh +++ b/eng/common/cross/armel/tizen-build-rootfs.sh @@ -9,13 +9,6 @@ if [[ -z "$ROOTFS_DIR" ]]; then exit 1; fi -# Clean-up (TODO-Cleanup: We may already delete $ROOTFS_DIR at ./cross/build-rootfs.sh.) -# hk0110 -if [ -d "$ROOTFS_DIR" ]; then - umount $ROOTFS_DIR/* - rm -rf $ROOTFS_DIR -fi - TIZEN_TMP_DIR=$ROOTFS_DIR/tizen_tmp mkdir -p $TIZEN_TMP_DIR diff --git a/eng/common/cross/build-rootfs.sh b/eng/common/cross/build-rootfs.sh index d780fefb..ffdff385 100755 --- a/eng/common/cross/build-rootfs.sh +++ b/eng/common/cross/build-rootfs.sh @@ -8,8 +8,10 @@ usage() echo "BuildArch can be: arm(default), armel, arm64, x86" echo "CodeName - optional, Code name for Linux, can be: trusty, xenial(default), zesty, bionic, alpine. If BuildArch is armel, LinuxCodeName is jessie(default) or tizen." echo " for FreeBSD can be: freebsd11 or freebsd12." + echo " for illumos can be: illumos." echo "lldbx.y - optional, LLDB version, can be: lldb3.9(default), lldb4.0, lldb5.0, lldb6.0 no-lldb. Ignored for alpine and FReeBSD" echo "--skipunmount - optional, will skip the unmount of rootfs folder." + echo "--use-mirror - optional, use mirror URL to fetch resources, when available." exit 1 } @@ -67,6 +69,13 @@ __FreeBSDPackages+=" libinotify" __FreeBSDPackages+=" lttng-ust" __FreeBSDPackages+=" krb5" +__IllumosPackages="icu-64.2nb2" +__IllumosPackages+=" mit-krb5-1.16.2nb4" +__IllumosPackages+=" openssl-1.1.1e" +__IllumosPackages+=" zlib-1.2.11" + +__UseMirror=0 + __UnprocessedBuildArgs= while :; do if [ $# -le 0 ]; then @@ -158,8 +167,8 @@ while :; do __LLDB_Package="liblldb-6.0-dev" ;; tizen) - if [ "$__BuildArch" != "armel" ]; then - echo "Tizen is available only for armel." + if [ "$__BuildArch" != "armel" ] && [ "$__BuildArch" != "arm64" ]; then + echo "Tizen is available only for armel and arm64." usage; exit 1; fi @@ -179,6 +188,11 @@ while :; do __BuildArch=x64 __SkipUnmount=1 ;; + illumos) + __CodeName=illumos + __BuildArch=x64 + __SkipUnmount=1 + ;; --skipunmount) __SkipUnmount=1 ;; @@ -186,6 +200,9 @@ while :; do shift __RootfsDir=$1 ;; + --use-mirror) + __UseMirror=1 + ;; *) __UnprocessedBuildArgs="$__UnprocessedBuildArgs $1" ;; @@ -214,6 +231,9 @@ if [ -d "$__RootfsDir" ]; then rm -rf $__RootfsDir fi +mkdir -p $__RootfsDir +__RootfsDir="$( cd "$__RootfsDir" && pwd )" + if [[ "$__CodeName" == "alpine" ]]; then __ApkToolsVersion=2.9.1 __AlpineVersion=3.9 @@ -257,6 +277,51 @@ elif [[ "$__CodeName" == "freebsd" ]]; then # install packages we need. INSTALL_AS_USER=$(whoami) $__RootfsDir/host/sbin/pkg -r $__RootfsDir -C $__RootfsDir/usr/local/etc/pkg.conf update INSTALL_AS_USER=$(whoami) $__RootfsDir/host/sbin/pkg -r $__RootfsDir -C $__RootfsDir/usr/local/etc/pkg.conf install --yes $__FreeBSDPackages +elif [[ "$__CodeName" == "illumos" ]]; then + mkdir "$__RootfsDir/tmp" + pushd "$__RootfsDir/tmp" + JOBS="$(getconf _NPROCESSORS_ONLN)" + echo "Downloading sysroot." + wget -O - https://github.com/illumos/sysroot/releases/download/20181213-de6af22ae73b-v1/illumos-sysroot-i386-20181213-de6af22ae73b-v1.tar.gz | tar -C "$__RootfsDir" -xzf - + echo "Building binutils. Please wait.." + wget -O - https://ftp.gnu.org/gnu/binutils/binutils-2.33.1.tar.bz2 | tar -xjf - + mkdir build-binutils && cd build-binutils + ../binutils-2.33.1/configure --prefix="$__RootfsDir" --target="x86_64-sun-solaris2.10" --program-prefix="x86_64-illumos-" --with-sysroot="$__RootfsDir" + make -j "$JOBS" && make install && cd .. + echo "Building gcc. Please wait.." + wget -O - https://ftp.gnu.org/gnu/gcc/gcc-8.4.0/gcc-8.4.0.tar.xz | tar -xJf - + CFLAGS="-fPIC" + CXXFLAGS="-fPIC" + CXXFLAGS_FOR_TARGET="-fPIC" + CFLAGS_FOR_TARGET="-fPIC" + export CFLAGS CXXFLAGS CXXFLAGS_FOR_TARGET CFLAGS_FOR_TARGET + mkdir build-gcc && cd build-gcc + ../gcc-8.4.0/configure --prefix="$__RootfsDir" --target="x86_64-sun-solaris2.10" --program-prefix="x86_64-illumos-" --with-sysroot="$__RootfsDir" --with-gnu-as \ + --with-gnu-ld --disable-nls --disable-libgomp --disable-libquadmath --disable-libssp --disable-libvtv --disable-libcilkrts --disable-libada --disable-libsanitizer \ + --disable-libquadmath-support --disable-shared --enable-tls + make -j "$JOBS" && make install && cd .. + BaseUrl=https://pkgsrc.joyent.com + if [[ "$__UseMirror" == 1 ]]; then + BaseUrl=http://pkgsrc.smartos.skylime.net + fi + BaseUrl="$BaseUrl"/packages/SmartOS/2020Q1/x86_64/All + echo "Downloading dependencies." + read -ra array <<<"$__IllumosPackages" + for package in "${array[@]}"; do + echo "Installing $package..." + wget "$BaseUrl"/"$package".tgz + ar -x "$package".tgz + tar --skip-old-files -xzf "$package".tmp.tgz -C "$__RootfsDir" 2>/dev/null + done + echo "Cleaning up temporary files." + popd + rm -rf "$__RootfsDir"/{tmp,+*} + mkdir -p "$__RootfsDir"/usr/include/net + mkdir -p "$__RootfsDir"/usr/include/netpacket + wget -P "$__RootfsDir"/usr/include/net https://raw.githubusercontent.com/illumos/illumos-gate/master/usr/src/uts/common/io/bpf/net/bpf.h + wget -P "$__RootfsDir"/usr/include/net https://raw.githubusercontent.com/illumos/illumos-gate/master/usr/src/uts/common/io/bpf/net/dlt.h + wget -P "$__RootfsDir"/usr/include/netpacket https://raw.githubusercontent.com/illumos/illumos-gate/master/usr/src/uts/common/inet/sockmods/netpacket/packet.h + wget -P "$__RootfsDir"/usr/include/sys https://raw.githubusercontent.com/illumos/illumos-gate/master/usr/src/uts/common/sys/sdt.h elif [[ -n $__CodeName ]]; then qemu-debootstrap --arch $__UbuntuArch $__CodeName $__RootfsDir $__UbuntuRepo cp $__CrossDir/$__BuildArch/sources.list.$__CodeName $__RootfsDir/etc/apt/sources.list @@ -275,7 +340,7 @@ elif [[ -n $__CodeName ]]; then patch -p1 < $__CrossDir/$__BuildArch/trusty-lttng-2.4.patch popd fi -elif [ "$__Tizen" == "tizen" ]; then +elif [[ "$__Tizen" == "tizen" ]]; then ROOTFS_DIR=$__RootfsDir $__CrossDir/$__BuildArch/tizen-build-rootfs.sh else echo "Unsupported target platform." diff --git a/eng/common/cross/toolchain.cmake b/eng/common/cross/toolchain.cmake index 534f1d19..88a758af 100644 --- a/eng/common/cross/toolchain.cmake +++ b/eng/common/cross/toolchain.cmake @@ -3,6 +3,9 @@ set(CROSS_ROOTFS $ENV{ROOTFS_DIR}) set(TARGET_ARCH_NAME $ENV{TARGET_BUILD_ARCH}) if(EXISTS ${CROSS_ROOTFS}/bin/freebsd-version) set(CMAKE_SYSTEM_NAME FreeBSD) +elseif(EXISTS ${CROSS_ROOTFS}/usr/platform/i86pc) + set(CMAKE_SYSTEM_NAME SunOS) + set(ILLUMOS 1) else() set(CMAKE_SYSTEM_NAME Linux) endif() @@ -28,12 +31,18 @@ elseif(TARGET_ARCH_NAME STREQUAL "arm64") else() set(TOOLCHAIN "aarch64-linux-gnu") endif() + if("$ENV{__DistroRid}" MATCHES "tizen.*") + set(TIZEN_TOOLCHAIN "aarch64-tizen-linux-gnu/9.2.0") + endif() elseif(TARGET_ARCH_NAME STREQUAL "x86") set(CMAKE_SYSTEM_PROCESSOR i686) set(TOOLCHAIN "i686-linux-gnu") elseif (CMAKE_SYSTEM_NAME STREQUAL "FreeBSD") set(CMAKE_SYSTEM_PROCESSOR "x86_64") set(triple "x86_64-unknown-freebsd11") +elseif (ILLUMOS) + set(CMAKE_SYSTEM_PROCESSOR "x86_64") + set(TOOLCHAIN "x86_64-illumos") else() message(FATAL_ERROR "Arch is ${TARGET_ARCH_NAME}. Only armel, arm, arm64 and x86 are supported!") endif() @@ -43,11 +52,15 @@ if(DEFINED ENV{TOOLCHAIN}) endif() # Specify include paths -if(TARGET_ARCH_NAME STREQUAL "armel") - if(DEFINED TIZEN_TOOLCHAIN) +if(DEFINED TIZEN_TOOLCHAIN) + if(TARGET_ARCH_NAME STREQUAL "armel") include_directories(SYSTEM ${CROSS_ROOTFS}/usr/lib/gcc/${TIZEN_TOOLCHAIN}/include/c++/) include_directories(SYSTEM ${CROSS_ROOTFS}/usr/lib/gcc/${TIZEN_TOOLCHAIN}/include/c++/armv7l-tizen-linux-gnueabi) endif() + if(TARGET_ARCH_NAME STREQUAL "arm64") + include_directories(SYSTEM ${CROSS_ROOTFS}/usr/lib64/gcc/${TIZEN_TOOLCHAIN}/include/c++/) + include_directories(SYSTEM ${CROSS_ROOTFS}/usr/lib64/gcc/${TIZEN_TOOLCHAIN}/include/c++/aarch64-tizen-linux-gnu) + endif() endif() if("$ENV{__DistroRid}" MATCHES "android.*") @@ -67,12 +80,43 @@ if("$ENV{__DistroRid}" MATCHES "android.*") # include official NDK toolchain script include(${CROSS_ROOTFS}/../build/cmake/android.toolchain.cmake) -elseif (CMAKE_SYSTEM_NAME STREQUAL "FreeBSD") +elseif(CMAKE_SYSTEM_NAME STREQUAL "FreeBSD") # we cross-compile by instructing clang set(CMAKE_C_COMPILER_TARGET ${triple}) set(CMAKE_CXX_COMPILER_TARGET ${triple}) set(CMAKE_ASM_COMPILER_TARGET ${triple}) set(CMAKE_SYSROOT "${CROSS_ROOTFS}") +elseif(ILLUMOS) + set(CMAKE_SYSROOT "${CROSS_ROOTFS}") + + include_directories(SYSTEM ${CROSS_ROOTFS}/include) + + set(TOOLSET_PREFIX ${TOOLCHAIN}-) + function(locate_toolchain_exec exec var) + string(TOUPPER ${exec} EXEC_UPPERCASE) + if(NOT "$ENV{CLR_${EXEC_UPPERCASE}}" STREQUAL "") + set(${var} "$ENV{CLR_${EXEC_UPPERCASE}}" PARENT_SCOPE) + return() + endif() + + find_program(EXEC_LOCATION_${exec} + NAMES + "${TOOLSET_PREFIX}${exec}${CLR_CMAKE_COMPILER_FILE_NAME_VERSION}" + "${TOOLSET_PREFIX}${exec}") + + if (EXEC_LOCATION_${exec} STREQUAL "EXEC_LOCATION_${exec}-NOTFOUND") + message(FATAL_ERROR "Unable to find toolchain executable. Name: ${exec}, Prefix: ${TOOLSET_PREFIX}.") + endif() + set(${var} ${EXEC_LOCATION_${exec}} PARENT_SCOPE) + endfunction() + + set(CMAKE_SYSTEM_PREFIX_PATH "${CROSS_ROOTFS}") + + locate_toolchain_exec(gcc CMAKE_C_COMPILER) + locate_toolchain_exec(g++ CMAKE_CXX_COMPILER) + + set(CMAKE_C_STANDARD_LIBRARIES "${CMAKE_C_STANDARD_LIBRARIES} -lssp") + set(CMAKE_CXX_STANDARD_LIBRARIES "${CMAKE_CXX_STANDARD_LIBRARIES} -lssp") else() set(CMAKE_SYSROOT "${CROSS_ROOTFS}") @@ -90,13 +134,27 @@ if(TARGET_ARCH_NAME STREQUAL "armel") add_link_options("-L${CROSS_ROOTFS}/usr/lib") add_link_options("-L${CROSS_ROOTFS}/usr/lib/gcc/${TIZEN_TOOLCHAIN}") endif() +elseif(TARGET_ARCH_NAME STREQUAL "arm64") + if(DEFINED TIZEN_TOOLCHAIN) # For Tizen only + add_link_options("-B${CROSS_ROOTFS}/usr/lib64/gcc/${TIZEN_TOOLCHAIN}") + add_link_options("-L${CROSS_ROOTFS}/lib64") + add_link_options("-L${CROSS_ROOTFS}/usr/lib64") + add_link_options("-L${CROSS_ROOTFS}/usr/lib64/gcc/${TIZEN_TOOLCHAIN}") + + add_link_options("-Wl,--rpath-link=${CROSS_ROOTFS}/lib64") + add_link_options("-Wl,--rpath-link=${CROSS_ROOTFS}/usr/lib64") + add_link_options("-Wl,--rpath-link=${CROSS_ROOTFS}/usr/lib64/gcc/${TIZEN_TOOLCHAIN}") + endif() elseif(TARGET_ARCH_NAME STREQUAL "x86") add_link_options(-m32) +elseif(ILLUMOS) + add_link_options("-L${CROSS_ROOTFS}/lib/amd64") + add_link_options("-L${CROSS_ROOTFS}/usr/amd64/lib") endif() # Specify compile options -if(TARGET_ARCH_NAME MATCHES "^(arm|armel|arm64)$" AND NOT "$ENV{__DistroRid}" MATCHES "android.*") +if((TARGET_ARCH_NAME MATCHES "^(arm|armel|arm64)$" AND NOT "$ENV{__DistroRid}" MATCHES "android.*") OR ILLUMOS) set(CMAKE_C_COMPILER_TARGET ${TOOLCHAIN}) set(CMAKE_CXX_COMPILER_TARGET ${TOOLCHAIN}) set(CMAKE_ASM_COMPILER_TARGET ${TOOLCHAIN}) @@ -117,16 +175,19 @@ if(TARGET_ARCH_NAME MATCHES "^(arm|armel)$") if(TARGET_ARCH_NAME STREQUAL "armel") add_compile_options(-mfloat-abi=softfp) - if(DEFINED TIZEN_TOOLCHAIN) - add_compile_options(-Wno-deprecated-declarations) # compile-time option - add_compile_options(-D__extern_always_inline=inline) # compile-time option - endif() endif() elseif(TARGET_ARCH_NAME STREQUAL "x86") add_compile_options(-m32) add_compile_options(-Wno-error=unused-command-line-argument) endif() +if(DEFINED TIZEN_TOOLCHAIN) + if(TARGET_ARCH_NAME MATCHES "^(armel|arm64)$") + add_compile_options(-Wno-deprecated-declarations) # compile-time option + add_compile_options(-D__extern_always_inline=inline) # compile-time option + endif() +endif() + # Set LLDB include and library paths for builds that need lldb. if(TARGET_ARCH_NAME MATCHES "^(arm|armel|x86)$") if(TARGET_ARCH_NAME STREQUAL "x86") @@ -155,7 +216,6 @@ if(TARGET_ARCH_NAME MATCHES "^(arm|armel|x86)$") endif() endif() - set(CMAKE_FIND_ROOT_PATH_MODE_PROGRAM NEVER) set(CMAKE_FIND_ROOT_PATH_MODE_LIBRARY ONLY) set(CMAKE_FIND_ROOT_PATH_MODE_INCLUDE ONLY) diff --git a/eng/common/internal-feed-operations.ps1 b/eng/common/internal-feed-operations.ps1 index db0baac9..b8f6529f 100644 --- a/eng/common/internal-feed-operations.ps1 +++ b/eng/common/internal-feed-operations.ps1 @@ -63,6 +63,7 @@ function SetupCredProvider { } if (($endpoints | Measure-Object).Count -gt 0) { + # [SuppressMessage("Microsoft.Security", "CS002:SecretInNextLine", Justification="Endpoint code example with no real credentials.")] # Create the JSON object. It should look like '{"endpointCredentials": [{"endpoint":"http://example.index.json", "username":"optional", "password":"accesstoken"}]}' $endpointCredentials = @{endpointCredentials=$endpoints} | ConvertTo-Json -Compress diff --git a/eng/common/internal-feed-operations.sh b/eng/common/internal-feed-operations.sh index 5941ea28..9ed225e7 100755 --- a/eng/common/internal-feed-operations.sh +++ b/eng/common/internal-feed-operations.sh @@ -62,6 +62,7 @@ function SetupCredProvider { endpoints+=']' if [ ${#endpoints} -gt 2 ]; then + # [SuppressMessage("Microsoft.Security", "CS002:SecretInNextLine", Justification="Endpoint code example with no real credentials.")] # Create the JSON object. It should look like '{"endpointCredentials": [{"endpoint":"http://example.index.json", "username":"optional", "password":"accesstoken"}]}' local endpointCredentials="{\"endpointCredentials\": "$endpoints"}" diff --git a/eng/common/internal/Tools.csproj b/eng/common/internal/Tools.csproj index 1a39a7ef..f46d5efe 100644 --- a/eng/common/internal/Tools.csproj +++ b/eng/common/internal/Tools.csproj @@ -4,6 +4,7 @@ net472 false + false diff --git a/eng/common/native/CommonLibrary.psm1 b/eng/common/native/CommonLibrary.psm1 index 41416862..d7d1a651 100644 --- a/eng/common/native/CommonLibrary.psm1 +++ b/eng/common/native/CommonLibrary.psm1 @@ -145,9 +145,12 @@ function Get-File { New-Item -path $DownloadDirectory -force -itemType "Directory" | Out-Null } + $TempPath = "$Path.tmp" if (Test-Path -IsValid -Path $Uri) { - Write-Verbose "'$Uri' is a file path, copying file to '$Path'" - Copy-Item -Path $Uri -Destination $Path + Write-Verbose "'$Uri' is a file path, copying temporarily to '$TempPath'" + Copy-Item -Path $Uri -Destination $TempPath + Write-Verbose "Moving temporary file to '$Path'" + Move-Item -Path $TempPath -Destination $Path return $? } else { @@ -157,8 +160,10 @@ function Get-File { while($Attempt -Lt $DownloadRetries) { try { - Invoke-WebRequest -UseBasicParsing -Uri $Uri -OutFile $Path - Write-Verbose "Downloaded to '$Path'" + Invoke-WebRequest -UseBasicParsing -Uri $Uri -OutFile $TempPath + Write-Verbose "Downloaded to temporary location '$TempPath'" + Move-Item -Path $TempPath -Destination $Path + Write-Verbose "Moved temporary file to '$Path'" return $True } catch { @@ -359,16 +364,21 @@ function Expand-Zip { return $False } } - if (-Not (Test-Path $OutputDirectory)) { - New-Item -path $OutputDirectory -Force -itemType "Directory" | Out-Null + + $TempOutputDirectory = Join-Path "$(Split-Path -Parent $OutputDirectory)" "$(Split-Path -Leaf $OutputDirectory).tmp" + if (Test-Path $TempOutputDirectory) { + Remove-Item $TempOutputDirectory -Force -Recurse } + New-Item -Path $TempOutputDirectory -Force -ItemType "Directory" | Out-Null Add-Type -assembly "system.io.compression.filesystem" - [io.compression.zipfile]::ExtractToDirectory("$ZipPath", "$OutputDirectory") + [io.compression.zipfile]::ExtractToDirectory("$ZipPath", "$TempOutputDirectory") if ($? -Eq $False) { Write-Error "Unable to extract '$ZipPath'" return $False } + + Move-Item -Path $TempOutputDirectory -Destination $OutputDirectory } catch { Write-Host $_ diff --git a/eng/common/performance/perfhelixpublish.proj b/eng/common/performance/perfhelixpublish.proj index cf5941e1..272366da 100644 --- a/eng/common/performance/perfhelixpublish.proj +++ b/eng/common/performance/perfhelixpublish.proj @@ -6,6 +6,7 @@ py -3 %HELIX_CORRELATION_PAYLOAD%\Core_Root\CoreRun.exe %HELIX_CORRELATION_PAYLOAD%\Baseline_Core_Root\CoreRun.exe + $(HelixPreCommands);call %HELIX_CORRELATION_PAYLOAD%\performance\tools\machine-setup.cmd;set PYTHONPATH=%HELIX_WORKITEM_PAYLOAD%\scripts%3B%HELIX_WORKITEM_PAYLOAD% %HELIX_CORRELATION_PAYLOAD%\artifacts\BenchmarkDotNet.Artifacts %HELIX_CORRELATION_PAYLOAD%\artifacts\BenchmarkDotNet.Artifacts_Baseline @@ -40,6 +41,13 @@ $HELIX_WORKITEM_ROOT/testResults.xml + + --corerun %HELIX_CORRELATION_PAYLOAD%\dotnet-mono\shared\Microsoft.NETCore.App\5.0.0\corerun.exe + + + --corerun $(BaseDirectory)/dotnet-mono/shared/Microsoft.NETCore.App/5.0.0/corerun + + --corerun $(CoreRun) @@ -55,6 +63,11 @@ $(WorkItemCommand) $(CliArguments) + + + 2:30 + 0:15 + @@ -63,7 +76,7 @@ - 5 + 30 @@ -71,6 +84,31 @@ + + + + + + + + + + + + + + + + + + + + + + + + + @@ -86,7 +124,7 @@ $(WorkItemCommand) --bdn-artifacts $(BaselineArtifactsDirectory) --bdn-arguments="--anyCategories $(BDNCategories) $(ExtraBenchmarkDotNetArguments) $(BaselineCoreRunArgument) --partition-count $(PartitionCount) --partition-index %(HelixWorkItem.Index)" $(WorkItemCommand) --bdn-artifacts $(ArtifactsDirectory) --bdn-arguments="--anyCategories $(BDNCategories) $(ExtraBenchmarkDotNetArguments) $(CoreRunArgument) --partition-count $(PartitionCount) --partition-index %(HelixWorkItem.Index)" $(DotnetExe) run -f $(_Framework) -p $(ResultsComparer) --base $(BaselineArtifactsDirectory) --diff $(ArtifactsDirectory) --threshold 2$(Percent) --xml $(XMLResults);$(FinalCommand) - 4:00 + $(WorkItemTimeout) @@ -105,17 +143,50 @@ $(WorkItemDirectory)\ScenarioCorrelation $(Python) %HELIX_CORRELATION_PAYLOAD%\performance\src\scenarios\crossgen\test.py crossgen --test-name System.Private.Xml.dll --core-root %HELIX_CORRELATION_PAYLOAD%\Core_Root - + $(WorkItemDirectory)\ScenarioCorrelation $(Python) %HELIX_CORRELATION_PAYLOAD%\performance\src\scenarios\crossgen\test.py crossgen --test-name System.Linq.Expressions.dll --core-root %HELIX_CORRELATION_PAYLOAD%\Core_Root - + $(WorkItemDirectory)\ScenarioCorrelation $(Python) %HELIX_CORRELATION_PAYLOAD%\performance\src\scenarios\crossgen\test.py crossgen --test-name Microsoft.CodeAnalysis.VisualBasic.dll --core-root %HELIX_CORRELATION_PAYLOAD%\Core_Root - + $(WorkItemDirectory)\ScenarioCorrelation $(Python) %HELIX_CORRELATION_PAYLOAD%\performance\src\scenarios\crossgen\test.py crossgen --test-name Microsoft.CodeAnalysis.CSharp.dll --core-root %HELIX_CORRELATION_PAYLOAD%\Core_Root + + $(WorkItemDirectory)\ScenarioCorrelation + $(Python) %HELIX_CORRELATION_PAYLOAD%\performance\src\scenarios\crossgen\test.py crossgen --test-name System.Private.CoreLib.dll --core-root %HELIX_CORRELATION_PAYLOAD%\Core_Root + + + + + + $(WorkItemDirectory)\ScenarioCorrelation + $(Python) %HELIX_CORRELATION_PAYLOAD%\performance\src\scenarios\crossgen2\test.py crossgen2 --single System.Private.Xml.dll --core-root %HELIX_CORRELATION_PAYLOAD%\Core_Root + + + $(WorkItemDirectory)\ScenarioCorrelation + $(Python) %HELIX_CORRELATION_PAYLOAD%\performance\src\scenarios\crossgen2\test.py crossgen2 --single System.Linq.Expressions.dll --core-root %HELIX_CORRELATION_PAYLOAD%\Core_Root + + + $(WorkItemDirectory)\ScenarioCorrelation + $(Python) %HELIX_CORRELATION_PAYLOAD%\performance\src\scenarios\crossgen2\test.py crossgen2 --single Microsoft.CodeAnalysis.VisualBasic.dll --core-root %HELIX_CORRELATION_PAYLOAD%\Core_Root + + + $(WorkItemDirectory)\ScenarioCorrelation + $(Python) %HELIX_CORRELATION_PAYLOAD%\performance\src\scenarios\crossgen2\test.py crossgen2 --single Microsoft.CodeAnalysis.CSharp.dll --core-root %HELIX_CORRELATION_PAYLOAD%\Core_Root + + + $(WorkItemDirectory)\ScenarioCorrelation + $(Python) %HELIX_CORRELATION_PAYLOAD%\performance\src\scenarios\crossgen2\test.py crossgen2 --single System.Private.CoreLib.dll --core-root %HELIX_CORRELATION_PAYLOAD%\Core_Root + + + $(WorkItemDirectory)\ScenarioCorrelation + $(Python) %HELIX_CORRELATION_PAYLOAD%\performance\src\scenarios\crossgen2\test.py crossgen2 --composite %HELIX_CORRELATION_PAYLOAD%\performance\src\scenarios\crossgen2\framework-r2r.dll.rsp --core-root %HELIX_CORRELATION_PAYLOAD%\Core_Root + 1:00 + + \ No newline at end of file diff --git a/eng/common/performance/performance-setup.ps1 b/eng/common/performance/performance-setup.ps1 index 1763a1a9..31a99e49 100644 --- a/eng/common/performance/performance-setup.ps1 +++ b/eng/common/performance/performance-setup.ps1 @@ -3,7 +3,7 @@ Param( [string] $CoreRootDirectory, [string] $BaselineCoreRootDirectory, [string] $Architecture="x64", - [string] $Framework="netcoreapp5.0", + [string] $Framework="net5.0", [string] $CompilationMode="Tiered", [string] $Repository=$env:BUILD_REPOSITORY_NAME, [string] $Branch=$env:BUILD_SOURCEBRANCH, @@ -12,8 +12,12 @@ Param( [string] $RunCategories="Libraries Runtime", [string] $Csproj="src\benchmarks\micro\MicroBenchmarks.csproj", [string] $Kind="micro", + [switch] $LLVM, + [switch] $MonoInterpreter, + [switch] $MonoAOT, [switch] $Internal, [switch] $Compare, + [string] $MonoDotnet="", [string] $Configurations="CompilationMode=$CompilationMode RunKind=$Kind" ) @@ -31,7 +35,8 @@ $HelixSourcePrefix = "pr" $Queue = "Windows.10.Amd64.ClientRS4.DevEx.15.8.Open" -if ($Framework.StartsWith("netcoreapp")) { +# TODO: Implement a better logic to determine if Framework is .NET Core or >= .NET 5. +if ($Framework.StartsWith("netcoreapp") -or ($Framework -eq "net5.0")) { $Queue = "Windows.10.Amd64.ClientRS5.Open" } @@ -49,6 +54,21 @@ if ($Internal) { $HelixSourcePrefix = "official" } +if($MonoDotnet -ne "") +{ + $Configurations += " LLVM=$LLVM MonoInterpreter=$MonoInterpreter MonoAOT=$MonoAOT" + if($ExtraBenchmarkDotNetArguments -eq "") + { + #FIX ME: We need to block these tests as they don't run on mono for now + $ExtraBenchmarkDotNetArguments = "--exclusion-filter *Perf_Image* *Perf_NamedPipeStream*" + } + else + { + #FIX ME: We need to block these tests as they don't run on mono for now + $ExtraBenchmarkDotNetArguments += " --exclusion-filter *Perf_Image* *Perf_NamedPipeStream*" + } +} + # FIX ME: This is a workaround until we get this from the actual pipeline $CommonSetupArguments="--channel master --queue $Queue --build-number $BuildNumber --build-configs $Configurations --architecture $Architecture" $SetupArguments = "--repository https://github.com/$Repository --branch $Branch --get-perf-hash --commit-sha $CommitSha $CommonSetupArguments" @@ -69,6 +89,13 @@ else { git clone --branch master --depth 1 --quiet https://github.com/dotnet/performance $PerformanceDirectory } +if($MonoDotnet -ne "") +{ + $UsingMono = "true" + $MonoDotnetPath = (Join-Path $PayloadDirectory "dotnet-mono") + Move-Item -Path $MonoDotnet -Destination $MonoDotnetPath +} + if ($UseCoreRun) { $NewCoreRoot = (Join-Path $PayloadDirectory "Core_Root") Move-Item -Path $CoreRootDirectory -Destination $NewCoreRoot @@ -104,6 +131,7 @@ Write-PipelineSetVariable -Name 'UseCoreRun' -Value "$UseCoreRun" -IsMultiJobVar Write-PipelineSetVariable -Name 'UseBaselineCoreRun' -Value "$UseBaselineCoreRun" -IsMultiJobVariable $false Write-PipelineSetVariable -Name 'RunFromPerfRepo' -Value "$RunFromPerformanceRepo" -IsMultiJobVariable $false Write-PipelineSetVariable -Name 'Compare' -Value "$Compare" -IsMultiJobVariable $false +Write-PipelineSetVariable -Name 'MonoDotnet' -Value "$UsingMono" -IsMultiJobVariable $false # Helix Arguments Write-PipelineSetVariable -Name 'Creator' -Value "$Creator" -IsMultiJobVariable $false diff --git a/eng/common/performance/performance-setup.sh b/eng/common/performance/performance-setup.sh index b9eecf94..9409e4d8 100755 --- a/eng/common/performance/performance-setup.sh +++ b/eng/common/performance/performance-setup.sh @@ -4,7 +4,7 @@ source_directory=$BUILD_SOURCESDIRECTORY core_root_directory= baseline_core_root_directory= architecture=x64 -framework=netcoreapp5.0 +framework=net5.0 compilation_mode=tiered repository=$BUILD_REPOSITORY_NAME branch=$BUILD_SOURCEBRANCH @@ -12,13 +12,18 @@ commit_sha=$BUILD_SOURCEVERSION build_number=$BUILD_BUILDNUMBER internal=false compare=false +mono_dotnet= kind="micro" +llvm=false +monointerpreter=false +monoaot=false run_categories="Libraries Runtime" csproj="src\benchmarks\micro\MicroBenchmarks.csproj" configurations="CompliationMode=$compilation_mode RunKind=$kind" run_from_perf_repo=false use_core_run=true use_baseline_core_run=true +using_mono=false while (($# > 0)); do lowerI="$(echo $1 | awk '{print tolower($0)}')" @@ -65,6 +70,7 @@ while (($# > 0)); do ;; --kind) kind=$2 + configurations="CompliationMode=$compilation_mode RunKind=$kind" shift 2 ;; --runcategories) @@ -79,6 +85,22 @@ while (($# > 0)); do internal=true shift 1 ;; + --llvm) + llvm=true + shift 1 + ;; + --monointerpreter) + monointerpreter=true + shift 1 + ;; + --monoaot) + monoaot=true + shift 1 + ;; + --monodotnet) + mono_dotnet=$2 + shift 2 + ;; --compare) compare=true shift 1 @@ -107,6 +129,7 @@ while (($# > 0)); do echo " --kind Related to csproj. The kind of benchmarks that should be run. Defaults to micro" echo " --runcategories Related to csproj. Categories of benchmarks to run. Defaults to \"coreclr corefx\"" echo " --internal If the benchmarks are running as an official job." + echo " --monodotnet Pass the path to the mono dotnet for mono performance testing." echo "" exit 0 ;; @@ -164,6 +187,10 @@ if [[ "$internal" == true ]]; then fi fi +if [[ "$mono_dotnet" != "" ]]; then + configurations="$configurations LLVM=$llvm MonoInterpreter=$monointerpreter MonoAOT=$monoaot" +fi + common_setup_arguments="--channel master --queue $queue --build-number $build_number --build-configs $configurations --architecture $architecture" setup_arguments="--repository https://github.com/$repository --branch $branch --get-perf-hash --commit-sha $commit_sha $common_setup_arguments" @@ -186,6 +213,12 @@ else mv $docs_directory $workitem_directory fi +if [[ "$mono_dotnet" != "" ]]; then + using_mono=true + mono_dotnet_path=$payload_directory/dotnet-mono + mv $mono_dotnet $mono_dotnet_path +fi + if [[ "$use_core_run" = true ]]; then new_core_root=$payload_directory/Core_Root mv $core_root_directory $new_core_root @@ -221,3 +254,4 @@ Write-PipelineSetVariable -name "HelixSourcePrefix" -value "$helix_source_prefix Write-PipelineSetVariable -name "Kind" -value "$kind" -is_multi_job_variable false Write-PipelineSetVariable -name "_BuildConfig" -value "$architecture.$kind.$framework" -is_multi_job_variable false Write-PipelineSetVariable -name "Compare" -value "$compare" -is_multi_job_variable false +Write-PipelineSetVariable -name "MonoDotnet" -value "$using_mono" -is_multi_job_variable false diff --git a/eng/common/pipeline-logging-functions.sh b/eng/common/pipeline-logging-functions.sh index 33c3f0d8..6cd0a340 100755 --- a/eng/common/pipeline-logging-functions.sh +++ b/eng/common/pipeline-logging-functions.sh @@ -31,26 +31,21 @@ function Write-PipelineTelemetryError { return fi - message="(NETCORE_ENGINEERING_TELEMETRY=$telemetry_category) $message" - function_args+=("$message") if [[ $force == true ]]; then function_args+=("-force") fi - - Write-PipelineTaskError $function_args + message="(NETCORE_ENGINEERING_TELEMETRY=$telemetry_category) $message" + function_args+=("$message") + Write-PipelineTaskError ${function_args[@]} } function Write-PipelineTaskError { - if [[ $force != true ]] && [[ "$ci" != true ]]; then - echo "$@" >&2 - return - fi - local message_type="error" local sourcepath='' local linenumber='' local columnnumber='' local error_code='' + local force=false while [[ $# -gt 0 ]]; do opt="$(echo "${1/#--/-}" | awk '{print tolower($0)}')" @@ -75,6 +70,9 @@ function Write-PipelineTaskError { error_code=$2 shift ;; + -force|-f) + force=true + ;; *) break ;; @@ -83,6 +81,11 @@ function Write-PipelineTaskError { shift done + if [[ $force != true ]] && [[ "$ci" != true ]]; then + echo "$@" >&2 + return + fi + local message="##vso[task.logissue" message="$message type=$message_type" diff --git a/eng/common/post-build/check-channel-consistency.ps1 b/eng/common/post-build/check-channel-consistency.ps1 index 38abc539..63f3464c 100644 --- a/eng/common/post-build/check-channel-consistency.ps1 +++ b/eng/common/post-build/check-channel-consistency.ps1 @@ -15,12 +15,22 @@ try { # is available in YAML $PromoteToChannelsIds = $PromoteToChannels -split "\D" | Where-Object { $_ } + $hasErrors = $false + foreach ($id in $PromoteToChannelsIds) { if (($id -ne 0) -and ($id -notin $AvailableChannelIds)) { Write-PipelineTaskError -Message "Channel $id is not present in the post-build YAML configuration! This is an error scenario. Please contact @dnceng." + $hasErrors = $true } } + # The `Write-PipelineTaskError` doesn't error the script and we might report several errors + # in the previous lines. The check below makes sure that we return an error state from the + # script if we reported any validation error + if ($hasErrors) { + ExitWithExitCode 1 + } + Write-Host 'done.' } catch { diff --git a/eng/common/post-build/publish-using-darc.ps1 b/eng/common/post-build/publish-using-darc.ps1 new file mode 100644 index 00000000..d22eb439 --- /dev/null +++ b/eng/common/post-build/publish-using-darc.ps1 @@ -0,0 +1,71 @@ +param( + [Parameter(Mandatory=$true)][int] $BuildId, + [Parameter(Mandatory=$true)][string] $AzdoToken, + [Parameter(Mandatory=$true)][string] $MaestroToken, + [Parameter(Mandatory=$false)][string] $MaestroApiEndPoint = 'https://maestro-prod.westus2.cloudapp.azure.com', + [Parameter(Mandatory=$true)][string] $WaitPublishingFinish, + [Parameter(Mandatory=$true)][string] $EnableSourceLinkValidation, + [Parameter(Mandatory=$true)][string] $EnableSigningValidation, + [Parameter(Mandatory=$true)][string] $EnableNugetValidation, + [Parameter(Mandatory=$true)][string] $PublishInstallersAndChecksums, + [Parameter(Mandatory=$false)][string] $ArtifactsPublishingAdditionalParameters, + [Parameter(Mandatory=$false)][string] $SigningValidationAdditionalParameters +) + +try { + . $PSScriptRoot\post-build-utils.ps1 + . $PSScriptRoot\..\darc-init.ps1 + + $optionalParams = [System.Collections.ArrayList]::new() + + if ("" -ne $ArtifactsPublishingAdditionalParameters) { + $optionalParams.Add("artifact-publishing-parameters") | Out-Null + $optionalParams.Add($ArtifactsPublishingAdditionalParameters) | Out-Null + } + + if ("false" -eq $WaitPublishingFinish) { + $optionalParams.Add("--no-wait") | Out-Null + } + + if ("true" -eq $PublishInstallersAndChecksums) { + $optionalParams.Add("--publish-installers-and-checksums") | Out-Null + } + + if ("true" -eq $EnableNugetValidation) { + $optionalParams.Add("--validate-nuget") | Out-Null + } + + if ("true" -eq $EnableSourceLinkValidation) { + $optionalParams.Add("--validate-sourcelinkchecksums") | Out-Null + } + + if ("true" -eq $EnableSigningValidation) { + $optionalParams.Add("--validate-signingchecksums") | Out-Null + + if ("" -ne $SigningValidationAdditionalParameters) { + $optionalParams.Add("--signing-validation-parameters") | Out-Null + $optionalParams.Add($SigningValidationAdditionalParameters) | Out-Null + } + } + + & darc add-build-to-channel ` + --id $buildId ` + --default-channels ` + --source-branch master ` + --azdev-pat $AzdoToken ` + --bar-uri $MaestroApiEndPoint ` + --password $MaestroToken ` + @optionalParams + + if ($LastExitCode -ne 0) { + Write-Host "Problems using Darc to promote build ${buildId} to default channels. Stopping execution..." + exit 1 + } + + Write-Host 'done.' +} +catch { + Write-Host $_ + Write-PipelineTelemetryError -Category 'PromoteBuild' -Message "There was an error while trying to publish build '$BuildId' to default channels." + ExitWithExitCode 1 +} diff --git a/eng/common/post-build/sourcelink-validation.ps1 b/eng/common/post-build/sourcelink-validation.ps1 index cc9d059d..cdd1bba4 100644 --- a/eng/common/post-build/sourcelink-validation.ps1 +++ b/eng/common/post-build/sourcelink-validation.ps1 @@ -196,6 +196,8 @@ function ValidateSourceLinkLinks { Remove-Item $ExtractPath -Force -Recurse -ErrorAction SilentlyContinue } + $ValidationFailures = 0 + # Process each NuGet package in parallel Get-ChildItem "$InputPath\*.symbols.nupkg" | ForEach-Object { @@ -209,17 +211,20 @@ function ValidateSourceLinkLinks { } foreach ($Job in @(Get-Job -State 'Completed')) { - Receive-Job -Id $Job.Id + $jobResult = Receive-Job -Id $Job.Id + if ($jobResult -ne '0') { + $ValidationFailures++ + } Remove-Job -Id $Job.Id } } - $ValidationFailures = 0 foreach ($Job in @(Get-Job)) { $jobResult = Wait-Job -Id $Job.Id | Receive-Job if ($jobResult -ne '0') { $ValidationFailures++ } + Remove-Job -Id $Job.Id } if ($ValidationFailures -gt 0) { Write-PipelineTelemetryError -Category 'SourceLink' -Message "$ValidationFailures package(s) failed validation." diff --git a/eng/common/post-build/symbols-validation.ps1 b/eng/common/post-build/symbols-validation.ps1 index 22eeb265..495428ea 100644 --- a/eng/common/post-build/symbols-validation.ps1 +++ b/eng/common/post-build/symbols-validation.ps1 @@ -2,72 +2,29 @@ param( [Parameter(Mandatory=$true)][string] $InputPath, # Full path to directory where NuGet packages to be checked are stored [Parameter(Mandatory=$true)][string] $ExtractPath, # Full path to directory where the packages will be extracted during validation [Parameter(Mandatory=$true)][string] $DotnetSymbolVersion, # Version of dotnet symbol to use - [Parameter(Mandatory=$false)][switch] $ContinueOnError # If we should keep checking symbols after an error + [Parameter(Mandatory=$false)][switch] $ContinueOnError, # If we should keep checking symbols after an error + [Parameter(Mandatory=$false)][switch] $Clean # Clean extracted symbols directory after checking symbols ) -function FirstMatchingSymbolDescriptionOrDefault { - param( - [string] $FullPath, # Full path to the module that has to be checked - [string] $TargetServerParam, # Parameter to pass to `Symbol Tool` indicating the server to lookup for symbols - [string] $SymbolsPath - ) - - $FileName = [System.IO.Path]::GetFileName($FullPath) - $Extension = [System.IO.Path]::GetExtension($FullPath) - - # Those below are potential symbol files that the `dotnet symbol` might - # return. Which one will be returned depend on the type of file we are - # checking and which type of file was uploaded. - - # The file itself is returned - $SymbolPath = $SymbolsPath + '\' + $FileName - - # PDB file for the module - $PdbPath = $SymbolPath.Replace($Extension, '.pdb') - - # PDB file for R2R module (created by crossgen) - $NGenPdb = $SymbolPath.Replace($Extension, '.ni.pdb') - - # DBG file for a .so library - $SODbg = $SymbolPath.Replace($Extension, '.so.dbg') +# Maximum number of jobs to run in parallel +$MaxParallelJobs = 6 - # DWARF file for a .dylib - $DylibDwarf = $SymbolPath.Replace($Extension, '.dylib.dwarf') - - $dotnetSymbolExe = "$env:USERPROFILE\.dotnet\tools" - $dotnetSymbolExe = Resolve-Path "$dotnetSymbolExe\dotnet-symbol.exe" - - & $dotnetSymbolExe --symbols --modules --windows-pdbs $TargetServerParam $FullPath -o $SymbolsPath | Out-Null - - if (Test-Path $PdbPath) { - return 'PDB' - } - elseif (Test-Path $NGenPdb) { - return 'NGen PDB' - } - elseif (Test-Path $SODbg) { - return 'DBG for SO' - } - elseif (Test-Path $DylibDwarf) { - return 'Dwarf for Dylib' - } - elseif (Test-Path $SymbolPath) { - return 'Module' - } - else { - return $null - } -} +# Wait time between check for system load +$SecondsBetweenLoadChecks = 10 -function CountMissingSymbols { +$CountMissingSymbols = { param( [string] $PackagePath # Path to a NuGet package ) + . $using:PSScriptRoot\..\tools.ps1 + + Add-Type -AssemblyName System.IO.Compression.FileSystem + # Ensure input file exist if (!(Test-Path $PackagePath)) { Write-PipelineTaskError "Input file does not exist: $PackagePath" - ExitWithExitCode 1 + return -2 } # Extensions for which we'll look for symbols @@ -78,23 +35,88 @@ function CountMissingSymbols { $PackageId = [System.IO.Path]::GetFileNameWithoutExtension($PackagePath) $PackageGuid = New-Guid - $ExtractPath = Join-Path -Path $ExtractPath -ChildPath $PackageGuid + $ExtractPath = Join-Path -Path $using:ExtractPath -ChildPath $PackageGuid $SymbolsPath = Join-Path -Path $ExtractPath -ChildPath 'Symbols' - [System.IO.Compression.ZipFile]::ExtractToDirectory($PackagePath, $ExtractPath) + try { + [System.IO.Compression.ZipFile]::ExtractToDirectory($PackagePath, $ExtractPath) + } + catch { + Write-Host "Something went wrong extracting $PackagePath" + Write-Host $_ + return [pscustomobject]@{ + result = -1 + packagePath = $PackagePath + } + } Get-ChildItem -Recurse $ExtractPath | Where-Object {$RelevantExtensions -contains $_.Extension} | ForEach-Object { - if ($_.FullName -Match '\\ref\\') { - Write-Host "`t Ignoring reference assembly file " $_.FullName + $FileName = $_.FullName + if ($FileName -Match '\\ref\\') { + Write-Host "`t Ignoring reference assembly file " $FileName return } - $SymbolsOnMSDL = FirstMatchingSymbolDescriptionOrDefault $_.FullName '--microsoft-symbol-server' $SymbolsPath - $SymbolsOnSymWeb = FirstMatchingSymbolDescriptionOrDefault $_.FullName '--internal-server' $SymbolsPath + $FirstMatchingSymbolDescriptionOrDefault = { + param( + [string] $FullPath, # Full path to the module that has to be checked + [string] $TargetServerParam, # Parameter to pass to `Symbol Tool` indicating the server to lookup for symbols + [string] $SymbolsPath + ) + + $FileName = [System.IO.Path]::GetFileName($FullPath) + $Extension = [System.IO.Path]::GetExtension($FullPath) - Write-Host -NoNewLine "`t Checking file " $_.FullName "... " + # Those below are potential symbol files that the `dotnet symbol` might + # return. Which one will be returned depend on the type of file we are + # checking and which type of file was uploaded. + + # The file itself is returned + $SymbolPath = $SymbolsPath + '\' + $FileName + + # PDB file for the module + $PdbPath = $SymbolPath.Replace($Extension, '.pdb') + + # PDB file for R2R module (created by crossgen) + $NGenPdb = $SymbolPath.Replace($Extension, '.ni.pdb') + + # DBG file for a .so library + $SODbg = $SymbolPath.Replace($Extension, '.so.dbg') + + # DWARF file for a .dylib + $DylibDwarf = $SymbolPath.Replace($Extension, '.dylib.dwarf') + + $dotnetSymbolExe = "$env:USERPROFILE\.dotnet\tools" + $dotnetSymbolExe = Resolve-Path "$dotnetSymbolExe\dotnet-symbol.exe" + + & $dotnetSymbolExe --symbols --modules --windows-pdbs $TargetServerParam $FullPath -o $SymbolsPath | Out-Null + + if (Test-Path $PdbPath) { + return 'PDB' + } + elseif (Test-Path $NGenPdb) { + return 'NGen PDB' + } + elseif (Test-Path $SODbg) { + return 'DBG for SO' + } + elseif (Test-Path $DylibDwarf) { + return 'Dwarf for Dylib' + } + elseif (Test-Path $SymbolPath) { + return 'Module' + } + else { + return $null + } + } + + $SymbolsOnMSDL = & $FirstMatchingSymbolDescriptionOrDefault $FileName '--microsoft-symbol-server' $SymbolsPath + $SymbolsOnSymWeb = & $FirstMatchingSymbolDescriptionOrDefault $FileName '--internal-server' $SymbolsPath + + Write-Host -NoNewLine "`t Checking file " $FileName "... " if ($SymbolsOnMSDL -ne $null -and $SymbolsOnSymWeb -ne $null) { Write-Host "Symbols found on MSDL ($SymbolsOnMSDL) and SymWeb ($SymbolsOnSymWeb)" @@ -116,9 +138,35 @@ function CountMissingSymbols { } } + if ($using:Clean) { + Remove-Item $ExtractPath -Recurse -Force + } + + if ($MissingSymbols -ne 0) + { + Write-PipelineTelemetryError -Category 'CheckSymbols' -Message "Missing symbols for $MissingSymbols modules in the package $FileName" + } + Pop-Location - return $MissingSymbols + return [pscustomobject]@{ + result = $MissingSymbols + packagePath = $PackagePath + } +} + +function CheckJobResult( + $result, + $packagePath, + [ref]$DupedSymbols, + [ref]$TotalFailures) { + if ($result -eq '-1') { + Write-PipelineTelemetryError -Category 'CheckSymbols' -Message "$packagePath has duplicated symbol files" + $DupedSymbols.Value++ + } + elseif ($jobResult.result -ne '0') { + $TotalFailures.Value++ + } } function CheckSymbolsAvailable { @@ -127,10 +175,12 @@ function CheckSymbolsAvailable { } $TotalFailures = 0 + $DupedSymbols = 0 Get-ChildItem "$InputPath\*.nupkg" | ForEach-Object { $FileName = $_.Name + $FullName = $_.FullName # These packages from Arcade-Services include some native libraries that # our current symbol uploader can't handle. Below is a workaround until @@ -147,26 +197,45 @@ function CheckSymbolsAvailable { } Write-Host "Validating $FileName " - $Status = CountMissingSymbols "$InputPath\$FileName" - if ($Status -ne 0) { - Write-PipelineTelemetryError -Category 'CheckSymbols' -Message "Missing symbols for $Status modules in the package $FileName" + Start-Job -ScriptBlock $CountMissingSymbols -ArgumentList $FullName | Out-Null - if ($ContinueOnError) { - $TotalFailures++ - } - else { - ExitWithExitCode 1 - } + $NumJobs = @(Get-Job -State 'Running').Count + Write-Host $NumJobs + + while ($NumJobs -ge $MaxParallelJobs) { + Write-Host "There are $NumJobs validation jobs running right now. Waiting $SecondsBetweenLoadChecks seconds to check again." + sleep $SecondsBetweenLoadChecks + $NumJobs = @(Get-Job -State 'Running').Count } + foreach ($Job in @(Get-Job -State 'Completed')) { + $jobResult = Wait-Job -Id $Job.Id | Receive-Job + CheckJobResult $jobResult.result $jobResult.packagePath ([ref]$DupedSymbols) ([ref]$TotalFailures) + Remove-Job -Id $Job.Id + } Write-Host } - if ($TotalFailures -ne 0) { - Write-PipelineTelemetryError -Category 'CheckSymbols' -Message "Symbols missing for $TotalFailures packages" + foreach ($Job in @(Get-Job)) { + $jobResult = Wait-Job -Id $Job.Id | Receive-Job + CheckJobResult $jobResult.result $jobResult.packagePath ([ref]$DupedSymbols) ([ref]$TotalFailures) + } + + if ($TotalFailures -gt 0 -or $DupedSymbols -gt 0) { + if ($TotalFailures -gt 0) { + Write-PipelineTelemetryError -Category 'CheckSymbols' -Message "Symbols missing for $TotalFailures packages" + } + + if ($DupedSymbols -gt 0) { + Write-PipelineTelemetryError -Category 'CheckSymbols' -Message "$DupedSymbols packages had duplicated symbol files" + } + ExitWithExitCode 1 } + else { + Write-Host "All symbols validated!" + } } function InstallDotnetSymbol { @@ -188,11 +257,13 @@ function InstallDotnetSymbol { try { . $PSScriptRoot\post-build-utils.ps1 - - Add-Type -AssemblyName System.IO.Compression.FileSystem InstallDotnetSymbol + foreach ($Job in @(Get-Job)) { + Remove-Job -Id $Job.Id + } + CheckSymbolsAvailable } catch { diff --git a/eng/common/sdk-task.ps1 b/eng/common/sdk-task.ps1 index 79c25e7f..32ad5c7e 100644 --- a/eng/common/sdk-task.ps1 +++ b/eng/common/sdk-task.ps1 @@ -59,14 +59,20 @@ try { if( $msbuildEngine -eq "vs") { # Ensure desktop MSBuild is available for sdk tasks. - if( -not ($GlobalJson.tools.PSObject.Properties.Name -match "vs" )) { - $GlobalJson.tools | Add-Member -Name "vs" -Value (ConvertFrom-Json "{ `"version`": `"16.4`" }") -MemberType NoteProperty + if( -not ($GlobalJson.tools.PSObject.Properties.Name -contains "vs" )) { + $GlobalJson.tools | Add-Member -Name "vs" -Value (ConvertFrom-Json "{ `"version`": `"16.5`" }") -MemberType NoteProperty } if( -not ($GlobalJson.tools.PSObject.Properties.Name -match "xcopy-msbuild" )) { - $GlobalJson.tools | Add-Member -Name "xcopy-msbuild" -Value "16.4.0-alpha" -MemberType NoteProperty + $GlobalJson.tools | Add-Member -Name "xcopy-msbuild" -Value "16.5.0-alpha" -MemberType NoteProperty + } + if ($GlobalJson.tools."xcopy-msbuild".Trim() -ine "none") { + $xcopyMSBuildToolsFolder = InitializeXCopyMSBuild $GlobalJson.tools."xcopy-msbuild" -install $true + } + if ($xcopyMSBuildToolsFolder -eq $null) { + throw 'Unable to get xcopy downloadable version of msbuild' } - InitializeXCopyMSBuild $GlobalJson.tools."xcopy-msbuild" -install $true + $global:_MSBuildExe = "$($xcopyMSBuildToolsFolder)\MSBuild\Current\Bin\MSBuild.exe" } $taskProject = GetSdkTaskProject $task diff --git a/eng/common/sdl/execute-all-sdl-tools.ps1 b/eng/common/sdl/execute-all-sdl-tools.ps1 index b7f61f9a..b681d797 100644 --- a/eng/common/sdl/execute-all-sdl-tools.ps1 +++ b/eng/common/sdl/execute-all-sdl-tools.ps1 @@ -24,7 +24,8 @@ Param( [string] $TsaIterationPath, # Optional: only needed if TsaOnboard is true; the iteration path where TSA will file bugs in AzDO; TSA is the automated framework used to upload test results as bugs. [string] $GuardianLoggerLevel='Standard', # Optional: the logger level for the Guardian CLI; options are Trace, Verbose, Standard, Warning, and Error [string[]] $CrScanAdditionalRunConfigParams, # Optional: Additional Params to custom build a CredScan run config in the format @("xyz:abc","sdf:1") - [string[]] $PoliCheckAdditionalRunConfigParams # Optional: Additional Params to custom build a Policheck run config in the format @("xyz:abc","sdf:1") + [string[]] $PoliCheckAdditionalRunConfigParams, # Optional: Additional Params to custom build a Policheck run config in the format @("xyz:abc","sdf:1") + [bool] $BreakOnFailure=$False # Optional: Fail the build if there were errors during the run ) try { @@ -106,6 +107,11 @@ try { ExitWithExitCode 1 } } + + if ($BreakOnFailure) { + Write-Host "Failing the build in case of breaking results..." + & $guardianCliLocation break + } } catch { Write-Host $_.ScriptStackTrace diff --git a/eng/common/templates/job/job.yml b/eng/common/templates/job/job.yml index fc39647f..c08bb416 100644 --- a/eng/common/templates/job/job.yml +++ b/eng/common/templates/job/job.yml @@ -26,7 +26,7 @@ parameters: enablePublishUsingPipelines: false useBuildManifest: false mergeTestResults: false - testRunTitle: $(AgentOsName)-$(BuildConfiguration)-xunit + testRunTitle: '' name: '' preSteps: [] runAsPublic: false @@ -197,7 +197,7 @@ jobs: testResultsFormat: 'xUnit' testResultsFiles: '*.xml' searchFolder: '$(Build.SourcesDirectory)/artifacts/TestResults/$(_BuildConfig)' - testRunTitle: ${{ parameters.testRunTitle }} + testRunTitle: ${{ coalesce(parameters.testRunTitle, parameters.name, '$(System.JobName)') }}-xunit mergeTestResults: ${{ parameters.mergeTestResults }} continueOnError: true condition: always() diff --git a/eng/common/templates/post-build/post-build.yml b/eng/common/templates/post-build/post-build.yml index 47be0bed..07fc2e98 100644 --- a/eng/common/templates/post-build/post-build.yml +++ b/eng/common/templates/post-build/post-build.yml @@ -1,4 +1,13 @@ parameters: + # When set to true the publishing templates from the repo will be used + # otherwise Darc add-build-to-channel will be used to trigger the promotion pipeline + inline: true + + # Only used if inline==false. When set to true will stall the current build until + # the Promotion Pipeline build finishes. Otherwise, the current build continue + # execution concurrently with the promotion build. + waitPublishingFinish: true + enableSourceLinkValidation: false enableSigningValidation: true enableSymbolValidation: false @@ -37,374 +46,478 @@ parameters: NETCoreExperimentalChannelId: 562 NetEngServicesIntChannelId: 678 NetEngServicesProdChannelId: 679 - Net5Preview3ChannelId: 739 - Net5Preview4ChannelId: 856 Net5Preview5ChannelId: 857 + Net5Preview6ChannelId: 1013 + Net5Preview7ChannelId: 1065 + NetCoreSDK313xxChannelId: 759 + NetCoreSDK313xxInternalChannelId: 760 NetCoreSDK314xxChannelId: 921 NetCoreSDK314xxInternalChannelId: 922 + VS166ChannelId: 1010 + VS167ChannelId: 1011 + VSMasterChannelId: 1012 stages: -- stage: Validate - dependsOn: ${{ parameters.validateDependsOn }} - displayName: Validate - variables: - - template: common-variables.yml - jobs: - - template: setup-maestro-vars.yml - - - job: - displayName: Post-build Checks - dependsOn: setupMaestroVars - variables: - - name: TargetChannels - value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.TargetChannels'] ] - pool: - vmImage: 'windows-2019' - steps: - - task: PowerShell@2 - displayName: Maestro Channels Consistency - inputs: - filePath: $(Build.SourcesDirectory)/eng/common/post-build/check-channel-consistency.ps1 - arguments: -PromoteToChannels "$(TargetChannels)" - -AvailableChannelIds ${{parameters.NetEngLatestChannelId}},${{parameters.NetEngValidationChannelId}},${{parameters.NetDev5ChannelId}},${{parameters.GeneralTestingChannelId}},${{parameters.NETCoreToolingDevChannelId}},${{parameters.NETCoreToolingReleaseChannelId}},${{parameters.NETInternalToolingChannelId}},${{parameters.NETCoreExperimentalChannelId}},${{parameters.NetEngServicesIntChannelId}},${{parameters.NetEngServicesProdChannelId}},${{parameters.Net5Preview3ChannelId}},${{parameters.Net5Preview4ChannelId}},${{parameters.Net5Preview5ChannelId}},${{parameters.NetCoreSDK314xxChannelId}},${{parameters.NetCoreSDK314xxInternalChannelId}} - - - job: - displayName: NuGet Validation - dependsOn: setupMaestroVars - condition: eq( ${{ parameters.enableNugetValidation }}, 'true') - pool: - vmImage: 'windows-2019' +- ${{ if ne(parameters.inline, 'true') }}: + - stage: publish_using_darc + dependsOn: ${{ parameters.validateDependsOn }} + displayName: Publish using Darc variables: - - name: AzDOProjectName - value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOProjectName'] ] - - name: AzDOPipelineId - value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOPipelineId'] ] - - name: AzDOBuildId - value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOBuildId'] ] - steps: - - task: DownloadBuildArtifacts@0 - displayName: Download Package Artifacts - inputs: - buildType: specific - buildVersionToDownload: specific - project: $(AzDOProjectName) - pipeline: $(AzDOPipelineId) - buildId: $(AzDOBuildId) - artifactName: PackageArtifacts - - - task: PowerShell@2 - displayName: Validate - inputs: - filePath: $(Build.SourcesDirectory)/eng/common/post-build/nuget-validation.ps1 - arguments: -PackagesPath $(Build.ArtifactStagingDirectory)/PackageArtifacts/ - -ToolDestinationPath $(Agent.BuildDirectory)/Extract/ - - - job: - displayName: Signing Validation - dependsOn: setupMaestroVars - condition: eq( ${{ parameters.enableSigningValidation }}, 'true') + - template: common-variables.yml + jobs: + - template: setup-maestro-vars.yml + + - job: + displayName: Publish Using Darc + dependsOn: setupMaestroVars + variables: + - name: BARBuildId + value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.BARBuildId'] ] + pool: + vmImage: 'windows-2019' + steps: + - task: PowerShell@2 + displayName: Publish Using Darc + inputs: + filePath: $(Build.SourcesDirectory)/eng/common/post-build/publish-using-darc.ps1 + arguments: -BuildId $(BARBuildId) + -AzdoToken '$(publishing-dnceng-devdiv-code-r-build-re)' + -MaestroToken '$(MaestroApiAccessToken)' + -WaitPublishingFinish ${{ parameters.waitPublishingFinish }} + -EnableSourceLinkValidation ${{ parameters.enableSourceLinkValidation }} + -EnableSigningValidation ${{ parameters.enableSourceLinkValidation }} + -EnableNugetValidation ${{ parameters.enableSourceLinkValidation }} + -PublishInstallersAndChecksums ${{ parameters.publishInstallersAndChecksums }} + -ArtifactsPublishingAdditionalParameters '${{ parameters.artifactsPublishingAdditionalParameters }}' + -SigningValidationAdditionalParameters '${{ parameters.signingValidationAdditionalParameters }}' + +- ${{ if eq(parameters.inline, 'true') }}: + - stage: Validate + dependsOn: ${{ parameters.validateDependsOn }} + displayName: Validate Build Assets variables: - template: common-variables.yml - - name: AzDOProjectName - value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOProjectName'] ] - - name: AzDOPipelineId - value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOPipelineId'] ] - - name: AzDOBuildId - value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOBuildId'] ] - pool: - vmImage: 'windows-2019' - steps: - - ${{ if eq(parameters.useBuildManifest, true) }}: + jobs: + - template: setup-maestro-vars.yml + + - job: + displayName: Post-build Checks + dependsOn: setupMaestroVars + variables: + - name: TargetChannels + value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.TargetChannels'] ] + pool: + vmImage: 'windows-2019' + steps: + - task: PowerShell@2 + displayName: Maestro Channels Consistency + inputs: + filePath: $(Build.SourcesDirectory)/eng/common/post-build/check-channel-consistency.ps1 + arguments: -PromoteToChannels "$(TargetChannels)" + -AvailableChannelIds ${{parameters.NetEngLatestChannelId}},${{parameters.NetEngValidationChannelId}},${{parameters.NetDev5ChannelId}},${{parameters.GeneralTestingChannelId}},${{parameters.NETCoreToolingDevChannelId}},${{parameters.NETCoreToolingReleaseChannelId}},${{parameters.NETInternalToolingChannelId}},${{parameters.NETCoreExperimentalChannelId}},${{parameters.NetEngServicesIntChannelId}},${{parameters.NetEngServicesProdChannelId}},${{parameters.Net5Preview5ChannelId}},${{parameters.Net5Preview6ChannelId}},${{parameters.Net5Preview7ChannelId}},${{parameters.NetCoreSDK313xxChannelId}},${{parameters.NetCoreSDK313xxInternalChannelId}},${{parameters.NetCoreSDK314xxChannelId}},${{parameters.NetCoreSDK314xxInternalChannelId}},${{parameters.VS166ChannelId}},${{parameters.VS167ChannelId}},${{parameters.VSMasterChannelId}} + + - job: + displayName: NuGet Validation + dependsOn: setupMaestroVars + condition: eq( ${{ parameters.enableNugetValidation }}, 'true') + pool: + vmImage: 'windows-2019' + variables: + - name: AzDOProjectName + value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOProjectName'] ] + - name: AzDOPipelineId + value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOPipelineId'] ] + - name: AzDOBuildId + value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOBuildId'] ] + steps: - task: DownloadBuildArtifacts@0 - displayName: Download build manifest + displayName: Download Package Artifacts inputs: buildType: specific buildVersionToDownload: specific project: $(AzDOProjectName) pipeline: $(AzDOPipelineId) buildId: $(AzDOBuildId) - artifactName: BuildManifests - - task: DownloadBuildArtifacts@0 - displayName: Download Package Artifacts - inputs: - buildType: specific - buildVersionToDownload: specific - project: $(AzDOProjectName) - pipeline: $(AzDOPipelineId) - buildId: $(AzDOBuildId) - artifactName: PackageArtifacts - - # This is necessary whenever we want to publish/restore to an AzDO private feed - # Since sdk-task.ps1 tries to restore packages we need to do this authentication here - # otherwise it'll complain about accessing a private feed. - - task: NuGetAuthenticate@0 - displayName: 'Authenticate to AzDO Feeds' - - - task: PowerShell@2 - displayName: Enable cross-org publishing - inputs: - filePath: eng\common\enable-cross-org-publishing.ps1 - arguments: -token $(dn-bot-dnceng-artifact-feeds-rw) - - # Signing validation will optionally work with the buildmanifest file which is downloaded from - # Azure DevOps above. - - task: PowerShell@2 - displayName: Validate - inputs: - filePath: eng\common\sdk-task.ps1 - arguments: -task SigningValidation -restore -msbuildEngine vs - /p:PackageBasePath='$(Build.ArtifactStagingDirectory)/PackageArtifacts' - /p:SignCheckExclusionsFile='$(Build.SourcesDirectory)/eng/SignCheckExclusionsFile.txt' - ${{ parameters.signingValidationAdditionalParameters }} - - - template: ../steps/publish-logs.yml - parameters: - StageLabel: 'Validation' - JobLabel: 'Signing' - - - job: - displayName: SourceLink Validation - dependsOn: setupMaestroVars - condition: eq( ${{ parameters.enableSourceLinkValidation }}, 'true') - variables: - - template: common-variables.yml - - name: AzDOProjectName - value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOProjectName'] ] - - name: AzDOPipelineId - value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOPipelineId'] ] - - name: AzDOBuildId - value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOBuildId'] ] - pool: - vmImage: 'windows-2019' - steps: - - task: DownloadBuildArtifacts@0 - displayName: Download Blob Artifacts - inputs: - buildType: specific - buildVersionToDownload: specific - project: $(AzDOProjectName) - pipeline: $(AzDOPipelineId) - buildId: $(AzDOBuildId) - artifactName: BlobArtifacts - - - task: PowerShell@2 - displayName: Validate - inputs: - filePath: $(Build.SourcesDirectory)/eng/common/post-build/sourcelink-validation.ps1 - arguments: -InputPath $(Build.ArtifactStagingDirectory)/BlobArtifacts/ - -ExtractPath $(Agent.BuildDirectory)/Extract/ - -GHRepoName $(Build.Repository.Name) - -GHCommit $(Build.SourceVersion) - -SourcelinkCliVersion $(SourceLinkCLIVersion) - continueOnError: true - - - template: /eng/common/templates/job/execute-sdl.yml - parameters: - enable: ${{ parameters.SDLValidationParameters.enable }} + artifactName: PackageArtifacts + + - task: PowerShell@2 + displayName: Validate + inputs: + filePath: $(Build.SourcesDirectory)/eng/common/post-build/nuget-validation.ps1 + arguments: -PackagesPath $(Build.ArtifactStagingDirectory)/PackageArtifacts/ + -ToolDestinationPath $(Agent.BuildDirectory)/Extract/ + + - job: + displayName: Signing Validation + dependsOn: setupMaestroVars + condition: eq( ${{ parameters.enableSigningValidation }}, 'true') + variables: + - template: common-variables.yml + - name: AzDOProjectName + value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOProjectName'] ] + - name: AzDOPipelineId + value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOPipelineId'] ] + - name: AzDOBuildId + value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOBuildId'] ] + pool: + vmImage: 'windows-2019' + steps: + - ${{ if eq(parameters.useBuildManifest, true) }}: + - task: DownloadBuildArtifacts@0 + displayName: Download build manifest + inputs: + buildType: specific + buildVersionToDownload: specific + project: $(AzDOProjectName) + pipeline: $(AzDOPipelineId) + buildId: $(AzDOBuildId) + artifactName: BuildManifests + - task: DownloadBuildArtifacts@0 + displayName: Download Package Artifacts + inputs: + buildType: specific + buildVersionToDownload: specific + project: $(AzDOProjectName) + pipeline: $(AzDOPipelineId) + buildId: $(AzDOBuildId) + artifactName: PackageArtifacts + + # This is necessary whenever we want to publish/restore to an AzDO private feed + # Since sdk-task.ps1 tries to restore packages we need to do this authentication here + # otherwise it'll complain about accessing a private feed. + - task: NuGetAuthenticate@0 + displayName: 'Authenticate to AzDO Feeds' + + - task: PowerShell@2 + displayName: Enable cross-org publishing + inputs: + filePath: eng\common\enable-cross-org-publishing.ps1 + arguments: -token $(dn-bot-dnceng-artifact-feeds-rw) + + # Signing validation will optionally work with the buildmanifest file which is downloaded from + # Azure DevOps above. + - task: PowerShell@2 + displayName: Validate + inputs: + filePath: eng\common\sdk-task.ps1 + arguments: -task SigningValidation -restore -msbuildEngine vs + /p:PackageBasePath='$(Build.ArtifactStagingDirectory)/PackageArtifacts' + /p:SignCheckExclusionsFile='$(Build.SourcesDirectory)/eng/SignCheckExclusionsFile.txt' + ${{ parameters.signingValidationAdditionalParameters }} + + - template: ../steps/publish-logs.yml + parameters: + StageLabel: 'Validation' + JobLabel: 'Signing' + + - job: + displayName: SourceLink Validation dependsOn: setupMaestroVars - additionalParameters: ${{ parameters.SDLValidationParameters.params }} - continueOnError: ${{ parameters.SDLValidationParameters.continueOnError }} - artifactNames: ${{ parameters.SDLValidationParameters.artifactNames }} - downloadArtifacts: ${{ parameters.SDLValidationParameters.downloadArtifacts }} - -- template: \eng\common\templates\post-build\channels\generic-public-channel.yml - parameters: - artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }} - dependsOn: ${{ parameters.publishDependsOn }} - publishInstallersAndChecksums: ${{ parameters.publishInstallersAndChecksums }} - symbolPublishingAdditionalParameters: ${{ parameters.symbolPublishingAdditionalParameters }} - stageName: 'NetCore_Dev5_Publish' - channelName: '.NET 5 Dev' - akaMSChannelName: 'net5/dev' - channelId: ${{ parameters.NetDev5ChannelId }} - transportFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet5-transport/nuget/v3/index.json' - shippingFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet5/nuget/v3/index.json' - symbolsFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet5-symbols/nuget/v3/index.json' - -- template: \eng\common\templates\post-build\channels\generic-public-channel.yml - parameters: - artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }} - dependsOn: ${{ parameters.publishDependsOn }} - publishInstallersAndChecksums: ${{ parameters.publishInstallersAndChecksums }} - symbolPublishingAdditionalParameters: ${{ parameters.symbolPublishingAdditionalParameters }} - stageName: 'Net5_Preview3_Publish' - channelName: '.NET 5 Preview 3' - akaMSChannelName: 'net5/preview3' - channelId: ${{ parameters.Net5Preview3ChannelId }} - transportFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet5-transport/nuget/v3/index.json' - shippingFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet5/nuget/v3/index.json' - symbolsFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet5-symbols/nuget/v3/index.json' - -- template: \eng\common\templates\post-build\channels\generic-public-channel.yml - parameters: - artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }} - dependsOn: ${{ parameters.publishDependsOn }} - publishInstallersAndChecksums: ${{ parameters.publishInstallersAndChecksums }} - symbolPublishingAdditionalParameters: ${{ parameters.symbolPublishingAdditionalParameters }} - stageName: 'Net5_Preview4_Publish' - channelName: '.NET 5 Preview 4' - akaMSChannelName: 'net5/preview4' - channelId: ${{ parameters.Net5Preview4ChannelId }} - transportFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet5-transport/nuget/v3/index.json' - shippingFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet5/nuget/v3/index.json' - symbolsFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet5-symbols/nuget/v3/index.json' - -- template: \eng\common\templates\post-build\channels\generic-public-channel.yml - parameters: - artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }} - dependsOn: ${{ parameters.publishDependsOn }} - publishInstallersAndChecksums: ${{ parameters.publishInstallersAndChecksums }} - symbolPublishingAdditionalParameters: ${{ parameters.symbolPublishingAdditionalParameters }} - stageName: 'Net5_Preview5_Publish' - channelName: '.NET 5 Preview 5' - akaMSChannelName: 'net5/preview5' - channelId: ${{ parameters.Net5Preview5ChannelId }} - transportFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet5-transport/nuget/v3/index.json' - shippingFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet5/nuget/v3/index.json' - symbolsFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet5-symbols/nuget/v3/index.json' - -- template: \eng\common\templates\post-build\channels\generic-public-channel.yml - parameters: - artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }} - dependsOn: ${{ parameters.publishDependsOn }} - publishInstallersAndChecksums: ${{ parameters.publishInstallersAndChecksums }} - symbolPublishingAdditionalParameters: ${{ parameters.symbolPublishingAdditionalParameters }} - stageName: 'Net_Eng_Latest_Publish' - channelName: '.NET Eng - Latest' - akaMSChannelName: 'eng/daily' - channelId: ${{ parameters.NetEngLatestChannelId }} - transportFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-eng/nuget/v3/index.json' - shippingFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-eng/nuget/v3/index.json' - symbolsFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-eng-symbols/nuget/v3/index.json' - -- template: \eng\common\templates\post-build\channels\generic-public-channel.yml - parameters: - artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }} - dependsOn: ${{ parameters.publishDependsOn }} - publishInstallersAndChecksums: ${{ parameters.publishInstallersAndChecksums }} - symbolPublishingAdditionalParameters: ${{ parameters.symbolPublishingAdditionalParameters }} - stageName: 'Net_Eng_Validation_Publish' - channelName: '.NET Eng - Validation' - akaMSChannelName: 'eng/validation' - channelId: ${{ parameters.NetEngValidationChannelId }} - transportFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-eng/nuget/v3/index.json' - shippingFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-eng/nuget/v3/index.json' - symbolsFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-eng-symbols/nuget/v3/index.json' - -- template: \eng\common\templates\post-build\channels\generic-public-channel.yml - parameters: - artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }} - dependsOn: ${{ parameters.publishDependsOn }} - publishInstallersAndChecksums: ${{ parameters.publishInstallersAndChecksums }} - symbolPublishingAdditionalParameters: ${{ parameters.symbolPublishingAdditionalParameters }} - stageName: 'General_Testing_Publish' - channelName: 'General Testing' - akaMSChannelName: 'generaltesting' - channelId: ${{ parameters.GeneralTestingChannelId }} - transportFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/general-testing/nuget/v3/index.json' - shippingFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/general-testing/nuget/v3/index.json' - symbolsFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/general-testing-symbols/nuget/v3/index.json' - -- template: \eng\common\templates\post-build\channels\generic-public-channel.yml - parameters: - artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }} - dependsOn: ${{ parameters.publishDependsOn }} - publishInstallersAndChecksums: ${{ parameters.publishInstallersAndChecksums }} - symbolPublishingAdditionalParameters: ${{ parameters.symbolPublishingAdditionalParameters }} - stageName: 'NETCore_Tooling_Dev_Publishing' - channelName: '.NET Core Tooling Dev' - channelId: ${{ parameters.NETCoreToolingDevChannelId }} - transportFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools/nuget/v3/index.json' - shippingFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools/nuget/v3/index.json' - symbolsFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools-symbols/nuget/v3/index.json' - -- template: \eng\common\templates\post-build\channels\generic-public-channel.yml - parameters: - artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }} - dependsOn: ${{ parameters.publishDependsOn }} - publishInstallersAndChecksums: ${{ parameters.publishInstallersAndChecksums }} - symbolPublishingAdditionalParameters: ${{ parameters.symbolPublishingAdditionalParameters }} - stageName: 'NETCore_Tooling_Release_Publishing' - channelName: '.NET Core Tooling Release' - channelId: ${{ parameters.NETCoreToolingReleaseChannelId }} - transportFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools/nuget/v3/index.json' - shippingFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools/nuget/v3/index.json' - symbolsFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools-symbols/nuget/v3/index.json' - -- template: \eng\common\templates\post-build\channels\generic-internal-channel.yml - parameters: - artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }} - dependsOn: ${{ parameters.publishDependsOn }} - publishInstallersAndChecksums: ${{ parameters.publishInstallersAndChecksums }} - symbolPublishingAdditionalParameters: ${{ parameters.symbolPublishingAdditionalParameters }} - stageName: 'NET_Internal_Tooling_Publishing' - channelName: '.NET Internal Tooling' - channelId: ${{ parameters.NETInternalToolingChannelId }} - transportFeed: 'https://pkgs.dev.azure.com/dnceng/internal/_packaging/dotnet-tools-internal/nuget/v3/index.json' - shippingFeed: 'https://pkgs.dev.azure.com/dnceng/internal/_packaging/dotnet-tools-internal/nuget/v3/index.json' - symbolsFeed: 'https://pkgs.dev.azure.com/dnceng/internal/_packaging/dotnet-tools-internal-symbols/nuget/v3/index.json' - -- template: \eng\common\templates\post-build\channels\generic-public-channel.yml - parameters: - artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }} - dependsOn: ${{ parameters.publishDependsOn }} - publishInstallersAndChecksums: ${{ parameters.publishInstallersAndChecksums }} - symbolPublishingAdditionalParameters: ${{ parameters.symbolPublishingAdditionalParameters }} - stageName: 'NETCore_Experimental_Publishing' - channelName: '.NET Core Experimental' - channelId: ${{ parameters.NETCoreExperimentalChannelId }} - transportFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-experimental/nuget/v3/index.json' - shippingFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-experimental/nuget/v3/index.json' - symbolsFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-experimental-symbols/nuget/v3/index.json' - -- template: \eng\common\templates\post-build\channels\generic-public-channel.yml - parameters: - artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }} - dependsOn: ${{ parameters.publishDependsOn }} - publishInstallersAndChecksums: ${{ parameters.publishInstallersAndChecksums }} - symbolPublishingAdditionalParameters: ${{ parameters.symbolPublishingAdditionalParameters }} - stageName: 'Net_Eng_Services_Int_Publish' - channelName: '.NET Eng Services - Int' - channelId: ${{ parameters.NetEngServicesIntChannelId }} - transportFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-eng/nuget/v3/index.json' - shippingFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-eng/nuget/v3/index.json' - symbolsFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-eng-symbols/nuget/v3/index.json' - -- template: \eng\common\templates\post-build\channels\generic-public-channel.yml - parameters: - artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }} - dependsOn: ${{ parameters.publishDependsOn }} - publishInstallersAndChecksums: ${{ parameters.publishInstallersAndChecksums }} - symbolPublishingAdditionalParameters: ${{ parameters.symbolPublishingAdditionalParameters }} - stageName: 'Net_Eng_Services_Prod_Publish' - channelName: '.NET Eng Services - Prod' - channelId: ${{ parameters.NetEngServicesProdChannelId }} - transportFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-eng/nuget/v3/index.json' - shippingFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-eng/nuget/v3/index.json' - symbolsFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-eng-symbols/nuget/v3/index.json' - -- template: \eng\common\templates\post-build\channels\generic-public-channel.yml - parameters: - artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }} - dependsOn: ${{ parameters.publishDependsOn }} - publishInstallersAndChecksums: ${{ parameters.publishInstallersAndChecksums }} - symbolPublishingAdditionalParameters: ${{ parameters.symbolPublishingAdditionalParameters }} - stageName: 'NETCore_SDK_314xx_Publishing' - channelName: '.NET Core SDK 3.1.4xx' - channelId: ${{ parameters.NetCoreSDK314xxChannelId }} - transportFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet3.1-transport/nuget/v3/index.json' - shippingFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet3.1/nuget/v3/index.json' - symbolsFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet3.1-symbols/nuget/v3/index.json' - -- template: \eng\common\templates\post-build\channels\generic-internal-channel.yml - parameters: - artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }} - dependsOn: ${{ parameters.publishDependsOn }} - publishInstallersAndChecksums: ${{ parameters.publishInstallersAndChecksums }} - symbolPublishingAdditionalParameters: ${{ parameters.symbolPublishingAdditionalParameters }} - stageName: 'NETCore_SDK_314xx_Internal_Publishing' - channelName: '.NET Core SDK 3.1.4xx Internal' - channelId: ${{ parameters.NetCoreSDK314xxInternalChannelId }} - transportFeed: 'https://pkgs.dev.azure.com/dnceng/_packaging/dotnet3.1-internal-transport/nuget/v3/index.json' - shippingFeed: 'https://pkgs.dev.azure.com/dnceng/_packaging/dotnet3.1-internal/nuget/v3/index.json' - symbolsFeed: 'https://pkgs.dev.azure.com/dnceng/_packaging/dotnet3.1-internal-symbols/nuget/v3/index.json' + condition: eq( ${{ parameters.enableSourceLinkValidation }}, 'true') + variables: + - template: common-variables.yml + - name: AzDOProjectName + value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOProjectName'] ] + - name: AzDOPipelineId + value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOPipelineId'] ] + - name: AzDOBuildId + value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOBuildId'] ] + pool: + vmImage: 'windows-2019' + steps: + - task: DownloadBuildArtifacts@0 + displayName: Download Blob Artifacts + inputs: + buildType: specific + buildVersionToDownload: specific + project: $(AzDOProjectName) + pipeline: $(AzDOPipelineId) + buildId: $(AzDOBuildId) + artifactName: BlobArtifacts + + - task: PowerShell@2 + displayName: Validate + inputs: + filePath: $(Build.SourcesDirectory)/eng/common/post-build/sourcelink-validation.ps1 + arguments: -InputPath $(Build.ArtifactStagingDirectory)/BlobArtifacts/ + -ExtractPath $(Agent.BuildDirectory)/Extract/ + -GHRepoName $(Build.Repository.Name) + -GHCommit $(Build.SourceVersion) + -SourcelinkCliVersion $(SourceLinkCLIVersion) + continueOnError: true + + - template: /eng/common/templates/job/execute-sdl.yml + parameters: + enable: ${{ parameters.SDLValidationParameters.enable }} + dependsOn: setupMaestroVars + additionalParameters: ${{ parameters.SDLValidationParameters.params }} + continueOnError: ${{ parameters.SDLValidationParameters.continueOnError }} + artifactNames: ${{ parameters.SDLValidationParameters.artifactNames }} + downloadArtifacts: ${{ parameters.SDLValidationParameters.downloadArtifacts }} + + - template: \eng\common\templates\post-build\channels\generic-public-channel.yml + parameters: + artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }} + dependsOn: ${{ parameters.publishDependsOn }} + publishInstallersAndChecksums: ${{ parameters.publishInstallersAndChecksums }} + symbolPublishingAdditionalParameters: ${{ parameters.symbolPublishingAdditionalParameters }} + stageName: 'NetCore_Dev5_Publish' + channelName: '.NET 5 Dev' + akaMSChannelName: 'net5/dev' + channelId: ${{ parameters.NetDev5ChannelId }} + transportFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet5-transport/nuget/v3/index.json' + shippingFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet5/nuget/v3/index.json' + symbolsFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet5-symbols/nuget/v3/index.json' + + - template: \eng\common\templates\post-build\channels\generic-public-channel.yml + parameters: + artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }} + dependsOn: ${{ parameters.publishDependsOn }} + publishInstallersAndChecksums: ${{ parameters.publishInstallersAndChecksums }} + symbolPublishingAdditionalParameters: ${{ parameters.symbolPublishingAdditionalParameters }} + stageName: 'Net5_Preview5_Publish' + channelName: '.NET 5 Preview 5' + akaMSChannelName: 'net5/preview5' + channelId: ${{ parameters.Net5Preview5ChannelId }} + transportFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet5-transport/nuget/v3/index.json' + shippingFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet5/nuget/v3/index.json' + symbolsFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet5-symbols/nuget/v3/index.json' + + - template: \eng\common\templates\post-build\channels\generic-public-channel.yml + parameters: + artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }} + dependsOn: ${{ parameters.publishDependsOn }} + publishInstallersAndChecksums: ${{ parameters.publishInstallersAndChecksums }} + symbolPublishingAdditionalParameters: ${{ parameters.symbolPublishingAdditionalParameters }} + stageName: 'Net5_Preview6_Publish' + channelName: '.NET 5 Preview 6' + akaMSChannelName: 'net5/preview6' + channelId: ${{ parameters.Net5Preview6ChannelId }} + transportFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet5-transport/nuget/v3/index.json' + shippingFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet5/nuget/v3/index.json' + symbolsFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet5-symbols/nuget/v3/index.json' + + - template: \eng\common\templates\post-build\channels\generic-public-channel.yml + parameters: + artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }} + dependsOn: ${{ parameters.publishDependsOn }} + publishInstallersAndChecksums: ${{ parameters.publishInstallersAndChecksums }} + symbolPublishingAdditionalParameters: ${{ parameters.symbolPublishingAdditionalParameters }} + stageName: 'Net5_Preview7_Publish' + channelName: '.NET 5 Preview 7' + akaMSChannelName: 'net5/preview7' + channelId: ${{ parameters.Net5Preview7ChannelId }} + transportFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet5-transport/nuget/v3/index.json' + shippingFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet5/nuget/v3/index.json' + symbolsFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet5-symbols/nuget/v3/index.json' + + - template: \eng\common\templates\post-build\channels\generic-public-channel.yml + parameters: + artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }} + dependsOn: ${{ parameters.publishDependsOn }} + publishInstallersAndChecksums: ${{ parameters.publishInstallersAndChecksums }} + symbolPublishingAdditionalParameters: ${{ parameters.symbolPublishingAdditionalParameters }} + stageName: 'Net_Eng_Latest_Publish' + channelName: '.NET Eng - Latest' + akaMSChannelName: 'eng/daily' + channelId: ${{ parameters.NetEngLatestChannelId }} + transportFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-eng/nuget/v3/index.json' + shippingFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-eng/nuget/v3/index.json' + symbolsFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-eng-symbols/nuget/v3/index.json' + + - template: \eng\common\templates\post-build\channels\generic-public-channel.yml + parameters: + artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }} + dependsOn: ${{ parameters.publishDependsOn }} + publishInstallersAndChecksums: ${{ parameters.publishInstallersAndChecksums }} + symbolPublishingAdditionalParameters: ${{ parameters.symbolPublishingAdditionalParameters }} + stageName: 'Net_Eng_Validation_Publish' + channelName: '.NET Eng - Validation' + akaMSChannelName: 'eng/validation' + channelId: ${{ parameters.NetEngValidationChannelId }} + transportFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-eng/nuget/v3/index.json' + shippingFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-eng/nuget/v3/index.json' + symbolsFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-eng-symbols/nuget/v3/index.json' + + - template: \eng\common\templates\post-build\channels\generic-public-channel.yml + parameters: + artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }} + dependsOn: ${{ parameters.publishDependsOn }} + publishInstallersAndChecksums: ${{ parameters.publishInstallersAndChecksums }} + symbolPublishingAdditionalParameters: ${{ parameters.symbolPublishingAdditionalParameters }} + stageName: 'General_Testing_Publish' + channelName: 'General Testing' + akaMSChannelName: 'generaltesting' + channelId: ${{ parameters.GeneralTestingChannelId }} + transportFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/general-testing/nuget/v3/index.json' + shippingFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/general-testing/nuget/v3/index.json' + symbolsFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/general-testing-symbols/nuget/v3/index.json' + + - template: \eng\common\templates\post-build\channels\generic-public-channel.yml + parameters: + artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }} + dependsOn: ${{ parameters.publishDependsOn }} + publishInstallersAndChecksums: ${{ parameters.publishInstallersAndChecksums }} + symbolPublishingAdditionalParameters: ${{ parameters.symbolPublishingAdditionalParameters }} + stageName: 'NETCore_Tooling_Dev_Publishing' + channelName: '.NET Core Tooling Dev' + channelId: ${{ parameters.NETCoreToolingDevChannelId }} + transportFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools/nuget/v3/index.json' + shippingFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools/nuget/v3/index.json' + symbolsFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools-symbols/nuget/v3/index.json' + + - template: \eng\common\templates\post-build\channels\generic-public-channel.yml + parameters: + artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }} + dependsOn: ${{ parameters.publishDependsOn }} + publishInstallersAndChecksums: ${{ parameters.publishInstallersAndChecksums }} + symbolPublishingAdditionalParameters: ${{ parameters.symbolPublishingAdditionalParameters }} + stageName: 'NETCore_Tooling_Release_Publishing' + channelName: '.NET Core Tooling Release' + channelId: ${{ parameters.NETCoreToolingReleaseChannelId }} + transportFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools/nuget/v3/index.json' + shippingFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools/nuget/v3/index.json' + symbolsFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools-symbols/nuget/v3/index.json' + + - template: \eng\common\templates\post-build\channels\generic-internal-channel.yml + parameters: + artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }} + dependsOn: ${{ parameters.publishDependsOn }} + publishInstallersAndChecksums: ${{ parameters.publishInstallersAndChecksums }} + symbolPublishingAdditionalParameters: ${{ parameters.symbolPublishingAdditionalParameters }} + stageName: 'NET_Internal_Tooling_Publishing' + channelName: '.NET Internal Tooling' + channelId: ${{ parameters.NETInternalToolingChannelId }} + transportFeed: 'https://pkgs.dev.azure.com/dnceng/internal/_packaging/dotnet-tools-internal/nuget/v3/index.json' + shippingFeed: 'https://pkgs.dev.azure.com/dnceng/internal/_packaging/dotnet-tools-internal/nuget/v3/index.json' + symbolsFeed: 'https://pkgs.dev.azure.com/dnceng/internal/_packaging/dotnet-tools-internal-symbols/nuget/v3/index.json' + + - template: \eng\common\templates\post-build\channels\generic-public-channel.yml + parameters: + artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }} + dependsOn: ${{ parameters.publishDependsOn }} + publishInstallersAndChecksums: ${{ parameters.publishInstallersAndChecksums }} + symbolPublishingAdditionalParameters: ${{ parameters.symbolPublishingAdditionalParameters }} + stageName: 'NETCore_Experimental_Publishing' + channelName: '.NET Core Experimental' + channelId: ${{ parameters.NETCoreExperimentalChannelId }} + transportFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-experimental/nuget/v3/index.json' + shippingFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-experimental/nuget/v3/index.json' + symbolsFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-experimental-symbols/nuget/v3/index.json' + + - template: \eng\common\templates\post-build\channels\generic-public-channel.yml + parameters: + artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }} + dependsOn: ${{ parameters.publishDependsOn }} + publishInstallersAndChecksums: ${{ parameters.publishInstallersAndChecksums }} + symbolPublishingAdditionalParameters: ${{ parameters.symbolPublishingAdditionalParameters }} + stageName: 'Net_Eng_Services_Int_Publish' + channelName: '.NET Eng Services - Int' + channelId: ${{ parameters.NetEngServicesIntChannelId }} + transportFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-eng/nuget/v3/index.json' + shippingFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-eng/nuget/v3/index.json' + symbolsFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-eng-symbols/nuget/v3/index.json' + + - template: \eng\common\templates\post-build\channels\generic-public-channel.yml + parameters: + artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }} + dependsOn: ${{ parameters.publishDependsOn }} + publishInstallersAndChecksums: ${{ parameters.publishInstallersAndChecksums }} + symbolPublishingAdditionalParameters: ${{ parameters.symbolPublishingAdditionalParameters }} + stageName: 'Net_Eng_Services_Prod_Publish' + channelName: '.NET Eng Services - Prod' + channelId: ${{ parameters.NetEngServicesProdChannelId }} + transportFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-eng/nuget/v3/index.json' + shippingFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-eng/nuget/v3/index.json' + symbolsFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-eng-symbols/nuget/v3/index.json' + + - template: \eng\common\templates\post-build\channels\generic-public-channel.yml + parameters: + artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }} + dependsOn: ${{ parameters.publishDependsOn }} + publishInstallersAndChecksums: ${{ parameters.publishInstallersAndChecksums }} + symbolPublishingAdditionalParameters: ${{ parameters.symbolPublishingAdditionalParameters }} + stageName: 'NETCore_SDK_314xx_Publishing' + channelName: '.NET Core SDK 3.1.4xx' + channelId: ${{ parameters.NetCoreSDK314xxChannelId }} + transportFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet3.1-transport/nuget/v3/index.json' + shippingFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet3.1/nuget/v3/index.json' + symbolsFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet3.1-symbols/nuget/v3/index.json' + + - template: \eng\common\templates\post-build\channels\generic-internal-channel.yml + parameters: + artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }} + dependsOn: ${{ parameters.publishDependsOn }} + publishInstallersAndChecksums: ${{ parameters.publishInstallersAndChecksums }} + symbolPublishingAdditionalParameters: ${{ parameters.symbolPublishingAdditionalParameters }} + stageName: 'NETCore_SDK_314xx_Internal_Publishing' + channelName: '.NET Core SDK 3.1.4xx Internal' + channelId: ${{ parameters.NetCoreSDK314xxInternalChannelId }} + transportFeed: 'https://pkgs.dev.azure.com/dnceng/_packaging/dotnet3.1-internal-transport/nuget/v3/index.json' + shippingFeed: 'https://pkgs.dev.azure.com/dnceng/_packaging/dotnet3.1-internal/nuget/v3/index.json' + symbolsFeed: 'https://pkgs.dev.azure.com/dnceng/_packaging/dotnet3.1-internal-symbols/nuget/v3/index.json' + + - template: \eng\common\templates\post-build\channels\generic-public-channel.yml + parameters: + artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }} + dependsOn: ${{ parameters.publishDependsOn }} + publishInstallersAndChecksums: ${{ parameters.publishInstallersAndChecksums }} + symbolPublishingAdditionalParameters: ${{ parameters.symbolPublishingAdditionalParameters }} + stageName: 'NETCore_SDK_313xx_Publishing' + channelName: '.NET Core SDK 3.1.3xx' + channelId: ${{ parameters.NetCoreSDK313xxChannelId }} + transportFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet3.1-transport/nuget/v3/index.json' + shippingFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet3.1/nuget/v3/index.json' + symbolsFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet3.1-symbols/nuget/v3/index.json' + + - template: \eng\common\templates\post-build\channels\generic-internal-channel.yml + parameters: + artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }} + dependsOn: ${{ parameters.publishDependsOn }} + publishInstallersAndChecksums: ${{ parameters.publishInstallersAndChecksums }} + symbolPublishingAdditionalParameters: ${{ parameters.symbolPublishingAdditionalParameters }} + stageName: 'NETCore_SDK_313xx_Internal_Publishing' + channelName: '.NET Core SDK 3.1.3xx Internal' + channelId: ${{ parameters.NetCoreSDK313xxInternalChannelId }} + transportFeed: 'https://pkgs.dev.azure.com/dnceng/_packaging/dotnet3.1-internal-transport/nuget/v3/index.json' + shippingFeed: 'https://pkgs.dev.azure.com/dnceng/_packaging/dotnet3.1-internal/nuget/v3/index.json' + symbolsFeed: 'https://pkgs.dev.azure.com/dnceng/_packaging/dotnet3.1-internal-symbols/nuget/v3/index.json' + + - template: \eng\common\templates\post-build\channels\generic-public-channel.yml + parameters: + artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }} + dependsOn: ${{ parameters.publishDependsOn }} + publishInstallersAndChecksums: ${{ parameters.publishInstallersAndChecksums }} + symbolPublishingAdditionalParameters: ${{ parameters.symbolPublishingAdditionalParameters }} + stageName: 'VS16_6_Publishing' + channelName: 'VS 16.6' + channelId: ${{ parameters.VS166ChannelId }} + transportFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools-transport/nuget/v3/index.json' + shippingFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools/nuget/v3/index.json' + symbolsFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools-symbols/nuget/v3/index.json' + + - template: \eng\common\templates\post-build\channels\generic-public-channel.yml + parameters: + artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }} + dependsOn: ${{ parameters.publishDependsOn }} + publishInstallersAndChecksums: ${{ parameters.publishInstallersAndChecksums }} + symbolPublishingAdditionalParameters: ${{ parameters.symbolPublishingAdditionalParameters }} + stageName: 'VS16_7_Publishing' + channelName: 'VS 16.7' + channelId: ${{ parameters.VS167ChannelId }} + transportFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools-transport/nuget/v3/index.json' + shippingFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools/nuget/v3/index.json' + symbolsFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools-symbols/nuget/v3/index.json' + + - template: \eng\common\templates\post-build\channels\generic-public-channel.yml + parameters: + artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }} + dependsOn: ${{ parameters.publishDependsOn }} + publishInstallersAndChecksums: ${{ parameters.publishInstallersAndChecksums }} + symbolPublishingAdditionalParameters: ${{ parameters.symbolPublishingAdditionalParameters }} + stageName: 'VS_Master_Publishing' + channelName: 'VS Master' + channelId: ${{ parameters.VSMasterChannelId }} + transportFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools-transport/nuget/v3/index.json' + shippingFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools/nuget/v3/index.json' + symbolsFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools-symbols/nuget/v3/index.json' diff --git a/eng/common/tools.ps1 b/eng/common/tools.ps1 index f31377a6..d68684e2 100644 --- a/eng/common/tools.ps1 +++ b/eng/common/tools.ps1 @@ -7,9 +7,11 @@ # Build configuration. Common values include 'Debug' and 'Release', but the repository may use other names. [string]$configuration = if (Test-Path variable:configuration) { $configuration } else { 'Debug' } +# Set to true to opt out of outputting binary log while running in CI +[bool]$excludeCIBinarylog = if (Test-Path variable:excludeCIBinarylog) { $excludeCIBinarylog } else { $false } + # Set to true to output binary log from msbuild. Note that emitting binary log slows down the build. -# Binary log must be enabled on CI. -[bool]$binaryLog = if (Test-Path variable:binaryLog) { $binaryLog } else { $ci } +[bool]$binaryLog = if (Test-Path variable:binaryLog) { $binaryLog } else { $ci -and !$excludeCIBinarylog } # Set to true to use the pipelines logger which will enable Azure logging output. # https://github.com/Microsoft/azure-pipelines-tasks/blob/master/docs/authoring/commands.md @@ -55,10 +57,8 @@ set-strictmode -version 2.0 $ErrorActionPreference = 'Stop' [Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12 -function Create-Directory([string[]] $path) { - if (!(Test-Path $path)) { - New-Item -path $path -force -itemType 'Directory' | Out-Null - } +function Create-Directory ([string[]] $path) { + New-Item -Path $path -Force -ItemType 'Directory' | Out-Null } function Unzip([string]$zipfile, [string]$outpath) { @@ -124,7 +124,9 @@ function InitializeDotNetCli([bool]$install, [bool]$createSdkLocationFile) { # Find the first path on %PATH% that contains the dotnet.exe if ($useInstalledDotNetCli -and (-not $globalJsonHasRuntimes) -and ($env:DOTNET_INSTALL_DIR -eq $null)) { - $dotnetCmd = Get-Command 'dotnet.exe' -ErrorAction SilentlyContinue + $dotnetExecutable = GetExecutableFileName 'dotnet' + $dotnetCmd = Get-Command $dotnetExecutable -ErrorAction SilentlyContinue + if ($dotnetCmd -ne $null) { $env:DOTNET_INSTALL_DIR = Split-Path $dotnetCmd.Path -Parent } @@ -283,12 +285,19 @@ function InstallDotNet([string] $dotnetRoot, # Throws on failure. # function InitializeVisualStudioMSBuild([bool]$install, [object]$vsRequirements = $null) { + if (-not (IsWindowsPlatform)) { + throw "Cannot initialize Visual Studio on non-Windows" + } + if (Test-Path variable:global:_MSBuildExe) { return $global:_MSBuildExe } + $vsMinVersionReqdStr = '16.5' + $vsMinVersionReqd = [Version]::new($vsMinVersionReqdStr) + if (!$vsRequirements) { $vsRequirements = $GlobalJson.tools.vs } - $vsMinVersionStr = if ($vsRequirements.version) { $vsRequirements.version } else { '15.9' } + $vsMinVersionStr = if ($vsRequirements.version) { $vsRequirements.version } else { $vsMinVersionReqdStr } $vsMinVersion = [Version]::new($vsMinVersionStr) # Try msbuild command available in the environment. @@ -321,8 +330,18 @@ function InitializeVisualStudioMSBuild([bool]$install, [object]$vsRequirements = $xcopyMSBuildVersion = $GlobalJson.tools.'xcopy-msbuild' $vsMajorVersion = $xcopyMSBuildVersion.Split('.')[0] } else { - $vsMajorVersion = $vsMinVersion.Major - $xcopyMSBuildVersion = "$vsMajorVersion.$($vsMinVersion.Minor).0-alpha" + #if vs version provided in global.json is incompatible then use the default version for xcopy msbuild download + if($vsMinVersion -lt $vsMinVersionReqd){ + Write-Host "Using xcopy-msbuild version of $vsMinVersionReqdStr.0-alpha since VS version $vsMinVersionStr provided in global.json is not compatible" + $vsMajorVersion = $vsMinVersionReqd.Major + $vsMinorVersion = $vsMinVersionReqd.Minor + } + else{ + $vsMajorVersion = $vsMinVersion.Major + $vsMinorVersion = $vsMinVersion.Minor + } + + $xcopyMSBuildVersion = "$vsMajorVersion.$vsMinorVersion.0-alpha" } $vsInstallDir = $null @@ -387,6 +406,10 @@ function InitializeXCopyMSBuild([string]$packageVersion, [bool]$install) { # or $null if no instance meeting the requirements is found on the machine. # function LocateVisualStudio([object]$vsRequirements = $null){ + if (-not (IsWindowsPlatform)) { + throw "Cannot run vswhere on non-Windows platforms." + } + if (Get-Member -InputObject $GlobalJson.tools -Name 'vswhere') { $vswhereVersion = $GlobalJson.tools.vswhere } else { @@ -452,7 +475,8 @@ function InitializeBuildTool() { Write-PipelineTelemetryError -Category 'InitializeToolset' -Message "/global.json must specify 'tools.dotnet'." ExitWithExitCode 1 } - $buildTool = @{ Path = Join-Path $dotnetRoot 'dotnet.exe'; Command = 'msbuild'; Tool = 'dotnet'; Framework = 'netcoreapp2.1' } + $dotnetPath = Join-Path $dotnetRoot (GetExecutableFileName 'dotnet') + $buildTool = @{ Path = $dotnetPath; Command = 'msbuild'; Tool = 'dotnet'; Framework = 'netcoreapp2.1' } } elseif ($msbuildEngine -eq "vs") { try { $msbuildPath = InitializeVisualStudioMSBuild -install:$restore @@ -605,8 +629,8 @@ function MSBuild() { # function MSBuild-Core() { if ($ci) { - if (!$binaryLog) { - Write-PipelineTelemetryError -Category 'Build' -Message 'Binary log must be enabled in CI build.' + if (!$binaryLog -and !$excludeCIBinarylog) { + Write-PipelineTelemetryError -Category 'Build' -Message 'Binary log must be enabled in CI build, or explicitly opted-out from with the -excludeCIBinarylog switch.' ExitWithExitCode 1 } @@ -666,6 +690,19 @@ function GetMSBuildBinaryLogCommandLineArgument($arguments) { return $null } +function GetExecutableFileName($baseName) { + if (IsWindowsPlatform) { + return "$baseName.exe" + } + else { + return $baseName + } +} + +function IsWindowsPlatform() { + return [environment]::OSVersion.Platform -eq [PlatformID]::Win32NT +} + . $PSScriptRoot\pipeline-logging-functions.ps1 $RepoRoot = Resolve-Path (Join-Path $PSScriptRoot '..\..') diff --git a/eng/common/tools.sh b/eng/common/tools.sh index a9dff440..e94fce22 100755 --- a/eng/common/tools.sh +++ b/eng/common/tools.sh @@ -18,9 +18,17 @@ fi # Build configuration. Common values include 'Debug' and 'Release', but the repository may use other names. configuration=${configuration:-'Debug'} +# Set to true to opt out of outputting binary log while running in CI +exclude_ci_binary_log=${exclude_ci_binary_log:-false} + +if [[ "$ci" == true && "$exclude_ci_binary_log" == false ]]; then + binary_log_default=true +else + binary_log_default=false +fi + # Set to true to output binary log from msbuild. Note that emitting binary log slows down the build. -# Binary log must be enabled on CI. -binary_log=${binary_log:-$ci} +binary_log=${binary_log:-$binary_log_default} # Turns on machine preparation/clean up code that changes the machine state (e.g. kills build processes). prepare_machine=${prepare_machine:-false} @@ -404,8 +412,8 @@ function MSBuild { function MSBuild-Core { if [[ "$ci" == true ]]; then - if [[ "$binary_log" != true ]]; then - Write-PipelineTelemetryError -category 'Build' "Binary log must be enabled in CI build." + if [[ "$binary_log" != true && "$exclude_ci_binary_log" != true ]]; then + Write-PipelineTelemetryError -category 'Build' "Binary log must be enabled in CI build, or explicitly opted-out from with the -noBinaryLog switch." ExitWithExitCode 1 fi diff --git a/global.json b/global.json index 79be6e17..c030b13a 100644 --- a/global.json +++ b/global.json @@ -1,13 +1,16 @@ { "tools": { - "dotnet": "3.1.102", + "dotnet": "5.0.100-preview.6.20310.4", "runtimes": { "dotnet": [ "3.1.2" + ], + "aspnetcore": [ + "3.1.4" ] } }, "msbuild-sdks": { - "Microsoft.DotNet.Arcade.Sdk": "5.0.0-beta.20228.4" + "Microsoft.DotNet.Arcade.Sdk": "5.0.0-beta.20330.3" } } diff --git a/src/Microsoft.Tye.Core/ApplicationFactory.cs b/src/Microsoft.Tye.Core/ApplicationFactory.cs index 138121fb..d1d95e15 100644 --- a/src/Microsoft.Tye.Core/ApplicationFactory.cs +++ b/src/Microsoft.Tye.Core/ApplicationFactory.cs @@ -88,7 +88,7 @@ namespace Microsoft.Tye if (!string.IsNullOrEmpty(configService.Project)) { var expandedProject = Environment.ExpandEnvironmentVariables(configService.Project); - var projectFile = new FileInfo(Path.Combine(config.Source.DirectoryName, expandedProject)); + var projectFile = new FileInfo(Path.Combine(config.Source.DirectoryName!, expandedProject)); var project = new DotnetProjectServiceBuilder(configService.Name!, projectFile); service = project; @@ -131,7 +131,7 @@ namespace Microsoft.Tye Args = configService.Args, Build = configService.Build ?? true, Replicas = configService.Replicas ?? 1, - DockerFile = Path.Combine(source.DirectoryName, configService.DockerFile), + DockerFile = Path.Combine(source.DirectoryName!, configService.DockerFile), // Supplying an absolute path with trailing slashes fails for DockerFileContext when calling docker build, so trim trailing slash. DockerFileContext = GetDockerFileContext(source, configService), BuildArgs = configService.DockerFileArgs @@ -156,7 +156,7 @@ namespace Microsoft.Tye // Special handling of .dlls as executables (it will be executed as dotnet {dll}) if (Path.GetExtension(expandedExecutable) == ".dll") { - expandedExecutable = Path.GetFullPath(Path.Combine(config.Source.Directory.FullName, expandedExecutable)); + expandedExecutable = Path.GetFullPath(Path.Combine(config.Source.Directory!.FullName, expandedExecutable)); workingDirectory = Path.GetDirectoryName(expandedExecutable)!; } @@ -164,7 +164,7 @@ namespace Microsoft.Tye { Args = configService.Args, WorkingDirectory = configService.WorkingDirectory != null ? - Path.GetFullPath(Path.Combine(config.Source.Directory.FullName, Environment.ExpandEnvironmentVariables(configService.WorkingDirectory))) : + Path.GetFullPath(Path.Combine(config.Source.Directory!.FullName, Environment.ExpandEnvironmentVariables(configService.WorkingDirectory))) : workingDirectory, Replicas = configService.Replicas ?? 1 }; @@ -177,7 +177,7 @@ namespace Microsoft.Tye { var expandedYaml = Environment.ExpandEnvironmentVariables(configService.Include); - var nestedConfig = GetNestedConfig(rootConfig, Path.Combine(config.Source.DirectoryName, expandedYaml)); + var nestedConfig = GetNestedConfig(rootConfig, Path.Combine(config.Source.DirectoryName!, expandedYaml)); queue.Enqueue((nestedConfig, new HashSet())); AddToRootServices(root, dependencies, configService.Name); @@ -227,7 +227,7 @@ namespace Microsoft.Tye { var functionBuilder = new AzureFunctionServiceBuilder( configService.Name, - Path.Combine(config.Source.DirectoryName, configService.AzureFunction)) + Path.Combine(config.Source.DirectoryName!, configService.AzureFunction)) { Args = configService.Args, Replicas = configService.Replicas ?? 1, @@ -395,11 +395,11 @@ namespace Microsoft.Tye // but it's the exact opposite on linux, where it needs to have the trailing slash. if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) { - return Path.TrimEndingDirectorySeparator(Path.Combine(source.DirectoryName, configService.DockerFileContext)); + return Path.TrimEndingDirectorySeparator(Path.Combine(source.DirectoryName!, configService.DockerFileContext)); } else { - var path = Path.Combine(source.DirectoryName, configService.DockerFileContext); + var path = Path.Combine(source.DirectoryName!, configService.DockerFileContext); if (!Path.EndsInDirectorySeparator(path)) { @@ -412,7 +412,7 @@ namespace Microsoft.Tye private static ConfigApplication GetNestedConfig(ConfigApplication rootConfig, string? file) { - var nestedConfig = ConfigFactory.FromFile(new FileInfo(file)); + var nestedConfig = ConfigFactory.FromFile(new FileInfo(file!)); nestedConfig.Validate(); if (nestedConfig.Name != rootConfig.Name) diff --git a/src/Microsoft.Tye.Core/ConfigModel/ConfigFactory.cs b/src/Microsoft.Tye.Core/ConfigModel/ConfigFactory.cs index 49a22322..cd511102 100644 --- a/src/Microsoft.Tye.Core/ConfigModel/ConfigFactory.cs +++ b/src/Microsoft.Tye.Core/ConfigModel/ConfigFactory.cs @@ -71,7 +71,7 @@ namespace Microsoft.Tye.ConfigModel // // We want a *fast* heuristic that excludes unit test projects and class libraries without // having to load all of the projects. - var launchSettings = Path.Combine(projectFile.DirectoryName, "Properties", "launchSettings.json"); + var launchSettings = Path.Combine(projectFile.DirectoryName!, "Properties", "launchSettings.json"); if (File.Exists(launchSettings) || ContainsOutputTypeExe(projectFile)) { var service = new ConfigService() diff --git a/src/Microsoft.Tye.Core/ConfigModel/NameInferer.cs b/src/Microsoft.Tye.Core/ConfigModel/NameInferer.cs index 2dd4195a..506ddff4 100644 --- a/src/Microsoft.Tye.Core/ConfigModel/NameInferer.cs +++ b/src/Microsoft.Tye.Core/ConfigModel/NameInferer.cs @@ -21,7 +21,7 @@ namespace Microsoft.Tye.ConfigModel return Path.GetFileNameWithoutExtension(fileInfo.Name).ToLowerInvariant(); } - return fileInfo.Directory.Parent.Name.ToLowerInvariant(); + return fileInfo.Directory?.Parent?.Name.ToLowerInvariant(); } } } diff --git a/src/Microsoft.Tye.Core/DockerContainerBuilder.cs b/src/Microsoft.Tye.Core/DockerContainerBuilder.cs index 409a8694..043ef814 100644 --- a/src/Microsoft.Tye.Core/DockerContainerBuilder.cs +++ b/src/Microsoft.Tye.Core/DockerContainerBuilder.cs @@ -36,7 +36,7 @@ namespace Microsoft.Tye var dockerFileInfo = new FileInfo(containerService.DockerFile); var contextDirectory = containerService.DockerFileContext ?? dockerFileInfo.DirectoryName; - var dockerFilePath = Path.Combine(dockerFileInfo.DirectoryName, "Dockerfile"); + var dockerFilePath = Path.Combine(dockerFileInfo.DirectoryName!, "Dockerfile"); output.WriteDebugLine($"Using existing Dockerfile '{dockerFilePath}'."); @@ -83,7 +83,7 @@ namespace Microsoft.Tye } string contextDirectory; - var dockerFilePath = Path.Combine(project.ProjectFile.DirectoryName, "Dockerfile"); + var dockerFilePath = Path.Combine(project.ProjectFile.DirectoryName!, "Dockerfile"); TempFile? tempFile = null; TempDirectory? tempDirectory = null; diff --git a/src/Microsoft.Tye.Core/FuncDetector.cs b/src/Microsoft.Tye.Core/FuncDetector.cs index a7189ded..209b5a92 100644 --- a/src/Microsoft.Tye.Core/FuncDetector.cs +++ b/src/Microsoft.Tye.Core/FuncDetector.cs @@ -165,7 +165,7 @@ namespace Microsoft.Tye if (!dryRun) { logger.LogInformation("Writing list of azure function versions to {FeedJsonFile}.", feedJsonFile); - Directory.CreateDirectory(new FileInfo(feedJsonFile).DirectoryName); + Directory.CreateDirectory(new FileInfo(feedJsonFile).DirectoryName!); await File.WriteAllTextAsync(feedJsonFile, responseString, cancellation); } } diff --git a/src/Microsoft.Tye.Core/GenerateApplicationKubernetesManifestStep.cs b/src/Microsoft.Tye.Core/GenerateApplicationKubernetesManifestStep.cs index b23041c6..30823fd6 100644 --- a/src/Microsoft.Tye.Core/GenerateApplicationKubernetesManifestStep.cs +++ b/src/Microsoft.Tye.Core/GenerateApplicationKubernetesManifestStep.cs @@ -16,7 +16,7 @@ namespace Microsoft.Tye public override async Task ExecuteAsync(OutputContext output, ApplicationBuilder application) { - var outputFilePath = Path.GetFullPath(Path.Combine(application.Source.DirectoryName, $"{application.Name}-generate-{Environment}.yaml")); + var outputFilePath = Path.GetFullPath(Path.Combine(application.Source.DirectoryName!, $"{application.Name}-generate-{Environment}.yaml")); output.WriteInfoLine($"Writing output to '{outputFilePath}'."); { File.Delete(outputFilePath); diff --git a/src/Microsoft.Tye.Core/GenerateDockerfileStep.cs b/src/Microsoft.Tye.Core/GenerateDockerfileStep.cs index f0fc9733..4bb13958 100644 --- a/src/Microsoft.Tye.Core/GenerateDockerfileStep.cs +++ b/src/Microsoft.Tye.Core/GenerateDockerfileStep.cs @@ -32,7 +32,7 @@ namespace Microsoft.Tye container.UseMultiphaseDockerfile ??= true; - var dockerFilePath = Path.Combine(project.ProjectFile.DirectoryName, "Dockerfile"); + var dockerFilePath = Path.Combine(project.ProjectFile.DirectoryName!, "Dockerfile"); if (File.Exists(dockerFilePath) && !Force) { throw new CommandException("'Dockerfile' already exists for project. use '--force' to overwrite."); diff --git a/src/Microsoft.Tye.Core/GenerateHelmChartStep.cs b/src/Microsoft.Tye.Core/GenerateHelmChartStep.cs index d2ebeddb..7bfeb278 100644 --- a/src/Microsoft.Tye.Core/GenerateHelmChartStep.cs +++ b/src/Microsoft.Tye.Core/GenerateHelmChartStep.cs @@ -25,7 +25,7 @@ namespace Microsoft.Tye return Task.CompletedTask; } - var chartDirectory = Path.Combine(project.ProjectFile.DirectoryName, "charts"); + var chartDirectory = Path.Combine(project.ProjectFile.DirectoryName!, "charts"); if (Directory.Exists(chartDirectory) && !Force) { throw new CommandException("'charts' directory already exists for project. use '--force' to overwrite."); diff --git a/src/Microsoft.Tye.Core/Microsoft.Tye.Core.csproj b/src/Microsoft.Tye.Core/Microsoft.Tye.Core.csproj index 2f4b6ad6..79d85550 100644 --- a/src/Microsoft.Tye.Core/Microsoft.Tye.Core.csproj +++ b/src/Microsoft.Tye.Core/Microsoft.Tye.Core.csproj @@ -1,7 +1,7 @@  - netcoreapp3.1 + net5.0 Tye Microsoft.Tye.Core Microsoft.Tye.Core @@ -13,7 +13,7 @@ The Microsoft.Build.Locator package takes care of dynamically loading these assemblies at runtime. We don't need/want to ship them, just to have them as references. --> - + diff --git a/src/Microsoft.Tye.Core/OutputContext.cs b/src/Microsoft.Tye.Core/OutputContext.cs index 8b766504..7f2c569a 100644 --- a/src/Microsoft.Tye.Core/OutputContext.cs +++ b/src/Microsoft.Tye.Core/OutputContext.cs @@ -158,7 +158,7 @@ namespace Microsoft.Tye if (allowEmpty || !string.IsNullOrEmpty(line)) { - return line.Trim(); + return line!.Trim(); } } } diff --git a/src/Microsoft.Tye.Core/ProcessExtensions.cs b/src/Microsoft.Tye.Core/ProcessExtensions.cs index a3885af2..de62be31 100644 --- a/src/Microsoft.Tye.Core/ProcessExtensions.cs +++ b/src/Microsoft.Tye.Core/ProcessExtensions.cs @@ -109,13 +109,13 @@ namespace Microsoft.Tye var process = System.Diagnostics.Process.Start(startInfo); stdout = null; - if (process.WaitForExit((int)timeout.TotalMilliseconds)) + if (process?.WaitForExit((int)timeout.TotalMilliseconds) == true) { - stdout = process.StandardOutput.ReadToEnd(); + stdout = process?.StandardOutput.ReadToEnd(); } else { - process.Kill(); + process?.Kill(); } } } diff --git a/src/Microsoft.Tye.Core/ProcessUtil.cs b/src/Microsoft.Tye.Core/ProcessUtil.cs index 4781c8fe..204e2fae 100644 --- a/src/Microsoft.Tye.Core/ProcessUtil.cs +++ b/src/Microsoft.Tye.Core/ProcessUtil.cs @@ -52,7 +52,7 @@ namespace Microsoft.Tye { foreach (var kvp in environmentVariables) { - process.StartInfo.Environment.Add(kvp); + process.StartInfo.Environment.Add(kvp!); } } diff --git a/src/Microsoft.Tye.Core/ProjectReader.cs b/src/Microsoft.Tye.Core/ProjectReader.cs index 421b7211..af5076c5 100644 --- a/src/Microsoft.Tye.Core/ProjectReader.cs +++ b/src/Microsoft.Tye.Core/ProjectReader.cs @@ -254,9 +254,9 @@ namespace Microsoft.Tye output.WriteDebugLine($"IntermediateOutputPath={project.IntermediateOutputPath}"); // Normalize directories to their absolute paths - project.IntermediateOutputPath = Path.Combine(project.ProjectFile.DirectoryName, NormalizePath(project.IntermediateOutputPath)); - project.TargetPath = Path.Combine(project.ProjectFile.DirectoryName, NormalizePath(project.TargetPath)); - project.PublishDir = Path.Combine(project.ProjectFile.DirectoryName, NormalizePath(project.PublishDir)); + project.IntermediateOutputPath = Path.Combine(project.ProjectFile.DirectoryName!, NormalizePath(project.IntermediateOutputPath)); + project.TargetPath = Path.Combine(project.ProjectFile.DirectoryName!, NormalizePath(project.TargetPath)); + project.PublishDir = Path.Combine(project.ProjectFile.DirectoryName!, NormalizePath(project.PublishDir)); var targetFramework = projectInstance.GetPropertyValue("TargetFramework"); project.TargetFramework = targetFramework; diff --git a/src/Microsoft.Tye.Core/StandardOptions.cs b/src/Microsoft.Tye.Core/StandardOptions.cs index b59a00b3..6d3ef878 100644 --- a/src/Microsoft.Tye.Core/StandardOptions.cs +++ b/src/Microsoft.Tye.Core/StandardOptions.cs @@ -182,7 +182,7 @@ namespace Microsoft.Tye { if (TryFindProjectFile(token, out var filePath, out var errorMessage)) { - return new FileInfo(filePath); + return new FileInfo(filePath!); } else { diff --git a/src/Microsoft.Tye.Extensions/Dapr/DaprExtension.cs b/src/Microsoft.Tye.Extensions/Dapr/DaprExtension.cs index 2b8c6edf..d185e5f5 100644 --- a/src/Microsoft.Tye.Extensions/Dapr/DaprExtension.cs +++ b/src/Microsoft.Tye.Extensions/Dapr/DaprExtension.cs @@ -55,7 +55,7 @@ namespace Microsoft.Tye.Extensions.Dapr // we'll assume the filename and config name are the same. if (config.Data.TryGetValue("config", out var obj) && obj?.ToString() is string daprConfig) { - var configFile = Path.Combine(context.Application.Source.DirectoryName, "components", $"{daprConfig}.yaml"); + var configFile = Path.Combine(context.Application.Source.DirectoryName!, "components", $"{daprConfig}.yaml"); if (File.Exists(configFile)) { proxy.Args += $" -config \"{configFile}\""; diff --git a/src/Microsoft.Tye.Extensions/Microsoft.Tye.Extensions.csproj b/src/Microsoft.Tye.Extensions/Microsoft.Tye.Extensions.csproj index 0ca4ab55..979268cd 100644 --- a/src/Microsoft.Tye.Extensions/Microsoft.Tye.Extensions.csproj +++ b/src/Microsoft.Tye.Extensions/Microsoft.Tye.Extensions.csproj @@ -1,7 +1,7 @@ - netcoreapp3.1 + net5.0 diff --git a/src/Microsoft.Tye.Hosting.Diagnostics/Logging/LoggerException.cs b/src/Microsoft.Tye.Hosting.Diagnostics/Logging/LoggerException.cs index 31327104..07844261 100644 --- a/src/Microsoft.Tye.Hosting.Diagnostics/Logging/LoggerException.cs +++ b/src/Microsoft.Tye.Hosting.Diagnostics/Logging/LoggerException.cs @@ -29,18 +29,18 @@ namespace Microsoft.Tye.Hosting.Diagnostics.Logging info.AddValue("RemoteStackTraceString", StackTrace, typeof(string)); // Do not rename (binary serialization) info.AddValue("RemoteStackIndex", 0, typeof(int)); // Do not rename (binary serialization) info.AddValue("ExceptionMethod", null, typeof(string)); // Do not rename (binary serialization) - info.AddValue("HResult", int.Parse(_exceptionMessage.GetProperty("HResult").GetString())); // Do not rename (binary serialization) + info.AddValue("HResult", int.Parse(_exceptionMessage.GetProperty("HResult").GetString()!)); // Do not rename (binary serialization) info.AddValue("Source", Source, typeof(string)); // Do not rename (binary serialization info.AddValue("WatsonBuckets", null, typeof(byte[])); // Do not rename (binary serialization) } - public override string Message => _exceptionMessage.GetProperty("Message").GetString(); + public override string Message => _exceptionMessage.GetProperty("Message").GetString()!; - public override string StackTrace => _exceptionMessage.GetProperty("VerboseMessage").GetString(); + public override string StackTrace => _exceptionMessage.GetProperty("VerboseMessage").GetString()!; public override string ToString() { - return _exceptionMessage.GetProperty("VerboseMessage").GetString(); + return _exceptionMessage.GetProperty("VerboseMessage").GetString()!; } } } diff --git a/src/Microsoft.Tye.Hosting.Diagnostics/LoggingSink.cs b/src/Microsoft.Tye.Hosting.Diagnostics/LoggingSink.cs index 01e0de22..5434224a 100644 --- a/src/Microsoft.Tye.Hosting.Diagnostics/LoggingSink.cs +++ b/src/Microsoft.Tye.Hosting.Diagnostics/LoggingSink.cs @@ -112,7 +112,7 @@ namespace Microsoft.Tye.Hosting.Diagnostics if (message.TryGetProperty("{OriginalFormat}", out var formatElement)) { var formatString = formatElement.GetString(); - var formatter = new LogValuesFormatter(formatString); + var formatter = new LogValuesFormatter(formatString!); var args = new object[formatter.ValueNames.Count]; for (var i = 0; i < args.Length; i++) { @@ -126,7 +126,7 @@ namespace Microsoft.Tye.Hosting.Diagnostics break; } - args[i] = argValue.GetString(); + args[i] = argValue.GetString()!; } logger.Log(logLevel, new EventId(eventId, eventName), exception, formatString, args); diff --git a/src/Microsoft.Tye.Hosting.Diagnostics/Microsoft.Tye.Hosting.Diagnostics.csproj b/src/Microsoft.Tye.Hosting.Diagnostics/Microsoft.Tye.Hosting.Diagnostics.csproj index 3b0db741..d0f0afb5 100644 --- a/src/Microsoft.Tye.Hosting.Diagnostics/Microsoft.Tye.Hosting.Diagnostics.csproj +++ b/src/Microsoft.Tye.Hosting.Diagnostics/Microsoft.Tye.Hosting.Diagnostics.csproj @@ -1,7 +1,7 @@  - netcoreapp3.1 + net5.0 Diagnostics collector and exporter for .NET Core applications. Microsoft.Tye.Hosting.Diagnostics Microsoft.Tye.Hosting.Diagnostics diff --git a/src/Microsoft.Tye.Hosting/Microsoft.Tye.Hosting.csproj b/src/Microsoft.Tye.Hosting/Microsoft.Tye.Hosting.csproj index 7b22cd00..97a6f467 100644 --- a/src/Microsoft.Tye.Hosting/Microsoft.Tye.Hosting.csproj +++ b/src/Microsoft.Tye.Hosting/Microsoft.Tye.Hosting.csproj @@ -2,7 +2,7 @@ Library - netcoreapp3.1 + net5.0 Orchestration host APIs. Microsoft.Tye.Hosting Microsoft.Tye.Hosting diff --git a/src/Microsoft.Tye.Hosting/Model/Application.cs b/src/Microsoft.Tye.Hosting/Model/Application.cs index ad8cf67b..7430a0c3 100644 --- a/src/Microsoft.Tye.Hosting/Model/Application.cs +++ b/src/Microsoft.Tye.Hosting/Model/Application.cs @@ -15,7 +15,7 @@ namespace Microsoft.Tye.Hosting.Model public Application(FileInfo source, Dictionary services) { Source = source.FullName; - ContextDirectory = source.DirectoryName; + ContextDirectory = source.DirectoryName!; Services = services; } diff --git a/src/Microsoft.Tye.Hosting/PortAssigner.cs b/src/Microsoft.Tye.Hosting/PortAssigner.cs index 2e188580..01692c3a 100644 --- a/src/Microsoft.Tye.Hosting/PortAssigner.cs +++ b/src/Microsoft.Tye.Hosting/PortAssigner.cs @@ -39,7 +39,7 @@ namespace Microsoft.Tye.Hosting // reuse being enabled by default by the OS. using var socket = new Socket(AddressFamily.InterNetwork, SocketType.Stream, ProtocolType.Tcp); socket.Bind(new IPEndPoint(IPAddress.Loopback, 0)); - return ((IPEndPoint)socket.LocalEndPoint).Port; + return ((IPEndPoint)socket.LocalEndPoint!).Port; } foreach (var binding in service.Description.Bindings) diff --git a/src/Microsoft.Tye.Hosting/ProcessRunner.cs b/src/Microsoft.Tye.Hosting/ProcessRunner.cs index b3f73438..29465301 100644 --- a/src/Microsoft.Tye.Hosting/ProcessRunner.cs +++ b/src/Microsoft.Tye.Hosting/ProcessRunner.cs @@ -61,7 +61,7 @@ namespace Microsoft.Tye.Hosting if (serviceDescription.RunInfo is ProjectRunInfo project) { path = project.RunCommand; - workingDirectory = project.ProjectFile.Directory.FullName; + workingDirectory = project.ProjectFile.Directory!.FullName; args = project.Args == null ? project.RunArguments : project.RunArguments + " " + project.Args; buildProperties = project.BuildProperties.Aggregate(string.Empty, (current, property) => current + $";{property.Key}={property.Value}").TrimStart(';'); @@ -211,7 +211,7 @@ namespace Microsoft.Tye.Hosting application.PopulateEnvironment(service, (k, v) => environment[k] = v); - if (_options.DebugMode && (_options.DebugAllServices || _options.ServicesToDebug.Contains(serviceName, StringComparer.OrdinalIgnoreCase))) + if (_options.DebugMode && (_options.DebugAllServices || _options.ServicesToDebug!.Contains(serviceName, StringComparer.OrdinalIgnoreCase))) { environment["DOTNET_STARTUP_HOOKS"] = typeof(Hosting.Runtime.HostingRuntimeHelpers).Assembly.Location; } @@ -517,7 +517,7 @@ namespace Microsoft.Tye.Hosting private static string GetEntryPointFilePath() { using var process = Process.GetCurrentProcess(); - return process.MainModule.FileName; + return process.MainModule!.FileName!; } private class ProcessInfo diff --git a/src/Microsoft.Tye.Hosting/ReplicaRegistry.cs b/src/Microsoft.Tye.Hosting/ReplicaRegistry.cs index 53efeef9..e31661c5 100644 --- a/src/Microsoft.Tye.Hosting/ReplicaRegistry.cs +++ b/src/Microsoft.Tye.Hosting/ReplicaRegistry.cs @@ -87,7 +87,7 @@ namespace Microsoft.Tye.Hosting return events.Where(e => !string.IsNullOrEmpty(e.Trim())) .Select(e => JsonSerializer.Deserialize>(e)) - .ToList(); + .ToList()!; } private object GetLockForStore(string storeName) diff --git a/src/Microsoft.Tye.Hosting/TokenReplacement.cs b/src/Microsoft.Tye.Hosting/TokenReplacement.cs index 81fdd72c..97c78864 100644 --- a/src/Microsoft.Tye.Hosting/TokenReplacement.cs +++ b/src/Microsoft.Tye.Hosting/TokenReplacement.cs @@ -79,12 +79,12 @@ namespace Microsoft.Tye.Hosting } else if (keys.Length == 3 && keys[0] == "service") { - binding = bindings.FirstOrDefault(b => b.Service == keys[1] && b.Name == null); + binding = bindings.FirstOrDefault(b => b.Service == keys[1] && b.Name == null)!; return GetValueFromBinding(binding, keys[2]); } else if (keys.Length == 4 && keys[0] == "service") { - binding = bindings.FirstOrDefault(b => b.Service == keys[1] && b.Name == keys[2]); + binding = bindings.FirstOrDefault(b => b.Service == keys[1] && b.Name == keys[2])!; return GetValueFromBinding(binding, keys[3]); } diff --git a/src/tye-diag-agent/tye-diag-agent.csproj b/src/tye-diag-agent/tye-diag-agent.csproj index 6278740f..53b67503 100644 --- a/src/tye-diag-agent/tye-diag-agent.csproj +++ b/src/tye-diag-agent/tye-diag-agent.csproj @@ -1,7 +1,7 @@ - netcoreapp3.1 + net5.0 Microsoft.Tye diff --git a/src/tye/ApplicationBuilderExtensions.cs b/src/tye/ApplicationBuilderExtensions.cs index 7e9c529c..a8305f0f 100644 --- a/src/tye/ApplicationBuilderExtensions.cs +++ b/src/tye/ApplicationBuilderExtensions.cs @@ -64,7 +64,7 @@ namespace Microsoft.Tye } else { - dockerRunInfo.DockerFileContext = new FileInfo(dockerRunInfo.DockerFile.DirectoryName); + dockerRunInfo.DockerFileContext = new FileInfo(dockerRunInfo.DockerFile.DirectoryName!); } } diff --git a/src/tye/CommonArguments.cs b/src/tye/CommonArguments.cs index 1a24d285..27e74a15 100644 --- a/src/tye/CommonArguments.cs +++ b/src/tye/CommonArguments.cs @@ -60,7 +60,7 @@ namespace Microsoft.Tye { if (ConfigFileFinder.TryFindSupportedFile(token, out var filePath, out var errorMessage)) { - return new FileInfo(filePath); + return new FileInfo(filePath!); } else if (required) { diff --git a/src/tye/InitHost.cs b/src/tye/InitHost.cs index 43634028..24e89a28 100644 --- a/src/tye/InitHost.cs +++ b/src/tye/InitHost.cs @@ -72,11 +72,11 @@ services: service.Bindings = null!; service.Configuration = null!; service.Volumes = null!; - service.Project = service.Project!.Substring(directory.FullName.Length).TrimStart('/'); + service.Project = service.Project!.Substring(directory!.FullName.Length).TrimStart('/'); } // If the input file is a sln/project then place the config next to it - outputFilePath = Path.Combine(directory.FullName, "tye.yaml"); + outputFilePath = Path.Combine(directory!.FullName, "tye.yaml"); } else { @@ -99,7 +99,7 @@ services: private static void ThrowIfTyeFilePresent(FileInfo? path, string yml) { - var tyeYaml = Path.Combine(path!.DirectoryName, yml); + var tyeYaml = Path.Combine(path!.DirectoryName!, yml); if (File.Exists(tyeYaml)) { throw new CommandException($"File '{tyeYaml}' already exists. Use --force to override the {yml} file if desired."); diff --git a/src/tye/tye.csproj b/src/tye/tye.csproj index b1c5e541..27957d32 100644 --- a/src/tye/tye.csproj +++ b/src/tye/tye.csproj @@ -2,7 +2,7 @@ Exe - netcoreapp3.1 + net5.0 Microsoft.Tye tye Microsoft.Tye diff --git a/test/E2ETest/HealthCheckTests.cs b/test/E2ETest/HealthCheckTests.cs index 288e48c6..3b70f539 100644 --- a/test/E2ETest/HealthCheckTests.cs +++ b/test/E2ETest/HealthCheckTests.cs @@ -254,7 +254,7 @@ namespace E2ETest // we assume that proxy will continue sending http request to the same replica var randomReplicaPortRes1 = await _client.GetAsync($"http://localhost:{host.Application.Services["health-proxy"].Description.Bindings.First().Port}/ports"); - var randomReplicaPort1 = JsonSerializer.Deserialize(await randomReplicaPortRes1.Content.ReadAsStringAsync())[0]; + var randomReplicaPort1 = JsonSerializer.Deserialize(await randomReplicaPortRes1.Content.ReadAsStringAsync())![0]; var randomReplica1 = replicasToBecomeReady.First(r => r.Bindings.Any(b => b.Port == randomReplicaPort1)); await DoOperationAndWaitForReplicasToChangeState(host, ReplicaState.Healthy, 1, new[] { randomReplica1.Name }.ToHashSet(), null, TimeSpan.Zero, async _ => @@ -263,7 +263,7 @@ namespace E2ETest }); var randomReplicaPortRes2 = await _client.GetAsync($"http://localhost:{host.Application.Services["health-proxy"].Description.Bindings.First().Port}/ports"); - var randomReplicaPort2 = JsonSerializer.Deserialize(await randomReplicaPortRes2.Content.ReadAsStringAsync())[0]; + var randomReplicaPort2 = JsonSerializer.Deserialize(await randomReplicaPortRes2.Content.ReadAsStringAsync())![0]; var randomReplica2 = replicasToBecomeReady.First(r => r.Bindings.Any(b => b.Port == randomReplicaPort2)); Assert.NotEqual(randomReplicaPort1, randomReplicaPort2); @@ -288,7 +288,7 @@ namespace E2ETest }); var randomReplicaPortRes3 = await _client.GetAsync($"http://localhost:{host.Application.Services["health-proxy"].Description.Bindings.First().Port}/ports"); - var randomReplicaPort3 = JsonSerializer.Deserialize(await randomReplicaPortRes3.Content.ReadAsStringAsync())[0]; + var randomReplicaPort3 = JsonSerializer.Deserialize(await randomReplicaPortRes3.Content.ReadAsStringAsync())![0]; Assert.Equal(randomReplicaPort3, randomReplicaPort2); } @@ -369,14 +369,14 @@ namespace E2ETest Assert.True(res.IsSuccessStatusCode); var headers = JsonSerializer.Deserialize>(await res.Content.ReadAsStringAsync()); - Assert.Equal("value1", headers["name1"]); + Assert.Equal("value1", headers!["name1"]); Assert.Equal("value2", headers["name2"]); res = await _client.GetAsync($"http://localhost:{host.Application.Services["health-all"].Description.Bindings.First().Port}/readinessHeaders"); Assert.True(res.IsSuccessStatusCode); headers = JsonSerializer.Deserialize>(await res.Content.ReadAsStringAsync()); - Assert.Equal("value3", headers["name3"]); + Assert.Equal("value3", headers!["name3"]); Assert.Equal("value4", headers["name4"]); } @@ -404,7 +404,7 @@ namespace E2ETest query.Add("readyDelay=" + readyDelay.Value); } - await _client.GetAsync($"http://localhost:{replica.Ports.First()}/set?" + string.Join("&", query)); + await _client.GetAsync($"http://localhost:{replica.Ports!.First()}/set?" + string.Join("&", query)); } private async Task ProbeNumberOfUniqueReplicas(string url) diff --git a/test/E2ETest/Microsoft.Tye.E2ETests.csproj b/test/E2ETest/Microsoft.Tye.E2ETests.csproj index 1caa1af9..4e6cb53e 100644 --- a/test/E2ETest/Microsoft.Tye.E2ETests.csproj +++ b/test/E2ETest/Microsoft.Tye.E2ETests.csproj @@ -2,7 +2,7 @@ - netcoreapp3.1 + net5.0 Microsoft.Tye.E2ETest true true @@ -14,8 +14,8 @@ The Microsoft.Build.Locator package takes care of dynamically loading these assemblies at runtime. We don't need/want to ship them, just to have them as references. --> - - + + diff --git a/test/E2ETest/TyeRunTests.cs b/test/E2ETest/TyeRunTests.cs index 4e982389..fb346ac2 100644 --- a/test/E2ETest/TyeRunTests.cs +++ b/test/E2ETest/TyeRunTests.cs @@ -452,7 +452,7 @@ namespace E2ETest Assert.NotNull(service); - Assert.Equal(dockerNetwork, service.Replicas.FirstOrDefault().Value.DockerNetwork); + Assert.Equal(dockerNetwork, service!.Replicas!.FirstOrDefault().Value.DockerNetwork); } }); } @@ -503,7 +503,7 @@ namespace E2ETest Assert.NotNull(service); - Assert.NotEqual(dockerNetwork, service.Replicas.FirstOrDefault().Value.DockerNetwork); + Assert.NotEqual(dockerNetwork, service!.Replicas!.FirstOrDefault().Value.DockerNetwork); } }); } @@ -627,7 +627,7 @@ namespace E2ETest var responseContent = JsonSerializer.Deserialize>(await response.Content.ReadAsStringAsync()); - Assert.Equal("some content", responseContent["content"]); + Assert.Equal("some content", responseContent!["content"]); Assert.Equal("?key1=value1&key2=value2", responseContent["query"]); }); } @@ -858,7 +858,7 @@ services: { var serviceResult = await client.GetStringAsync($"{uri}api/v1/services/{serviceName}"); var service = JsonSerializer.Deserialize(serviceResult, _options); - var binding = service.Description!.Bindings.Where(b => b.Protocol == "http").Single(); + var binding = service!.Description!.Bindings!.Where(b => b.Protocol == "http").Single(); return $"{binding.Protocol ?? "http"}://localhost:{binding.Port}"; } diff --git a/test/Microsoft.Tye.Extensions.Configuration.Tests/Microsoft.Tye.Extensions.Configuration.Tests.csproj b/test/Microsoft.Tye.Extensions.Configuration.Tests/Microsoft.Tye.Extensions.Configuration.Tests.csproj index dffc8b0d..16490ce8 100644 --- a/test/Microsoft.Tye.Extensions.Configuration.Tests/Microsoft.Tye.Extensions.Configuration.Tests.csproj +++ b/test/Microsoft.Tye.Extensions.Configuration.Tests/Microsoft.Tye.Extensions.Configuration.Tests.csproj @@ -1,7 +1,7 @@ - netcoreapp3.1 + net5.0 true diff --git a/test/Test.Infrastructure/Test.Infrastructure.csproj b/test/Test.Infrastructure/Test.Infrastructure.csproj index d114d434..9a82f29a 100644 --- a/test/Test.Infrastructure/Test.Infrastructure.csproj +++ b/test/Test.Infrastructure/Test.Infrastructure.csproj @@ -1,16 +1,19 @@  - netcoreapp3.1 + net5.0 false XUnit disable + false + + diff --git a/test/UnitTests/Microsoft.Tye.UnitTests.csproj b/test/UnitTests/Microsoft.Tye.UnitTests.csproj index cda06b56..72487d8b 100644 --- a/test/UnitTests/Microsoft.Tye.UnitTests.csproj +++ b/test/UnitTests/Microsoft.Tye.UnitTests.csproj @@ -1,7 +1,7 @@ - netcoreapp3.1 + net5.0 Microsoft.Tye.UnitTests true true