Bump Arcade to 5.0.0-beta.20510.1

eng/Version.Details.xml

@@ -5,14 +5,14 @@
   </ProductDependencies>
 
   <ToolsetDependencies>
-    <Dependency Name="Microsoft.DotNet.Arcade.Sdk" Version="5.0.0-beta.20426.1">
+    <Dependency Name="Microsoft.DotNet.Arcade.Sdk" Version="5.0.0-beta.20510.1">
       <Uri>https://github.com/dotnet/arcade</Uri>
-      <Sha>3b720900ac55e9cc3675584c41da6460d6193f37</Sha>
+      <Sha>6813f5aa511a7a4498fa217a54219b5704a01f83</Sha>
     </Dependency>
 
-    <Dependency Name="Microsoft.DotNet.Helix.Sdk" Version="5.0.0-beta.20426.1">
+    <Dependency Name="Microsoft.DotNet.Helix.Sdk" Version="5.0.0-beta.20510.1">
       <Uri>https://github.com/dotnet/arcade</Uri>
-      <Sha>3b720900ac55e9cc3675584c41da6460d6193f37</Sha>
+      <Sha>6813f5aa511a7a4498fa217a54219b5704a01f83</Sha>
     </Dependency>
   </ToolsetDependencies>
 

eng/common/internal/Directory.Build.props

@@ -1,4 +1,4 @@
-<!-- Licensed to the .NET Foundation under one or more agreements. The .NET Foundation licenses this file to you under the MIT license. See the LICENSE file in the project root for more information. -->
+<!-- Licensed to the .NET Foundation under one or more agreements. The .NET Foundation licenses this file to you under the MIT license. -->
 <Project>
   <Import Project="Sdk.props" Sdk="Microsoft.DotNet.Arcade.Sdk" />
 </Project>

eng/common/performance/performance-setup.sh

@@ -201,7 +201,6 @@ if [[ "$internal" == true ]]; then
 fi
 
 if [[ "$mono_dotnet" != "" ]] && [[ "$monointerpreter" == "false" ]]; then
-    configurations="$configurations LLVM=$llvm MonoInterpreter=$monointerpreter MonoAOT=$monoaot"
     extra_benchmark_dotnet_arguments="$extra_benchmark_dotnet_arguments --category-exclusion-filter NoMono"
 fi
 
@@ -211,6 +210,7 @@ if [[ "$wasm_runtime_loc" != "" ]]; then
 fi
 
 if [[ "$mono_dotnet" != "" ]] && [[ "$monointerpreter" == "true" ]]; then
+    configurations="$configurations LLVM=$llvm MonoInterpreter=$monointerpreter MonoAOT=$monoaot"
     extra_benchmark_dotnet_arguments="$extra_benchmark_dotnet_arguments --category-exclusion-filter NoInterpreter NoMono"
 fi
 

eng/common/post-build/publish-using-darc.ps1

@@ -8,7 +8,7 @@ param(
   [Parameter(Mandatory=$false)][string] $EnableSourceLinkValidation,
   [Parameter(Mandatory=$false)][string] $EnableSigningValidation,
   [Parameter(Mandatory=$false)][string] $EnableNugetValidation,
-  [Parameter(Mandatory=$true)][string] $PublishInstallersAndChecksums,
+  [Parameter(Mandatory=$false)][string] $PublishInstallersAndChecksums,
   [Parameter(Mandatory=$false)][string] $ArtifactsPublishingAdditionalParameters,
   [Parameter(Mandatory=$false)][string] $SigningValidationAdditionalParameters
 )
@@ -16,7 +16,7 @@ param(
 try {
   . $PSScriptRoot\post-build-utils.ps1
   # Hard coding darc version till the next arcade-services roll out, cos this version has required API changes for darc add-build-to-channel
-  . $PSScriptRoot\..\darc-init.ps1 -darcVersion "1.1.0-beta.20418.1"
+  $darc = Get-Darc "1.1.0-beta.20418.1"
 
   $optionalParams = [System.Collections.ArrayList]::new()
 
@@ -29,7 +29,7 @@ try {
     $optionalParams.Add("--no-wait") | Out-Null
   }
 
-  if ("true" -eq $PublishInstallersAndChecksums) {
+  if ("false" -ne $PublishInstallersAndChecksums) {
     $optionalParams.Add("--publish-installers-and-checksums") | Out-Null
   }
 
@@ -50,7 +50,7 @@ try {
     }
   }
 
-  & darc add-build-to-channel `
+  & $darc add-build-to-channel `
   --id $buildId `
   --publishing-infra-version $PublishingInfraVersion `
   --default-channels `

eng/common/post-build/sourcelink-validation.ps1

@@ -144,11 +144,27 @@ $ValidatePackage = {
 
   if ($FailedFiles -eq 0) {
     Write-Host 'Passed.'
-    return 0
+    return [pscustomobject]@{
+      result = 0
+      packagePath = $PackagePath
+    }
   }
   else {
     Write-PipelineTelemetryError -Category 'SourceLink' -Message "$PackagePath has broken SourceLink links."
-    return 1
+    return [pscustomobject]@{
+      result = 1
+      packagePath = $PackagePath
+    }
+  }
+}
+
+function CheckJobResult(
+    $result, 
+    $packagePath,
+    [ref]$ValidationFailures) {
+  if ($jobResult.result -ne '0') {
+    Write-PipelineTelemetryError -Category 'SourceLink' -Message "$packagePath has broken SourceLink links."
+    $ValidationFailures.Value++
   }
 }
 
@@ -211,10 +227,8 @@ function ValidateSourceLinkLinks {
       }
 
       foreach ($Job in @(Get-Job -State 'Completed')) {
-        $jobResult = Receive-Job -Id $Job.Id
-        if ($jobResult -ne '0') {
-          $ValidationFailures++
-        }
+        $jobResult = Wait-Job -Id $Job.Id | Receive-Job
+        CheckJobResult $jobResult.result $jobResult.packagePath ([ref]$ValidationFailures)
         Remove-Job -Id $Job.Id
       }
     }

eng/common/post-build/symbols-validation.ps1

@@ -141,11 +141,6 @@ $CountMissingSymbols = {
   if ($using:Clean) {
     Remove-Item $ExtractPath -Recurse -Force
   }
-
-  if ($MissingSymbols -ne 0)
-  {
-    Write-PipelineTelemetryError -Category 'CheckSymbols' -Message "Missing symbols for $MissingSymbols modules in the package $PackagePath"
-  }
   
   Pop-Location
 
@@ -165,6 +160,7 @@ function CheckJobResult(
     $DupedSymbols.Value++
   } 
   elseif ($jobResult.result -ne '0') {
+    Write-PipelineTelemetryError -Category 'CheckSymbols' -Message "Missing symbols for $result modules in the package $packagePath"
     $TotalFailures.Value++
   }
 }
@@ -201,7 +197,6 @@ function CheckSymbolsAvailable {
       Start-Job -ScriptBlock $CountMissingSymbols -ArgumentList $FullName | Out-Null
 
       $NumJobs = @(Get-Job -State 'Running').Count
-      Write-Host $NumJobs
 
       while ($NumJobs -ge $MaxParallelJobs) {
         Write-Host "There are $NumJobs validation jobs running right now. Waiting $SecondsBetweenLoadChecks seconds to check again."

eng/common/sdk-task.ps1

@@ -64,7 +64,7 @@ try {
       $GlobalJson.tools | Add-Member -Name "vs" -Value (ConvertFrom-Json "{ `"version`": `"16.5`" }") -MemberType NoteProperty
     }
     if( -not ($GlobalJson.tools.PSObject.Properties.Name -match "xcopy-msbuild" )) {
-      $GlobalJson.tools | Add-Member -Name "xcopy-msbuild" -Value "16.5.0-alpha" -MemberType NoteProperty
+      $GlobalJson.tools | Add-Member -Name "xcopy-msbuild" -Value "16.8.0-preview3" -MemberType NoteProperty
     }
     if ($GlobalJson.tools."xcopy-msbuild".Trim() -ine "none") {
         $xcopyMSBuildToolsFolder = InitializeXCopyMSBuild $GlobalJson.tools."xcopy-msbuild" -install $true

eng/common/templates/job/job.yml

@@ -204,7 +204,7 @@ jobs:
 
   - ${{ if eq(parameters.enablePublishTestResults, 'true') }}:
     - task: PublishTestResults@2
-      displayName: Publish Test Results
+      displayName: Publish XUnit Test Results
       inputs:
         testResultsFormat: 'xUnit'
         testResultsFiles: '*.xml' 
@@ -213,6 +213,16 @@ jobs:
         mergeTestResults: ${{ parameters.mergeTestResults }}
       continueOnError: true
       condition: always()
+    - task: PublishTestResults@2
+      displayName: Publish TRX Test Results
+      inputs:
+        testResultsFormat: 'VSTest'
+        testResultsFiles: '*.trx' 
+        searchFolder: '$(Build.SourcesDirectory)/artifacts/TestResults/$(_BuildConfig)'
+        testRunTitle: ${{ coalesce(parameters.testRunTitle, parameters.name, '$(System.JobName)') }}-trx
+        mergeTestResults: ${{ parameters.mergeTestResults }}
+      continueOnError: true
+      condition: always()
     
   - ${{ if and(eq(parameters.enablePublishBuildAssets, true), ne(parameters.enablePublishUsingPipelines, 'true'), eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}:
     - task: CopyFiles@2

eng/common/templates/job/source-build.yml

@@ -0,0 +1,49 @@
+parameters:
+  # This template adds arcade-powered source-build to CI. The template produces a server job with a
+  # default ID 'Source_Build_Complete' to put in a dependency list if necessary.
+
+  # Specifies the prefix for source-build jobs added to pipeline. Use this if disambiguation needed.
+  jobNamePrefix: 'Source_Build'
+
+  # Defines the platform on which to run the job. By default, a linux-x64 machine, suitable for
+  # managed-only repositories. This is an object with these properties:
+  #
+  # name: ''
+  #   The name of the job. This is included in the job ID.
+  # targetRID: ''
+  #   The name of the target RID to use, instead of the one auto-detected by Arcade.
+  # nonPortable: false
+  #   Enables non-portable mode. This means a more specific RID (e.g. fedora.32-x64 rather than
+  #   linux-x64), and compiling against distro-provided packages rather than portable ones.
+  # container: ''
+  #   A container to use. Runs in docker.
+  # pool: {}
+  #   A pool to use. Runs directly on an agent.
+  # buildScript: ''
+  #   Specifies the build script to invoke to perform the build in the repo. The default
+  #   './build.sh' should work for typical Arcade repositories, but this is customizable for
+  #   difficult situations.
+  # jobProperties: {}
+  #   A list of job properties to inject at the top level, for potential extensibility beyond
+  #   container and pool.
+  platform: {}
+
+jobs:
+- job: ${{ parameters.jobNamePrefix }}_${{ parameters.platform.name }}
+  displayName: Source-Build (${{ parameters.platform.name }})
+
+  ${{ each property in parameters.platform.jobProperties }}:
+    ${{ property.key }}: ${{ property.value }}
+
+  ${{ if ne(parameters.platform.container, '') }}:
+    container: ${{ parameters.platform.container }}
+  ${{ if ne(parameters.platform.pool, '') }}:
+    pool: ${{ parameters.platform.pool }}
+
+  workspace:
+    clean: all
+
+  steps:
+  - template: /eng/common/templates/steps/source-build.yml
+    parameters:
+      platform: ${{ parameters.platform }}

eng/common/templates/jobs/jobs.yml

@@ -24,6 +24,13 @@ parameters:
   #           if 'true', the build won't run any of the internal only steps, even if it is running in non-public projects.
   runAsPublic: false
 
+  # Optional: Enable running the source-build jobs to build repo from source
+  runSourceBuild: false
+
+  # Optional: Parameters for source-build template.
+  #           See /eng/common/templates/jobs/source-build.yml for options
+  sourceBuildParameters: []
+
 # Internal resources (telemetry, microbuild) can only be accessed from non-public projects,
 # and some (Microbuild) should only be applied to non-PR cases for internal builds.
 
@@ -43,6 +50,13 @@ jobs:
 
       name: ${{ job.job }}
 
+- ${{ if eq(parameters.runSourceBuild, true) }}:
+  - template: /eng/common/templates/jobs/source-build.yml
+    parameters:
+      allCompletedJobId: Source_Build_Complete
+      ${{ each parameter in parameters.sourceBuildParameters }}:
+        ${{ parameter.key }}: ${{ parameter.value }}
+
 - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}:
   - ${{ if or(eq(parameters.enablePublishBuildAssets, true), eq(parameters.artifacts.publish.manifests, 'true'), ne(parameters.artifacts.publish.manifests, '')) }}:
     - template: ../job/publish-build-assets.yml
@@ -55,6 +69,8 @@ jobs:
         - ${{ if eq(parameters.publishBuildAssetsDependsOn, '') }}:
           - ${{ each job in parameters.jobs }}:
             - ${{ job.job }}
+        - ${{ if eq(parameters.runSourceBuild, true) }}:
+          - Source_Build_Complete
         pool:
           vmImage: vs2017-win2016
         runAsPublic: ${{ parameters.runAsPublic }}

eng/common/templates/jobs/source-build.yml

@@ -0,0 +1,48 @@
+parameters:
+  # This template adds arcade-powered source-build to CI. A job is created for each platform, as
+  # well as an optional server job that completes when all platform jobs complete.
+
+  # The name of the "join" job for all source-build platforms. If set to empty string, the job is
+  # not included. Existing repo pipelines can use this job depend on all source-build jobs
+  # completing without maintaining a separate list of every single job ID: just depend on this one
+  # server job. By default, not included. Recommended name if used: 'Source_Build_Complete'.
+  allCompletedJobId: ''
+
+  # See /eng/common/templates/job/source-build.yml
+  jobNamePrefix: 'Source_Build'
+
+  # If changed to true, causes this template to include the default platform for a managed-only
+  # repo. The exact Docker image used for this build will be provided by Arcade. This has some risk,
+  # but since the repo is supposed to be managed-only, the risk should be very low.
+  includeDefaultManagedPlatform: false
+  defaultManagedPlatform:
+    name: 'Managed'
+    container: 'mcr.microsoft.com/dotnet-buildtools/prereqs:centos-7-3e800f1-20190501005343'
+
+  # Defines the platforms on which to run build jobs. One job is created for each platform, and the
+  # object in this array is sent to the job template as 'platform'.
+  platforms: []
+
+jobs:
+
+- ${{ if ne(parameters.allCompletedJobId, '') }}:
+  - job: ${{ parameters.allCompletedJobId }}
+    displayName: Source-Build Complete
+    pool: server
+    dependsOn:
+    - ${{ each platform in parameters.platforms }}:
+      - ${{ parameters.jobNamePrefix }}_${{ platform.name }}
+    - ${{ if eq(parameters.includeDefaultManagedPlatform, true) }}:
+      - ${{ parameters.jobNamePrefix }}_${{ parameters.defaultManagedPlatform.name }}
+
+- ${{ each platform in parameters.platforms }}:
+  - template: /eng/common/templates/job/source-build.yml
+    parameters:
+      jobNamePrefix: ${{ parameters.jobNamePrefix }}
+      platform: ${{ platform }}
+
+- ${{ if eq(parameters.includeDefaultManagedPlatform, true) }}:
+  - template: /eng/common/templates/job/source-build.yml
+    parameters:
+      jobNamePrefix: ${{ parameters.jobNamePrefix }}
+      platform: ${{ parameters.defaultManagedPlatform }}

eng/common/templates/post-build/channels/generic-internal-channel.yml

@@ -4,7 +4,7 @@ parameters:
   artifactsPublishingAdditionalParameters: ''
   dependsOn:
   - Validate
-  publishInstallersAndChecksums: false
+  publishInstallersAndChecksums: true
   symbolPublishingAdditionalParameters: ''
   stageName: ''
   channelName: ''
@@ -158,7 +158,7 @@ stages:
             /p:BlobBasePath='$(Build.ArtifactStagingDirectory)/BlobArtifacts/'
             /p:PackageBasePath='$(Build.ArtifactStagingDirectory)/PackageArtifacts/'
             /p:Configuration=Release
-            /p:PublishInstallersAndChecksums=true
+            /p:PublishInstallersAndChecksums=${{ parameters.publishInstallersAndChecksums }}
             /p:ChecksumsTargetStaticFeed=$(InternalChecksumsBlobFeedUrl)
             /p:ChecksumsAzureAccountKey=$(InternalChecksumsBlobFeedKey)
             /p:InstallersTargetStaticFeed=$(InternalInstallersBlobFeedUrl)

eng/common/templates/post-build/channels/generic-public-channel.yml

@@ -4,7 +4,7 @@ parameters:
   artifactsPublishingAdditionalParameters: ''
   dependsOn:
   - Validate
-  publishInstallersAndChecksums: false
+  publishInstallersAndChecksums: true
   symbolPublishingAdditionalParameters: ''
   stageName: ''
   channelName: ''

eng/common/templates/post-build/post-build.yml

@@ -19,7 +19,7 @@ parameters:
   enableSigningValidation: true
   enableSymbolValidation: false
   enableNugetValidation: true
-  publishInstallersAndChecksums: false
+  publishInstallersAndChecksums: true
   SDLValidationParameters:
     enable: false
     continueOnError: false
@@ -78,21 +78,22 @@ stages:
       BARBuildId: ${{ parameters.BARBuildId }}
       PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }}
 
-  - job:
-    displayName: Post-build Checks
-    dependsOn: setupMaestroVars
-    variables:
-      - name: TargetChannels
-        value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.TargetChannels'] ]
-    pool:
-      vmImage: 'windows-2019'
-    steps:
-      - task: PowerShell@2
-        displayName: Maestro Channels Consistency
-        inputs:
-          filePath: $(Build.SourcesDirectory)/eng/common/post-build/check-channel-consistency.ps1
-          arguments: -PromoteToChannels "$(TargetChannels)"
-            -AvailableChannelIds ${{parameters.NetEngLatestChannelId}},${{parameters.NetEngValidationChannelId}},${{parameters.NetDev5ChannelId}},${{parameters.NetDev6ChannelId}},${{parameters.GeneralTestingChannelId}},${{parameters.NETCoreToolingDevChannelId}},${{parameters.NETCoreToolingReleaseChannelId}},${{parameters.NETInternalToolingChannelId}},${{parameters.NETCoreExperimentalChannelId}},${{parameters.NetEngServicesIntChannelId}},${{parameters.NetEngServicesProdChannelId}},${{parameters.Net5Preview8ChannelId}},${{parameters.Net5RC1ChannelId}},${{parameters.Net5RC2ChannelId}},${{parameters.NetCoreSDK313xxChannelId}},${{parameters.NetCoreSDK313xxInternalChannelId}},${{parameters.NetCoreSDK314xxChannelId}},${{parameters.NetCoreSDK314xxInternalChannelId}},${{parameters.VS166ChannelId}},${{parameters.VS167ChannelId}},${{parameters.VS168ChannelId}},${{parameters.VSMasterChannelId}}
+  - ${{ if and(le(parameters.publishingInfraVersion, 2), eq(parameters.inline, 'true')) }}:
+    - job:
+      displayName: Post-build Checks
+      dependsOn: setupMaestroVars
+      variables:
+        - name: TargetChannels
+          value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.TargetChannels'] ]
+      pool:
+        vmImage: 'windows-2019'
+      steps:
+        - task: PowerShell@2
+          displayName: Maestro Channels Consistency
+          inputs:
+            filePath: $(Build.SourcesDirectory)/eng/common/post-build/check-channel-consistency.ps1
+            arguments: -PromoteToChannels "$(TargetChannels)"
+              -AvailableChannelIds ${{parameters.NetEngLatestChannelId}},${{parameters.NetEngValidationChannelId}},${{parameters.NetDev5ChannelId}},${{parameters.NetDev6ChannelId}},${{parameters.GeneralTestingChannelId}},${{parameters.NETCoreToolingDevChannelId}},${{parameters.NETCoreToolingReleaseChannelId}},${{parameters.NETInternalToolingChannelId}},${{parameters.NETCoreExperimentalChannelId}},${{parameters.NetEngServicesIntChannelId}},${{parameters.NetEngServicesProdChannelId}},${{parameters.Net5Preview8ChannelId}},${{parameters.Net5RC1ChannelId}},${{parameters.Net5RC2ChannelId}},${{parameters.NetCoreSDK313xxChannelId}},${{parameters.NetCoreSDK313xxInternalChannelId}},${{parameters.NetCoreSDK314xxChannelId}},${{parameters.NetCoreSDK314xxInternalChannelId}},${{parameters.VS166ChannelId}},${{parameters.VS167ChannelId}},${{parameters.VS168ChannelId}},${{parameters.VSMasterChannelId}}
 
   - job:
     displayName: NuGet Validation

eng/common/templates/steps/perf-send-to-helix.yml

@@ -11,7 +11,7 @@ parameters:
   WorkItemDirectory: ''                  # optional -- a payload directory to zip up and send to Helix; requires WorkItemCommand; incompatible with XUnitProjects
   CorrelationPayloadDirectory: ''        # optional -- a directory to zip up and send to Helix as a correlation payload
   IncludeDotNetCli: false                # optional -- true will download a version of the .NET CLI onto the Helix machine as a correlation payload; requires DotNetCliPackageType and DotNetCliVersion
-  DotNetCliPackageType: ''               # optional -- either 'sdk' or 'runtime'; determines whether the sdk or runtime will be sent to Helix; see https://raw.githubusercontent.com/dotnet/core/master/release-notes/releases.json
+  DotNetCliPackageType: ''               # optional -- either 'sdk', 'runtime' or 'aspnetcore-runtime'; determines whether the sdk or runtime will be sent to Helix; see https://raw.githubusercontent.com/dotnet/core/master/release-notes/releases.json
   DotNetCliVersion: ''                   # optional -- version of the CLI to send to Helix; based on this: https://raw.githubusercontent.com/dotnet/core/master/release-notes/releases.json
   EnableXUnitReporter: false             # optional -- true enables XUnit result reporting to Mission Control
   WaitForWorkItemCompletion: true        # optional -- true will make the task wait until work items have been completed and fail the build if work items fail. False is "fire and forget."
@@ -19,34 +19,18 @@ parameters:
   DisplayNamePrefix: 'Send job to Helix' # optional -- rename the beginning of the displayName of the steps in AzDO 
   condition: succeeded()                 # optional -- condition for step to execute; defaults to succeeded()
   continueOnError: false                 # optional -- determines whether to continue the build if the step errors; defaults to false
+  osGroup: ''                            # required -- operating system for the job
             
 
 steps:
-  - powershell: $(Build.SourcesDirectory)\eng\common\msbuild.ps1 $(Build.SourcesDirectory)\eng\common\performance\${{ parameters.ProjectFile }} /restore /t:Test /bl:$(Build.SourcesDirectory)\artifacts\log\$env:BuildConfig\SendToHelix.binlog
-    displayName: ${{ parameters.DisplayNamePrefix }} (Windows)
-    env:
-      BuildConfig: $(_BuildConfig)
-      HelixSource: ${{ parameters.HelixSource }}
-      HelixType: ${{ parameters.HelixType }}
-      HelixBuild: ${{ parameters.HelixBuild }}
-      HelixTargetQueues: ${{ parameters.HelixTargetQueues }}
-      HelixAccessToken: ${{ parameters.HelixAccessToken }}
-      HelixPreCommands: ${{ parameters.HelixPreCommands }}
-      HelixPostCommands: ${{ parameters.HelixPostCommands }}
-      WorkItemDirectory: ${{ parameters.WorkItemDirectory }}
-      CorrelationPayloadDirectory: ${{ parameters.CorrelationPayloadDirectory }}
-      IncludeDotNetCli: ${{ parameters.IncludeDotNetCli }}
-      DotNetCliPackageType: ${{ parameters.DotNetCliPackageType }}
-      DotNetCliVersion: ${{ parameters.DotNetCliVersion }}
-      EnableXUnitReporter: ${{ parameters.EnableXUnitReporter }}
-      WaitForWorkItemCompletion: ${{ parameters.WaitForWorkItemCompletion }}
-      Creator: ${{ parameters.Creator }}
-      SYSTEM_ACCESSTOKEN: $(System.AccessToken)
-    condition: and(${{ parameters.condition }}, eq(variables['Agent.Os'], 'Windows_NT'))
+- template: /eng/pipelines/common/templates/runtimes/send-to-helix-inner-step.yml
+  parameters:
+    osGroup: ${{ parameters.osGroup }}
+    sendParams: $(Build.SourcesDirectory)/eng/common/performance/${{ parameters.ProjectFile }} /restore /t:Test /bl:$(Build.SourcesDirectory)/artifacts/log/$(_BuildConfig)/SendToHelix.binlog
+    displayName: ${{ parameters.DisplayNamePrefix }}
+    condition: ${{ parameters.condition }}
     continueOnError: ${{ parameters.continueOnError }}
-  - script: $BUILD_SOURCESDIRECTORY/eng/common/msbuild.sh $BUILD_SOURCESDIRECTORY/eng/common/performance/${{ parameters.ProjectFile }} /restore /t:Test /bl:$BUILD_SOURCESDIRECTORY/artifacts/log/$BuildConfig/SendToHelix.binlog
-    displayName: ${{ parameters.DisplayNamePrefix }} (Unix)
-    env:
+    environment:
       BuildConfig: $(_BuildConfig)
       HelixSource: ${{ parameters.HelixSource }}
       HelixType: ${{ parameters.HelixType }}
@@ -64,5 +48,3 @@ steps:
       WaitForWorkItemCompletion: ${{ parameters.WaitForWorkItemCompletion }}
       Creator: ${{ parameters.Creator }}
       SYSTEM_ACCESSTOKEN: $(System.AccessToken)
-    condition: and(${{ parameters.condition }}, ne(variables['Agent.Os'], 'Windows_NT'))
-    continueOnError: ${{ parameters.continueOnError }}

eng/common/templates/steps/publish-logs.yml

@@ -3,20 +3,21 @@ parameters:
   JobLabel: ''
 
 steps:
-- task: CopyFiles@2
-  displayName: Copy Logs to $(Build.StagingDirectory)\BuildLogs
+- task: Powershell@2
+  displayName: Prepare Binlogs to Upload
   inputs:
-    SourceFolder: $(Build.SourcesDirectory)\artifacts
-    Contents: |
-      **/*.log
-      **/*.binlog
-    TargetFolder: '$(Build.StagingDirectory)\BuildLogs'
+    targetType: inline
+    script: |
+      New-Item -ItemType Directory $(Build.SourcesDirectory)/PostBuildLogs/${{parameters.StageLabel}}/${{parameters.JobLabel}}/
+      Move-Item -Path $(Build.SourcesDirectory)/artifacts/log/Debug/* $(Build.SourcesDirectory)/PostBuildLogs/${{parameters.StageLabel}}/${{parameters.JobLabel}}/
   continueOnError: true
-  condition: succeededOrFailed()
+  condition: always()
 
-- task: PublishPipelineArtifact@1
-  displayName: Publish BuildLogs
+- task: PublishBuildArtifacts@1
+  displayName: Publish Logs
   inputs:
-    targetPath: '$(Build.StagingDirectory)\BuildLogs'
-    artifactName: ${{ parameters.JobLabel }}
-  condition: succeededOrFailed()
+    PathtoPublish: '$(Build.SourcesDirectory)/PostBuildLogs'
+    PublishLocation: Container
+    ArtifactName: PostBuildLogs
+  continueOnError: true
+  condition: always()

eng/common/templates/steps/send-to-helix.yml

@@ -18,7 +18,7 @@ parameters:
   XUnitRuntimeTargetFramework: ''        # optional -- framework to use for the xUnit console runner
   XUnitRunnerVersion: ''                 # optional -- version of the xUnit nuget package you wish to use on Helix; required for XUnitProjects
   IncludeDotNetCli: false                # optional -- true will download a version of the .NET CLI onto the Helix machine as a correlation payload; requires DotNetCliPackageType and DotNetCliVersion
-  DotNetCliPackageType: ''               # optional -- either 'sdk' or 'runtime'; determines whether the sdk or runtime will be sent to Helix; see https://raw.githubusercontent.com/dotnet/core/master/release-notes/releases-index.json
+  DotNetCliPackageType: ''               # optional -- either 'sdk', 'runtime' or 'aspnetcore-runtime'; determines whether the sdk or runtime will be sent to Helix; see https://raw.githubusercontent.com/dotnet/core/master/release-notes/releases-index.json
   DotNetCliVersion: ''                   # optional -- version of the CLI to send to Helix; based on this: https://raw.githubusercontent.com/dotnet/core/master/release-notes/releases-index.json
   EnableXUnitReporter: false             # optional -- true enables XUnit result reporting to Mission Control
   WaitForWorkItemCompletion: true        # optional -- true will make the task wait until work items have been completed and fail the build if work items fail. False is "fire and forget."

eng/common/templates/steps/source-build.yml

@@ -0,0 +1,66 @@
+parameters:
+  # This template adds arcade-powered source-build to CI.
+
+  # This is a 'steps' template, and is intended for advanced scenarios where the existing build
+  # infra has a careful build methodology that must be followed. For example, a repo
+  # (dotnet/runtime) might choose to clone the GitHub repo only once and store it as a pipeline
+  # artifact for all subsequent jobs to use, to reduce dependence on a strong network connection to
+  # GitHub. Using this steps template leaves room for that infra to be included.
+
+  # Defines the platform on which to run the steps. See 'eng/common/templates/job/source-build.yml'
+  # for details. The entire object is described in the 'job' template for simplicity, even though
+  # the usage of the properties on this object is split between the 'job' and 'steps' templates.
+  platform: {}
+
+steps:
+# Build. Keep it self-contained for simple reusability. (No source-build-specific job variables.)
+- script: |
+    set -x
+    df -h
+
+    buildConfig=Release
+    # Check if AzDO substitutes in a build config from a variable, and use it if so.
+    if [ '$(_BuildConfig)' != '$''(_BuildConfig)' ]; then
+      buildConfig='$(_BuildConfig)'
+    fi
+
+    officialBuildArgs=
+    if [ '${{ and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}' = 'True' ]; then
+      officialBuildArgs='/p:DotNetPublishUsingPipelines=true /p:OfficialBuildId=$(BUILD.BUILDNUMBER)'
+    fi
+
+    targetRidArgs=
+    if [ '${{ parameters.platform.targetRID }}' != '' ]; then
+      targetRidArgs='/p:TargetRid=${{ parameters.platform.targetRID }}'
+    fi
+
+    ${{ coalesce(parameters.platform.buildScript, './build.sh') }} --ci \
+      --configuration $buildConfig \
+      --restore --build --pack --publish \
+      $officialBuildArgs \
+      $targetRidArgs \
+      /p:SourceBuildNonPortable=${{ parameters.platform.nonPortable }} \
+      /p:ArcadeBuildFromSource=true
+  displayName: Build
+
+# Upload build logs for diagnosis.
+- task: CopyFiles@2
+  displayName: Prepare BuildLogs staging directory
+  inputs:
+    SourceFolder: '$(Build.SourcesDirectory)'
+    Contents: |
+      **/*.log
+      **/*.binlog
+      artifacts/source-build/self/prebuilt-report/**
+    TargetFolder: '$(Build.StagingDirectory)/BuildLogs'
+    CleanTargetFolder: true
+  continueOnError: true
+  condition: succeededOrFailed()
+
+- task: PublishPipelineArtifact@1
+  displayName: Publish BuildLogs
+  inputs:
+    targetPath: '$(Build.StagingDirectory)/BuildLogs'
+    artifactName: BuildLogs_SourceBuild_${{ parameters.platform.name }}_Attempt$(System.JobAttempt)
+  continueOnError: true
+  condition: succeededOrFailed()

eng/common/tools.ps1

@@ -240,8 +240,8 @@ function GetDotNetInstallScript([string] $dotnetRoot) {
   return $installScript
 }
 
-function InstallDotNetSdk([string] $dotnetRoot, [string] $version, [string] $architecture = '') {
-  InstallDotNet $dotnetRoot $version $architecture '' $false $runtimeSourceFeed $runtimeSourceFeedKey
+function InstallDotNetSdk([string] $dotnetRoot, [string] $version, [string] $architecture = '', [switch] $noPath) {
+  InstallDotNet $dotnetRoot $version $architecture '' $false $runtimeSourceFeed $runtimeSourceFeedKey -noPath:$noPath
 }
 
 function InstallDotNet([string] $dotnetRoot,
@@ -250,7 +250,8 @@ function InstallDotNet([string] $dotnetRoot,
   [string] $runtime = '',
   [bool] $skipNonVersionedFiles = $false,
   [string] $runtimeSourceFeed = '',
-  [string] $runtimeSourceFeedKey = '') {
+  [string] $runtimeSourceFeedKey = '',
+  [switch] $noPath) {
 
   $installScript = GetDotNetInstallScript $dotnetRoot
   $installParameters = @{
@@ -261,6 +262,7 @@ function InstallDotNet([string] $dotnetRoot,
   if ($architecture) { $installParameters.Architecture = $architecture }
   if ($runtime) { $installParameters.Runtime = $runtime }
   if ($skipNonVersionedFiles) { $installParameters.SkipNonVersionedFiles = $skipNonVersionedFiles }
+  if ($noPath) { $installParameters.NoPath = $True }
 
   try {
     & $installScript @installParameters
@@ -310,9 +312,15 @@ function InitializeVisualStudioMSBuild([bool]$install, [object]$vsRequirements =
     return $global:_MSBuildExe
   }
 
-  $vsMinVersionReqdStr = '16.5'
+  # Minimum VS version to require.
+  $vsMinVersionReqdStr = '16.8'
   $vsMinVersionReqd = [Version]::new($vsMinVersionReqdStr)
 
+  # If the version of msbuild is going to be xcopied,
+  # use this version. Version matches a package here:
+  # https://dev.azure.com/dnceng/public/_packaging?_a=package&feed=dotnet-eng&package=RoslynTools.MSBuild&protocolType=NuGet&version=16.8.0-preview3&view=overview
+  $defaultXCopyMSBuildVersion = '16.8.0-preview3'
+
   if (!$vsRequirements) { $vsRequirements = $GlobalJson.tools.vs }
   $vsMinVersionStr = if ($vsRequirements.version) { $vsRequirements.version } else { $vsMinVersionReqdStr }
   $vsMinVersion = [Version]::new($vsMinVersionStr)
@@ -347,23 +355,28 @@ function InitializeVisualStudioMSBuild([bool]$install, [object]$vsRequirements =
       $xcopyMSBuildVersion = $GlobalJson.tools.'xcopy-msbuild'
       $vsMajorVersion = $xcopyMSBuildVersion.Split('.')[0]
     } else {
-      #if vs version provided in global.json is incompatible then use the default version for xcopy msbuild download
+      #if vs version provided in global.json is incompatible (too low) then use the default version for xcopy msbuild download
       if($vsMinVersion -lt $vsMinVersionReqd){
-        Write-Host "Using xcopy-msbuild version of $vsMinVersionReqdStr.0-alpha since VS version $vsMinVersionStr provided in global.json is not compatible"
-        $vsMajorVersion = $vsMinVersionReqd.Major
-        $vsMinorVersion = $vsMinVersionReqd.Minor
+        Write-Host "Using xcopy-msbuild version of $defaultXCopyMSBuildVersion since VS version $vsMinVersionStr provided in global.json is not compatible"
+        $xcopyMSBuildVersion = $defaultXCopyMSBuildVersion
       }
       else{
+        # If the VS version IS compatible, look for an xcopy msbuild package
+        # with a version matching VS.
+        # Note: If this version does not exist, then an explicit version of xcopy msbuild
+        # can be specified in global.json. This will be required for pre-release versions of msbuild.
         $vsMajorVersion = $vsMinVersion.Major
         $vsMinorVersion = $vsMinVersion.Minor
+        $xcopyMSBuildVersion = "$vsMajorVersion.$vsMinorVersion.0"
       }
-
-      $xcopyMSBuildVersion = "$vsMajorVersion.$vsMinorVersion.0-alpha"
     }
 
     $vsInstallDir = $null
     if ($xcopyMSBuildVersion.Trim() -ine "none") {
         $vsInstallDir = InitializeXCopyMSBuild $xcopyMSBuildVersion $install
+        if ($vsInstallDir -eq $null) {
+            throw "Could not xcopy msbuild. Please check that package 'RoslynTools.MSBuild @ $xcopyMSBuildVersion' exists on feed 'dotnet-eng'."
+        }
     }
     if ($vsInstallDir -eq $null) {
       throw 'Unable to find Visual Studio that has required version and components installed'
@@ -527,13 +540,15 @@ function GetDefaultMSBuildEngine() {
 
 function GetNuGetPackageCachePath() {
   if ($env:NUGET_PACKAGES -eq $null) {
-    # Use local cache on CI to ensure deterministic build,
+    # Use local cache on CI to ensure deterministic build. 
+    # Avoid using the http cache as workaround for https://github.com/NuGet/Home/issues/3116
     # use global cache in dev builds to avoid cost of downloading packages.
     # For directory normalization, see also: https://github.com/NuGet/Home/issues/7968
     if ($useGlobalNuGetCache) {
       $env:NUGET_PACKAGES = Join-Path $env:UserProfile '.nuget\packages\'
     } else {
       $env:NUGET_PACKAGES = Join-Path $RepoRoot '.packages\'
+      $env:RESTORENOCACHE = $true
     }
   }
 
@@ -716,6 +731,16 @@ function IsWindowsPlatform() {
   return [environment]::OSVersion.Platform -eq [PlatformID]::Win32NT
 }
 
+function Get-Darc($version) {
+  $darcPath  = "$TempDir\darc\$(New-Guid)"
+  if ($version -ne $null) {
+    & $PSScriptRoot\darc-init.ps1 -toolpath $darcPath -darcVersion $version | Out-Host
+  } else {
+    & $PSScriptRoot\darc-init.ps1 -toolpath $darcPath | Out-Host
+  }
+  return "$darcPath\darc.exe"
+}
+
 . $PSScriptRoot\pipeline-logging-functions.ps1
 
 $RepoRoot = Resolve-Path (Join-Path $PSScriptRoot '..\..')

eng/common/tools.sh

@@ -312,12 +312,14 @@ function InitializeBuildTool {
   _InitializeBuildToolFramework="netcoreapp2.1"
 }
 
+# Set RestoreNoCache as a workaround for https://github.com/NuGet/Home/issues/3116
 function GetNuGetPackageCachePath {
   if [[ -z ${NUGET_PACKAGES:-} ]]; then
     if [[ "$use_global_nuget_cache" == true ]]; then
       export NUGET_PACKAGES="$HOME/.nuget/packages"
     else
       export NUGET_PACKAGES="$repo_root/.packages"
+      export RESTORENOCACHE=true
     fi
   fi
 

global.json

@@ -11,7 +11,7 @@
   },
 
   "msbuild-sdks": {
-    "Microsoft.DotNet.Arcade.Sdk": "5.0.0-beta.20426.1",
-    "Microsoft.DotNet.Helix.Sdk": "5.0.0-beta.20426.1"
+    "Microsoft.DotNet.Arcade.Sdk": "5.0.0-beta.20510.1",
+    "Microsoft.DotNet.Helix.Sdk": "5.0.0-beta.20510.1"
   }
 }