Update dependencies from https://github.com/dotnet/arcade build 20220406.10 (#1352)

[main] Update dependencies from dotnet/arcade

eng/Version.Details.xml

@@ -3,25 +3,25 @@
   <ProductDependencies>
   </ProductDependencies>
   <ToolsetDependencies>
-    <Dependency Name="Microsoft.DotNet.Arcade.Sdk" Version="7.0.0-beta.22181.2">
+    <Dependency Name="Microsoft.DotNet.Arcade.Sdk" Version="7.0.0-beta.22206.10">
       <Uri>https://github.com/dotnet/arcade</Uri>
-      <Sha>e0b311bcd81fc9e27bcf7715dcda62fa38dfa49a</Sha>
+      <Sha>549523c3fc8929da1a3073d1a97f298e0d1dc342</Sha>
     </Dependency>
-    <Dependency Name="Microsoft.DotNet.Build.Tasks.Feed" Version="7.0.0-beta.22181.2">
+    <Dependency Name="Microsoft.DotNet.Build.Tasks.Feed" Version="7.0.0-beta.22206.10">
       <Uri>https://github.com/dotnet/arcade</Uri>
-      <Sha>e0b311bcd81fc9e27bcf7715dcda62fa38dfa49a</Sha>
+      <Sha>549523c3fc8929da1a3073d1a97f298e0d1dc342</Sha>
     </Dependency>
-    <Dependency Name="Microsoft.DotNet.SignTool" Version="7.0.0-beta.22181.2">
+    <Dependency Name="Microsoft.DotNet.SignTool" Version="7.0.0-beta.22206.10">
       <Uri>https://github.com/dotnet/arcade</Uri>
-      <Sha>e0b311bcd81fc9e27bcf7715dcda62fa38dfa49a</Sha>
+      <Sha>549523c3fc8929da1a3073d1a97f298e0d1dc342</Sha>
     </Dependency>
-    <Dependency Name="Microsoft.DotNet.Helix.Sdk" Version="7.0.0-beta.22181.2">
+    <Dependency Name="Microsoft.DotNet.Helix.Sdk" Version="7.0.0-beta.22206.10">
       <Uri>https://github.com/dotnet/arcade</Uri>
-      <Sha>e0b311bcd81fc9e27bcf7715dcda62fa38dfa49a</Sha>
+      <Sha>549523c3fc8929da1a3073d1a97f298e0d1dc342</Sha>
     </Dependency>
-    <Dependency Name="Microsoft.DotNet.SwaggerGenerator.MSBuild" Version="7.0.0-beta.22181.2">
+    <Dependency Name="Microsoft.DotNet.SwaggerGenerator.MSBuild" Version="7.0.0-beta.22206.10">
       <Uri>https://github.com/dotnet/arcade</Uri>
-      <Sha>e0b311bcd81fc9e27bcf7715dcda62fa38dfa49a</Sha>
+      <Sha>549523c3fc8929da1a3073d1a97f298e0d1dc342</Sha>
     </Dependency>
     <Dependency Name="Microsoft.DotNet.Maestro.Client" Version="1.1.0-beta.19556.4">
       <Uri>https://github.com/dotnet/arcade-services</Uri>

eng/common/init-tools-native.ps1

@@ -31,6 +31,10 @@ Wait time between retry attempts in seconds
 .PARAMETER GlobalJsonFile
 File path to global.json file
 
+.PARAMETER PathPromotion
+Optional switch to enable either promote native tools specified in the global.json to the path (in Azure Pipelines)
+or break the build if a native tool is not found on the path (on a local dev machine)
+
 .NOTES
 #>
 [CmdletBinding(PositionalBinding=$false)]
@@ -41,7 +45,8 @@ Param (
   [switch] $Force = $False,
   [int] $DownloadRetries = 5,
   [int] $RetryWaitTimeInSeconds = 30,
-  [string] $GlobalJsonFile
+  [string] $GlobalJsonFile,
+  [switch] $PathPromotion
 )
 
 if (!$GlobalJsonFile) {
@@ -77,53 +82,97 @@ try {
                     ConvertFrom-Json |
                     Select-Object -Expand 'native-tools' -ErrorAction SilentlyContinue
   if ($NativeTools) {
-    $NativeTools.PSObject.Properties | ForEach-Object {
+    if ($PathPromotion -eq $True) {
-      $ToolName = $_.Name
+      if ($env:SYSTEM_TEAMPROJECT) { # check to see if we're in an Azure pipelines build
-      $ToolVersion = $_.Value
+        $NativeTools.PSObject.Properties | ForEach-Object {
-      $LocalInstallerArguments =  @{ ToolName = "$ToolName" }
+          $ToolName = $_.Name
-      $LocalInstallerArguments += @{ InstallPath = "$InstallBin" }
+          $ToolVersion = $_.Value
-      $LocalInstallerArguments += @{ BaseUri = "$BaseUri" }
+
-      $LocalInstallerArguments += @{ CommonLibraryDirectory = "$EngCommonBaseDir" }
+          if ((Get-Command "$ToolName" -ErrorAction SilentlyContinue) -eq $null) {
-      $LocalInstallerArguments += @{ Version = "$ToolVersion" }
+            if ($ToolVersion -eq "latest") {
-
+              $ToolVersion = ""
-      if ($Verbose) {
+            }
-        $LocalInstallerArguments += @{ Verbose = $True }
+            $ArcadeToolsDirectory = "C:\arcade-tools"
-      }
+            if (Test-Path $ArcadeToolsDirectory -eq $False) {
-      if (Get-Variable 'Force' -ErrorAction 'SilentlyContinue') {
+              Write-Error "Arcade tools directory '$ArcadeToolsDirectory' was not found; artifacts were not properly installed."
-        if($Force) {
+              exit 1
-          $LocalInstallerArguments += @{ Force = $True }
-        }
-      }
-      if ($Clean) {
-        $LocalInstallerArguments += @{ Clean = $True }
-      }
-
-      Write-Verbose "Installing $ToolName version $ToolVersion"
-      Write-Verbose "Executing '$InstallerPath $($LocalInstallerArguments.Keys.ForEach({"-$_ '$($LocalInstallerArguments.$_)'"}) -join ' ')'"
-      & $InstallerPath @LocalInstallerArguments
-      if ($LASTEXITCODE -Ne "0") {
-        $errMsg = "$ToolName installation failed"
-        if ((Get-Variable 'DoNotAbortNativeToolsInstallationOnFailure' -ErrorAction 'SilentlyContinue') -and $DoNotAbortNativeToolsInstallationOnFailure) {
-            $showNativeToolsWarning = $true
-            if ((Get-Variable 'DoNotDisplayNativeToolsInstallationWarnings' -ErrorAction 'SilentlyContinue') -and $DoNotDisplayNativeToolsInstallationWarnings) {
-                $showNativeToolsWarning = $false
             }
-            if ($showNativeToolsWarning) {
+            $ToolDirectory = (Get-ChildItem -Path "$ArcadeToolsDirectory" -Filter "$ToolName-$ToolVersion*" | Sort-Object -Descending)[0]
-                Write-Warning $errMsg
+            if ([string]::IsNullOrWhiteSpace($ToolDirectory)) {
+              Write-Error "Unable to find directory for $ToolName $ToolVersion; please make sure the tool is installed on this image."
+              exit 1
             }
-            $toolInstallationFailure = $true
+            $BinPathFile = "$($ToolDirectory.FullName)\binpath.txt"
-        } else {
+            if (Test-Path -Path "$BinPathFile" -eq $False) {
-            # We cannot change this to Write-PipelineTelemetryError because of https://github.com/dotnet/arcade/issues/4482
+              Write-Error "Unable to find binpath.txt in '$($ToolDirectory.FullName)' ($ToolName $ToolVersion); artifact is either installed incorrectly or is not a bootstrappable tool."
-            Write-Host $errMsg
+              exit 1
-            exit 1
+            }
+            $BinPath = Get-Content "$BinPathFile"
+            Write-Host "Adding $ToolName to the path ($(Convert-Path -Path $BinPath))..."
+            Write-Host "##vso[task.prependpath]$(Convert-Path -Path $BinPath)"
+          }
+        }
+        exit 0
+      } else {
+        $NativeTools.PSObject.Properties | ForEach-Object {
+          $ToolName = $_.Name
+          $ToolVersion = $_.Value
+
+          if ((Get-Command "$ToolName" -ErrorAction SilentlyContinue) -eq $null) {
+            Write-PipelineTelemetryError -Category 'NativeToolsBootstrap' -Message "$ToolName not found on path. Please install $ToolName $ToolVersion before proceeding."
+          }
         }
+        exit 0
+      }
+    } else {
+      $NativeTools.PSObject.Properties | ForEach-Object {
+        $ToolName = $_.Name
+        $ToolVersion = $_.Value
+        $LocalInstallerArguments =  @{ ToolName = "$ToolName" }
+        $LocalInstallerArguments += @{ InstallPath = "$InstallBin" }
+        $LocalInstallerArguments += @{ BaseUri = "$BaseUri" }
+        $LocalInstallerArguments += @{ CommonLibraryDirectory = "$EngCommonBaseDir" }
+        $LocalInstallerArguments += @{ Version = "$ToolVersion" }
+  
+        if ($Verbose) {
+          $LocalInstallerArguments += @{ Verbose = $True }
+        }
+        if (Get-Variable 'Force' -ErrorAction 'SilentlyContinue') {
+          if($Force) {
+            $LocalInstallerArguments += @{ Force = $True }
+          }
+        }
+        if ($Clean) {
+          $LocalInstallerArguments += @{ Clean = $True }
+        }
+  
+        Write-Verbose "Installing $ToolName version $ToolVersion"
+        Write-Verbose "Executing '$InstallerPath $($LocalInstallerArguments.Keys.ForEach({"-$_ '$($LocalInstallerArguments.$_)'"}) -join ' ')'"
+        & $InstallerPath @LocalInstallerArguments
+        if ($LASTEXITCODE -Ne "0") {
+          $errMsg = "$ToolName installation failed"
+          if ((Get-Variable 'DoNotAbortNativeToolsInstallationOnFailure' -ErrorAction 'SilentlyContinue') -and $DoNotAbortNativeToolsInstallationOnFailure) {
+              $showNativeToolsWarning = $true
+              if ((Get-Variable 'DoNotDisplayNativeToolsInstallationWarnings' -ErrorAction 'SilentlyContinue') -and $DoNotDisplayNativeToolsInstallationWarnings) {
+                  $showNativeToolsWarning = $false
+              }
+              if ($showNativeToolsWarning) {
+                  Write-Warning $errMsg
+              }
+              $toolInstallationFailure = $true
+          } else {
+              # We cannot change this to Write-PipelineTelemetryError because of https://github.com/dotnet/arcade/issues/4482
+              Write-Host $errMsg
+              exit 1
+          }
+        }
+      }
+  
+      if ((Get-Variable 'toolInstallationFailure' -ErrorAction 'SilentlyContinue') -and $toolInstallationFailure) {
+          # We cannot change this to Write-PipelineTelemetryError because of https://github.com/dotnet/arcade/issues/4482
+          Write-Host 'Native tools bootstrap failed'
+          exit 1
       }
-    }
-
-    if ((Get-Variable 'toolInstallationFailure' -ErrorAction 'SilentlyContinue') -and $toolInstallationFailure) {
-        # We cannot change this to Write-PipelineTelemetryError because of https://github.com/dotnet/arcade/issues/4482
-        Write-Host 'Native tools bootstrap failed'
-        exit 1
     }
   }
   else {

eng/common/templates/job/publish-build-assets.yml

@@ -23,23 +23,33 @@ parameters:
   # Optional: whether the build's artifacts will be published using release pipelines or direct feed publishing
   publishUsingPipelines: false
 
+  # Optional: whether the build's artifacts will be published using release pipelines or direct feed publishing
+  publishAssetsImmediately: false
+
+  artifactsPublishingAdditionalParameters: ''
+
+  signingValidationAdditionalParameters: ''
+
 jobs:
 - job: Asset_Registry_Publish
 
   dependsOn: ${{ parameters.dependsOn }}
 
-  displayName: Publish to Build Asset Registry
+  ${{ if eq(parameters.publishAssetsImmediately, 'true') }}:
+    displayName: Publish Assets
+  ${{ else }}:
+    displayName: Publish to Build Asset Registry
 
   pool: ${{ parameters.pool }}
 
   variables:
   - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}:
-    - name: _BuildConfig
-      value: ${{ parameters.configuration }}
     - group: Publish-Build-Assets
     - group: AzureDevOps-Artifact-Feeds-Pats
     - name: runCodesignValidationInjection
       value: false
+    - ${{ if eq(parameters.publishAssetsImmediately, 'true') }}:
+      - template: /eng/common/templates/post-build/common-variables.yml
 
   steps:
   - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}:
@@ -52,14 +62,13 @@ jobs:
       condition: ${{ parameters.condition }}
       continueOnError: ${{ parameters.continueOnError }}
     
-    - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}:
+    - task: NuGetAuthenticate@0
-      - task: NuGetAuthenticate@0
 
-      - task: PowerShell@2 
+    - task: PowerShell@2 
-        displayName: Enable cross-org NuGet feed authentication 
+      displayName: Enable cross-org NuGet feed authentication 
-        inputs: 
+      inputs: 
-          filePath: $(Build.SourcesDirectory)/eng/common/enable-cross-org-publishing.ps1 
+        filePath: $(Build.SourcesDirectory)/eng/common/enable-cross-org-publishing.ps1 
-          arguments: -token $(dn-bot-all-orgs-artifact-feeds-rw) 
+        arguments: -token $(dn-bot-all-orgs-artifact-feeds-rw) 
 
     - task: PowerShell@2
       displayName: Publish Build Assets
@@ -70,7 +79,6 @@ jobs:
           /p:BuildAssetRegistryToken=$(MaestroAccessToken)
           /p:MaestroApiEndpoint=https://maestro-prod.westus2.cloudapp.azure.com
           /p:PublishUsingPipelines=${{ parameters.publishUsingPipelines }}
-          /p:Configuration=$(_BuildConfig)
           /p:OfficialBuildId=$(Build.BuildNumber)
       condition: ${{ parameters.condition }}
       continueOnError: ${{ parameters.continueOnError }}
@@ -114,7 +122,25 @@ jobs:
         PathtoPublish: '$(Build.SourcesDirectory)/eng/SymbolPublishingExclusionsFile.txt'
         PublishLocation: Container
         ArtifactName: ReleaseConfigs
-        
+
+    - ${{ if eq(parameters.publishAssetsImmediately, 'true') }}:
+      - template: /eng/common/templates/post-build/setup-maestro-vars.yml
+        parameters:
+          BARBuildId: ${{ parameters.BARBuildId }}
+          PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }}
+
+      - task: PowerShell@2
+        displayName: Publish Using Darc
+        inputs:
+          filePath: $(Build.SourcesDirectory)/eng/common/post-build/publish-using-darc.ps1
+          arguments: -BuildId $(BARBuildId) 
+            -PublishingInfraVersion 3
+            -AzdoToken '$(publishing-dnceng-devdiv-code-r-build-re)'
+            -MaestroToken '$(MaestroApiAccessToken)'
+            -WaitPublishingFinish true
+            -ArtifactsPublishingAdditionalParameters '${{ parameters.artifactsPublishingAdditionalParameters }}'
+            -SymbolPublishingAdditionalParameters '${{ parameters.symbolPublishingAdditionalParameters }}'
+
     - ${{ if eq(parameters.enablePublishBuildArtifacts, 'true') }}:
       - template: /eng/common/templates/steps/publish-logs.yml
         parameters:

eng/common/templates/jobs/jobs.yml

@@ -27,6 +27,13 @@ parameters:
   # Optional: Override automatically derived dependsOn value for "publish build assets" job
   publishBuildAssetsDependsOn: ''
 
+  # Optional: Publish the assets as soon as the publish to BAR stage is complete, rather doing so in a separate stage.
+  publishAssetsImmediately: false
+
+  # Optional: If using publishAssetsImmediately and additional parameters are needed, can be used to send along additional parameters (normally sent to post-build.yml)
+  artifactsPublishingAdditionalParameters: ''
+  signingValidationAdditionalParameters: ''
+
   # Optional: should run as a public build even in the internal project
   #           if 'true', the build won't run any of the internal only steps, even if it is running in non-public projects.
   runAsPublic: false
@@ -68,7 +75,6 @@ jobs:
         ${{ parameter.key }}: ${{ parameter.value }}
 
 - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}:
-  
   - ${{ if or(eq(parameters.enablePublishBuildAssets, true), eq(parameters.artifacts.publish.manifests, 'true'), ne(parameters.artifacts.publish.manifests, '')) }}:
     - template: ../job/publish-build-assets.yml
       parameters:
@@ -94,4 +100,7 @@ jobs:
 
         runAsPublic: ${{ parameters.runAsPublic }}
         publishUsingPipelines: ${{ parameters.enablePublishUsingPipelines }}
+        publishAssetsImmediately: ${{ parameters.publishAssetsImmediately }}
         enablePublishBuildArtifacts: ${{ parameters.enablePublishBuildArtifacts }}
+        artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }}
+        signingValidationAdditionalParameters: ${{ parameters.signingValidationAdditionalParameters }}

eng/common/templates/post-build/post-build.yml

@@ -82,6 +82,11 @@ parameters:
     default:
     - Validate
 
+  # Optional: Call asset publishing rather than running in a separate stage
+  - name: publishAssetsImmediately
+    type: boolean
+    default: false
+
 stages:
 - ${{ if or(eq( parameters.enableNugetValidation, 'true'), eq(parameters.enableSigningValidation, 'true'), eq(parameters.enableSourceLinkValidation, 'true'), eq(parameters.SDLValidationParameters.enable, 'true')) }}:
   - stage: Validate
@@ -99,7 +104,7 @@ stages:
           name: VSEngSS-MicroBuild2022-1ES
           demands: Cmd
         # If it's not devdiv, it's dnceng
-        ${{ if ne(variables['System.TeamProject'], 'DevDiv') }}:
+        ${{ else }}:
           name: NetCore1ESPool-Internal
           demands: ImageOverride -equals Build.Server.Amd64.VS2019
 
@@ -136,7 +141,7 @@ stages:
           name: VSEngSS-MicroBuild2022-1ES
           demands: Cmd
         # If it's not devdiv, it's dnceng
-        ${{ if ne(variables['System.TeamProject'], 'DevDiv') }}:
+        ${{ else }}:
           name: NetCore1ESPool-Internal
           demands: ImageOverride -equals Build.Server.Amd64.VS2019
       steps:
@@ -196,7 +201,7 @@ stages:
           name: VSEngSS-MicroBuild2022-1ES
           demands: Cmd
         # If it's not devdiv, it's dnceng
-        ${{ if ne(variables['System.TeamProject'], 'DevDiv') }}:
+        ${{ else }}:
           name: NetCore1ESPool-Internal
           demands: ImageOverride -equals Build.Server.Amd64.VS2019
       steps:
@@ -235,43 +240,44 @@ stages:
         artifactNames: ${{ parameters.SDLValidationParameters.artifactNames }}
         downloadArtifacts: ${{ parameters.SDLValidationParameters.downloadArtifacts }}
 
-- stage: publish_using_darc
+- ${{ if ne(parameters.publishAssetsImmediately, 'true') }}:
-  ${{ if or(eq(parameters.enableNugetValidation, 'true'), eq(parameters.enableSigningValidation, 'true'), eq(parameters.enableSourceLinkValidation, 'true'), eq(parameters.SDLValidationParameters.enable, 'true')) }}:
+  - stage: publish_using_darc
-    dependsOn: ${{ parameters.publishDependsOn }}
+    ${{ if or(eq(parameters.enableNugetValidation, 'true'), eq(parameters.enableSigningValidation, 'true'), eq(parameters.enableSourceLinkValidation, 'true'), eq(parameters.SDLValidationParameters.enable, 'true')) }}:
-  ${{ if and(ne(parameters.enableNugetValidation, 'true'), ne(parameters.enableSigningValidation, 'true'), ne(parameters.enableSourceLinkValidation, 'true'), ne(parameters.SDLValidationParameters.enable, 'true')) }}:
+      dependsOn: ${{ parameters.publishDependsOn }}
-    dependsOn: ${{ parameters.validateDependsOn }}
+    ${{ else }}:
-  displayName: Publish using Darc
+      dependsOn: ${{ parameters.validateDependsOn }}
-  variables:
+    displayName: Publish using Darc
-    - template: common-variables.yml
+    variables:
-  jobs:
+      - template: common-variables.yml
-  - job:
+    jobs:
-    displayName: Publish Using Darc
+    - job:
-    timeoutInMinutes: 120
+      displayName: Publish Using Darc
-    pool:
+      timeoutInMinutes: 120
-      # We don't use the collection uri here because it might vary (.visualstudio.com vs. dev.azure.com)
+      pool:
+        # We don't use the collection uri here because it might vary (.visualstudio.com vs. dev.azure.com)
         ${{ if eq(variables['System.TeamProject'], 'DevDiv') }}:
           name: VSEngSS-MicroBuild2022-1ES
           demands: Cmd
         # If it's not devdiv, it's dnceng
-        ${{ if ne(variables['System.TeamProject'], 'DevDiv') }}:
+        ${{ else }}:
           name: NetCore1ESPool-Internal
           demands: ImageOverride -equals Build.Server.Amd64.VS2019
-    steps:
+      steps:
-      - template: setup-maestro-vars.yml
+        - template: setup-maestro-vars.yml
-        parameters:
+          parameters:
-          BARBuildId: ${{ parameters.BARBuildId }}
+            BARBuildId: ${{ parameters.BARBuildId }}
-          PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }}
+            PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }}
 
-      - task: NuGetAuthenticate@0
+        - task: NuGetAuthenticate@0
 
-      - task: PowerShell@2
+        - task: PowerShell@2
-        displayName: Publish Using Darc
+          displayName: Publish Using Darc
-        inputs:
+          inputs:
-          filePath: $(Build.SourcesDirectory)/eng/common/post-build/publish-using-darc.ps1
+            filePath: $(Build.SourcesDirectory)/eng/common/post-build/publish-using-darc.ps1
-          arguments: -BuildId $(BARBuildId) 
+            arguments: -BuildId $(BARBuildId) 
-            -PublishingInfraVersion ${{ parameters.publishingInfraVersion }}
+              -PublishingInfraVersion ${{ parameters.publishingInfraVersion }}
-            -AzdoToken '$(publishing-dnceng-devdiv-code-r-build-re)'
+              -AzdoToken '$(publishing-dnceng-devdiv-code-r-build-re)'
-            -MaestroToken '$(MaestroApiAccessToken)'
+              -MaestroToken '$(MaestroApiAccessToken)'
-            -WaitPublishingFinish true
+              -WaitPublishingFinish true
-            -ArtifactsPublishingAdditionalParameters '${{ parameters.artifactsPublishingAdditionalParameters }}'
+              -ArtifactsPublishingAdditionalParameters '${{ parameters.artifactsPublishingAdditionalParameters }}'
-            -SymbolPublishingAdditionalParameters '${{ parameters.symbolPublishingAdditionalParameters }}'
+              -SymbolPublishingAdditionalParameters '${{ parameters.symbolPublishingAdditionalParameters }}'

global.json

@@ -14,6 +14,6 @@
     }
   },
   "msbuild-sdks": {
-    "Microsoft.DotNet.Arcade.Sdk": "7.0.0-beta.22181.2"
+    "Microsoft.DotNet.Arcade.Sdk": "7.0.0-beta.22206.10"
   }
 }