[main] Update dependencies from dotnet/arcade (#1032)

[main] Update dependencies from dotnet/arcade

eng/Version.Details.xml

@@ -3,25 +3,25 @@
   <ProductDependencies>
   </ProductDependencies>
   <ToolsetDependencies>
-    <Dependency Name="Microsoft.DotNet.Arcade.Sdk" Version="6.0.0-beta.21253.2">
+    <Dependency Name="Microsoft.DotNet.Arcade.Sdk" Version="6.0.0-beta.21274.6">
       <Uri>https://github.com/dotnet/arcade</Uri>
-      <Sha>e9fd640e1ec1890489ea66ad0f59e733448056da</Sha>
+      <Sha>f22435d0f9cc367e75e1875d93ef980be1be6b79</Sha>
     </Dependency>
-    <Dependency Name="Microsoft.DotNet.Build.Tasks.Feed" Version="6.0.0-beta.21253.2">
+    <Dependency Name="Microsoft.DotNet.Build.Tasks.Feed" Version="6.0.0-beta.21274.6">
       <Uri>https://github.com/dotnet/arcade</Uri>
-      <Sha>e9fd640e1ec1890489ea66ad0f59e733448056da</Sha>
+      <Sha>f22435d0f9cc367e75e1875d93ef980be1be6b79</Sha>
     </Dependency>
-    <Dependency Name="Microsoft.DotNet.SignTool" Version="6.0.0-beta.21253.2">
+    <Dependency Name="Microsoft.DotNet.SignTool" Version="6.0.0-beta.21274.6">
       <Uri>https://github.com/dotnet/arcade</Uri>
-      <Sha>e9fd640e1ec1890489ea66ad0f59e733448056da</Sha>
+      <Sha>f22435d0f9cc367e75e1875d93ef980be1be6b79</Sha>
     </Dependency>
-    <Dependency Name="Microsoft.DotNet.Helix.Sdk" Version="6.0.0-beta.21253.2">
+    <Dependency Name="Microsoft.DotNet.Helix.Sdk" Version="6.0.0-beta.21274.6">
       <Uri>https://github.com/dotnet/arcade</Uri>
-      <Sha>e9fd640e1ec1890489ea66ad0f59e733448056da</Sha>
+      <Sha>f22435d0f9cc367e75e1875d93ef980be1be6b79</Sha>
     </Dependency>
-    <Dependency Name="Microsoft.DotNet.SwaggerGenerator.MSBuild" Version="6.0.0-beta.21253.2">
+    <Dependency Name="Microsoft.DotNet.SwaggerGenerator.MSBuild" Version="6.0.0-beta.21274.6">
       <Uri>https://github.com/dotnet/arcade</Uri>
-      <Sha>e9fd640e1ec1890489ea66ad0f59e733448056da</Sha>
+      <Sha>f22435d0f9cc367e75e1875d93ef980be1be6b79</Sha>
     </Dependency>
     <Dependency Name="Microsoft.DotNet.Maestro.Client" Version="1.1.0-beta.19556.4">
       <Uri>https://github.com/dotnet/arcade-services</Uri>

eng/common/build.ps1

@@ -25,6 +25,7 @@ Param(
   [switch] $prepareMachine,
   [string] $runtimeSourceFeed = '',
   [string] $runtimeSourceFeedKey = '',
+  [switch] $excludePrereleaseVS,
   [switch] $help,
   [Parameter(ValueFromRemainingArguments=$true)][String[]]$properties
 )
@@ -65,6 +66,7 @@ function Print-Usage() {
   Write-Host "  -prepareMachine         Prepare machine for CI run, clean up processes after build"
   Write-Host "  -warnAsError <value>    Sets warnaserror msbuild parameter ('true' or 'false')"
   Write-Host "  -msbuildEngine <value>  Msbuild engine to use to run build ('dotnet', 'vs', or unspecified)."
+  Write-Host "  -excludePrereleaseVS    Set to exclude build engines in prerelease versions of Visual Studio"
   Write-Host ""
 
   Write-Host "Command line arguments not listed above are passed thru to msbuild."

eng/common/cross/build-android-rootfs.sh

@@ -106,7 +106,6 @@ __AndroidPackages+=" libandroid-glob"
 __AndroidPackages+=" liblzma"
 __AndroidPackages+=" krb5"
 __AndroidPackages+=" openssl"
-__AndroidPackages+=" openldap"
 
 for path in $(wget -qO- http://termux.net/dists/stable/main/binary-$__AndroidArch/Packages |\
     grep -A15 "Package: \(${__AndroidPackages// /\\|}\)" | grep -v "static\|tool" | grep Filename); do

eng/common/cross/build-rootfs.sh

@@ -55,13 +55,11 @@ __UbuntuPackages+=" libcurl4-openssl-dev"
 __UbuntuPackages+=" libkrb5-dev"
 __UbuntuPackages+=" libssl-dev"
 __UbuntuPackages+=" zlib1g-dev"
-__UbuntuPackages+=" libldap2-dev"
 
 __AlpinePackages+=" curl-dev"
 __AlpinePackages+=" krb5-dev"
 __AlpinePackages+=" openssl-dev"
 __AlpinePackages+=" zlib-dev"
-__AlpinePackages+=" openldap-dev"
 
 __FreeBSDBase="12.1-RELEASE"
 __FreeBSDPkg="1.12.0"
@@ -70,13 +68,15 @@ __FreeBSDPackages+=" icu"
 __FreeBSDPackages+=" libinotify"
 __FreeBSDPackages+=" lttng-ust"
 __FreeBSDPackages+=" krb5"
-__FreeBSDPackages+=" libslapi-2.4"
 
 __IllumosPackages="icu-64.2nb2"
 __IllumosPackages+=" mit-krb5-1.16.2nb4"
 __IllumosPackages+=" openssl-1.1.1e"
 __IllumosPackages+=" zlib-1.2.11"
-__IllumosPackages+=" openldap-client-2.4.49"
+
+# ML.NET dependencies
+__UbuntuPackages+=" libomp5"
+__UbuntuPackages+=" libomp-dev"
 
 __UseMirror=0
 

eng/common/generate-locproject.ps1

@@ -14,7 +14,7 @@ $ErrorActionPreference = "Stop"
 
 Import-Module -Name (Join-Path $PSScriptRoot 'native\CommonLibrary.psm1')
 
-$exclusionsFilePath = "$SourcesDirectory\Localize\LocExclusions.json"
+$exclusionsFilePath = "$SourcesDirectory\eng\Localize\LocExclusions.json"
 $exclusions = @{ Exclusions = @() }
 if (Test-Path -Path $exclusionsFilePath)
 {
@@ -92,14 +92,14 @@ Write-Host "LocProject.json generated:`n`n$json`n`n"
 Pop-Location
 
 if (!$UseCheckedInLocProjectJson) {
-    New-Item "$SourcesDirectory\Localize\LocProject.json" -Force # Need this to make sure the Localize directory is created
-    Set-Content "$SourcesDirectory\Localize\LocProject.json" $json
+    New-Item "$SourcesDirectory\eng\Localize\LocProject.json" -Force # Need this to make sure the Localize directory is created
+    Set-Content "$SourcesDirectory\eng\Localize\LocProject.json" $json
 }
 else {
-    New-Item "$SourcesDirectory\Localize\LocProject-generated.json" -Force # Need this to make sure the Localize directory is created
-    Set-Content "$SourcesDirectory\Localize\LocProject-generated.json" $json
+    New-Item "$SourcesDirectory\eng\Localize\LocProject-generated.json" -Force # Need this to make sure the Localize directory is created
+    Set-Content "$SourcesDirectory\eng\Localize\LocProject-generated.json" $json
 
-    if ((Get-FileHash "$SourcesDirectory\Localize\LocProject-generated.json").Hash -ne (Get-FileHash "$SourcesDirectory\Localize\LocProject.json").Hash) {
+    if ((Get-FileHash "$SourcesDirectory\eng\Localize\LocProject-generated.json").Hash -ne (Get-FileHash "$SourcesDirectory\eng\Localize\LocProject.json").Hash) {
         Write-PipelineTelemetryError -Category "OneLocBuild" -Message "Existing LocProject.json differs from generated LocProject.json. Download LocProject-generated.json and compare them."
         
         exit 1

eng/common/msbuild.ps1

@@ -5,6 +5,7 @@ Param(
   [bool] $nodeReuse = $true,
   [switch] $ci,
   [switch] $prepareMachine,
+  [switch] $excludePrereleaseVS,
   [Parameter(ValueFromRemainingArguments=$true)][String[]]$extraArgs
 )
 

eng/common/post-build/sourcelink-validation.ps1

@@ -16,6 +16,8 @@ $global:RepoFiles = @{}
 # Maximum number of jobs to run in parallel
 $MaxParallelJobs = 16
 
+$MaxRetries = 5
+
 # Wait time between check for system load
 $SecondsBetweenLoadChecks = 10
 
@@ -29,7 +31,10 @@ $ValidatePackage = {
   # Ensure input file exist
   if (!(Test-Path $PackagePath)) {
     Write-Host "Input file does not exist: $PackagePath"
-    return 1
+    return [pscustomobject]@{
+      result = 1
+      packagePath = $PackagePath
+    }
   }
 
   # Extensions for which we'll look for SourceLink information
@@ -59,7 +64,10 @@ $ValidatePackage = {
 
           # We ignore resource DLLs
           if ($FileName.EndsWith('.resources.dll')) {
-            return
+            return [pscustomobject]@{
+              result = 0
+              packagePath = $PackagePath
+            }
           }
 
           [System.IO.Compression.ZipFileExtensions]::ExtractToFile($_, $TargetFile, $true)
@@ -91,36 +99,49 @@ $ValidatePackage = {
                     $Status = 200
                     $Cache = $using:RepoFiles
 
-                    if ( !($Cache.ContainsKey($FilePath)) ) {
-                      try {
-                        $Uri = $Link -as [System.URI]
-                      
-                        # Only GitHub links are valid
-                        if ($Uri.AbsoluteURI -ne $null -and ($Uri.Host -match 'github' -or $Uri.Host -match 'githubusercontent')) {
-                          $Status = (Invoke-WebRequest -Uri $Link -UseBasicParsing -Method HEAD -TimeoutSec 5).StatusCode
+                    $totalRetries = 0
+
+                    while ($totalRetries -lt $using:MaxRetries) {
+                      if ( !($Cache.ContainsKey($FilePath)) ) {
+                        try {
+                          $Uri = $Link -as [System.URI]
+                        
+                          # Only GitHub links are valid
+                          if ($Uri.AbsoluteURI -ne $null -and ($Uri.Host -match 'github' -or $Uri.Host -match 'githubusercontent')) {
+                            $Status = (Invoke-WebRequest -Uri $Link -UseBasicParsing -Method HEAD -TimeoutSec 5).StatusCode
+                          }
+                          else {
+                            # If it's not a github link, we want to break out of the loop and not retry.
+                            $Status = 0
+                            $totalRetries = $using:MaxRetries
+                          }
                         }
-                        else {
+                        catch {
+                          Write-Host $_
                           $Status = 0
                         }
                       }
-                      catch {
-                        write-host $_
-                        $Status = 0
-                      }
-                    }
 
-                    if ($Status -ne 200) {
-                      if ($NumFailedLinks -eq 0) {
-                        if ($FailedFiles.Value -eq 0) {
-                          Write-Host
+                      if ($Status -ne 200) {
+                        $totalRetries++
+                        
+                        if ($totalRetries -ge $using:MaxRetries) {
+                          if ($NumFailedLinks -eq 0) {
+                            if ($FailedFiles.Value -eq 0) {
+                              Write-Host
+                            }
+  
+                            Write-Host "`tFile $RealPath has broken links:"
+                          }
+  
+                          Write-Host "`t`tFailed to retrieve $Link"
+  
+                          $NumFailedLinks++
                         }
-
-                        Write-Host "`tFile $RealPath has broken links:"
                       }
-
-                      Write-Host "`t`tFailed to retrieve $Link"
-
-                      $NumFailedLinks++
+                      else {
+                        break
+                      }
                     }
                   }
               }
@@ -136,7 +157,7 @@ $ValidatePackage = {
         }
   }
   catch {
-  
+    Write-Host $_
   }
   finally {
     $zip.Dispose() 
@@ -220,6 +241,7 @@ function ValidateSourceLinkLinks {
   # Process each NuGet package in parallel
   Get-ChildItem "$InputPath\*.symbols.nupkg" |
     ForEach-Object {
+      Write-Host "Starting $($_.FullName)"
       Start-Job -ScriptBlock $ValidatePackage -ArgumentList $_.FullName | Out-Null
       $NumJobs = @(Get-Job -State 'Running').Count
       
@@ -267,6 +289,10 @@ function InstallSourcelinkCli {
 try {
   InstallSourcelinkCli
 
+  foreach ($Job in @(Get-Job)) {
+    Remove-Job -Id $Job.Id
+  }
+
   ValidateSourceLinkLinks 
 }
 catch {

eng/common/post-build/symbols-validation.ps1

@@ -114,6 +114,7 @@ $CountMissingSymbols = {
       $totalRetries = 0
 
       while ($totalRetries -lt $using:MaxRetry) {
+
         # Save the output and get diagnostic output
         $output = & $dotnetSymbolExe --symbols --modules $WindowsPdbVerificationParam $TargetServerParam $FullPath -o $SymbolsPath --diagnostics | Out-String
 
@@ -132,20 +133,28 @@ $CountMissingSymbols = {
         elseif (Test-Path $SymbolPath) {
           return 'Module'
         }
-        elseif ($output.Contains("503 Service Unavailable")) {
-          # If we got a 503 error, we should retry.
+        else
+        {
           $totalRetries++
         }
-        else {
-          return $null
-        }
       }
       
       return $null
     }
 
-    $SymbolsOnMSDL = & $FirstMatchingSymbolDescriptionOrDefault $FileName '--microsoft-symbol-server' $SymbolsPath $WindowsPdbVerificationParam
-    $SymbolsOnSymWeb = & $FirstMatchingSymbolDescriptionOrDefault $FileName '--internal-server' $SymbolsPath $WindowsPdbVerificationParam
+    $FileGuid = New-Guid
+    $ExpandedSymbolsPath = Join-Path -Path $SymbolsPath -ChildPath $FileGuid
+
+    $SymbolsOnMSDL = & $FirstMatchingSymbolDescriptionOrDefault `
+        -FullPath $FileName `
+        -TargetServerParam '--microsoft-symbol-server' `
+        -SymbolsPath "$ExpandedSymbolsPath-msdl" `
+        -WindowsPdbVerificationParam $WindowsPdbVerificationParam
+    $SymbolsOnSymWeb = & $FirstMatchingSymbolDescriptionOrDefault `
+        -FullPath $FileName `
+        -TargetServerParam '--internal-server' `
+        -SymbolsPath "$ExpandedSymbolsPath-symweb" `
+        -WindowsPdbVerificationParam $WindowsPdbVerificationParam
 
     Write-Host -NoNewLine "`t Checking file " $FileName "... "
   
@@ -208,6 +217,7 @@ function CheckSymbolsAvailable {
     Remove-Item $ExtractPath -Force  -Recurse -ErrorAction SilentlyContinue
   }
 
+  $TotalPackages = 0
   $TotalFailures = 0
   $DupedSymbols = 0
 
@@ -230,6 +240,8 @@ function CheckSymbolsAvailable {
         return
       }
 
+      $TotalPackages++
+
       Start-Job -ScriptBlock $CountMissingSymbols -ArgumentList @($FullName,$WindowsPdbVerificationParam) | Out-Null
 
       $NumJobs = @(Get-Job -State 'Running').Count
@@ -255,11 +267,11 @@ function CheckSymbolsAvailable {
 
   if ($TotalFailures -gt 0 -or $DupedSymbols -gt 0) {
     if ($TotalFailures -gt 0) {
-      Write-PipelineTelemetryError -Category 'CheckSymbols' -Message "Symbols missing for $TotalFailures packages"
+      Write-PipelineTelemetryError -Category 'CheckSymbols' -Message "Symbols missing for $TotalFailures/$TotalPackages packages"
     }
 
     if ($DupedSymbols -gt 0) {
-      Write-PipelineTelemetryError -Category 'CheckSymbols' -Message "$DupedSymbols packages had duplicated symbol files"
+      Write-PipelineTelemetryError -Category 'CheckSymbols' -Message "$DupedSymbols/$TotalPackages packages had duplicated symbol files and could not be extracted"
     }
     
     ExitWithExitCode 1

eng/common/templates/job/onelocbuild.yml

@@ -12,6 +12,7 @@ parameters:
   SourcesDirectory: $(Build.SourcesDirectory)
   CreatePr: true
   AutoCompletePr: false
+  UseLfLineEndings: true
   UseCheckedInLocProjectJson: false
   LanguageSet: VS_Main_Languages
   LclSource: lclFilesInRepo
@@ -52,13 +53,14 @@ jobs:
       env:
         SYSTEM_ACCESSTOKEN: $(System.AccessToken)
       inputs:
-        locProj: Localize/LocProject.json
+        locProj: eng/Localize/LocProject.json
         outDir: $(Build.ArtifactStagingDirectory)
         lclSource: ${{ parameters.LclSource }}
         lclPackageId: ${{ parameters.LclPackageId }}
         isCreatePrSelected: ${{ parameters.CreatePr }}
         ${{ if eq(parameters.CreatePr, true) }}:
           isAutoCompletePrSelected: ${{ parameters.AutoCompletePr }}
+          isUseLfLineEndingsSelected: ${{ parameters.UseLfLineEndings }}
         packageSourceAuth: patAuth
         patVariable: ${{ parameters.CeapexPat }}
         ${{ if eq(parameters.RepoType, 'gitHub') }}:
@@ -77,7 +79,7 @@ jobs:
     - task: PublishBuildArtifacts@1
       displayName: Publish LocProject.json
       inputs:
-        PathtoPublish: '$(Build.SourcesDirectory)/Localize/'
+        PathtoPublish: '$(Build.SourcesDirectory)/eng/Localize/'
         PublishLocation: Container
         ArtifactName: Loc
       condition: ${{ parameters.condition }}

eng/common/templates/job/source-build.yml

@@ -15,6 +15,9 @@ parameters:
   # nonPortable: false
   #   Enables non-portable mode. This means a more specific RID (e.g. fedora.32-x64 rather than
   #   linux-x64), and compiling against distro-provided packages rather than portable ones.
+  # skipPublishValidation: false
+  #   Disables publishing validation.  By default, a check is performed to ensure no packages are
+  #   published by source-build.
   # container: ''
   #   A container to use. Runs in docker.
   # pool: {}

eng/common/templates/post-build/channels/generic-internal-channel.yml

@@ -40,6 +40,9 @@ stages:
     pool:
       vmImage: 'windows-2019'
     steps:
+      - script: echo "##vso[task.logissue type=warning]Going forward, v2 Arcade publishing is no longer supported. Please read https://github.com/dotnet/arcade/blob/main/Documentation/CorePackages/Publishing.md for details, then contact dnceng if you have further questions."
+        displayName: Warn about v2 Arcade Publishing Usage
+
       # This is necessary whenever we want to publish/restore to an AzDO private feed
       - task: NuGetAuthenticate@0
         displayName: 'Authenticate to AzDO Feeds'
@@ -110,6 +113,9 @@ stages:
     pool:
       vmImage: 'windows-2019'
     steps:
+      - script: echo "##vso[task.logissue type=warning]Going forward, v2 Arcade publishing is no longer supported. Please read https://github.com/dotnet/arcade/blob/main/Documentation/CorePackages/Publishing.md for details, then contact dnceng if you have further questions."
+        displayName: Warn about v2 Arcade Publishing Usage
+
       - task: DownloadBuildArtifacts@0
         displayName: Download Build Assets
         continueOnError: true

eng/common/templates/post-build/channels/generic-public-channel.yml

@@ -42,6 +42,9 @@ stages:
     pool:
       vmImage: 'windows-2019'
     steps:
+      - script: echo "##vso[task.logissue type=warning]Going forward, v2 Arcade publishing is no longer supported. Please read https://github.com/dotnet/arcade/blob/main/Documentation/CorePackages/Publishing.md for details, then contact dnceng if you have further questions."
+        displayName: Warn about v2 Arcade Publishing Usage
+
       - task: DownloadBuildArtifacts@0
         displayName: Download Build Assets
         continueOnError: true
@@ -109,6 +112,9 @@ stages:
     pool:
       vmImage: 'windows-2019'
     steps:
+      - script: echo "##vso[task.logissue type=warning]Going forward, v2 Arcade publishing is no longer supported. Please read https://github.com/dotnet/arcade/blob/main/Documentation/CorePackages/Publishing.md for details, then contact dnceng if you have further questions."
+        displayName: Warn about v2 Arcade Publishing Usage
+
       - task: DownloadBuildArtifacts@0
         displayName: Download Build Assets
         continueOnError: true

eng/common/templates/steps/source-build.yml

@@ -34,9 +34,14 @@ steps:
       targetRidArgs='/p:TargetRid=${{ parameters.platform.targetRID }}'
     fi
 
+    publishArgs=
+    if [ '${{ parameters.platform.skipPublishValidation }}' != 'true' ]; then
+      publishArgs='--publish'
+    fi
+
     ${{ coalesce(parameters.platform.buildScript, './build.sh') }} --ci \
       --configuration $buildConfig \
-      --restore --build --pack --publish -bl \
+      --restore --build --pack $publishArgs -bl \
       $officialBuildArgs \
       $targetRidArgs \
       /p:SourceBuildNonPortable=${{ parameters.platform.nonPortable }} \

eng/common/tools.ps1

@@ -48,6 +48,9 @@
 # True to use global NuGet cache instead of restoring packages to repository-local directory.
 [bool]$useGlobalNuGetCache = if (Test-Path variable:useGlobalNuGetCache) { $useGlobalNuGetCache } else { !$ci }
 
+# True to exclude prerelease versions Visual Studio during build
+[bool]$excludePrereleaseVS = if (Test-Path variable:excludePrereleaseVS) { $excludePrereleaseVS } else { $false }
+
 # An array of names of processes to stop on script exit if prepareMachine is true.
 $processesToStopOnExit = if (Test-Path variable:processesToStopOnExit) { $processesToStopOnExit } else { @('msbuild', 'dotnet', 'vbcscompiler') }
 
@@ -463,7 +466,11 @@ function LocateVisualStudio([object]$vsRequirements = $null){
   }
 
   if (!$vsRequirements) { $vsRequirements = $GlobalJson.tools.vs }
-  $args = @('-latest', '-prerelease', '-format', 'json', '-requires', 'Microsoft.Component.MSBuild', '-products', '*')
+  $args = @('-latest', '-format', 'json', '-requires', 'Microsoft.Component.MSBuild', '-products', '*')
+
+  if (!$excludePrereleaseVS) {
+    $args += '-prerelease'
+  }
 
   if (Get-Member -InputObject $vsRequirements -Name 'version') {
     $args += '-version'
@@ -489,7 +496,13 @@ function LocateVisualStudio([object]$vsRequirements = $null){
 
 function InitializeBuildTool() {
   if (Test-Path variable:global:_BuildTool) {
-    return $global:_BuildTool
+    # If the requested msbuild parameters do not match, clear the cached variables.
+    if($global:_BuildTool.Contains('ExcludePrereleaseVS') -and $global:_BuildTool.ExcludePrereleaseVS -ne $excludePrereleaseVS) {
+      Remove-Item variable:global:_BuildTool 
+      Remove-Item variable:global:_MSBuildExe
+    } else {
+      return $global:_BuildTool
+    }
   }
 
   if (-not $msbuildEngine) {
@@ -517,7 +530,7 @@ function InitializeBuildTool() {
       ExitWithExitCode 1
     }
 
-    $buildTool = @{ Path = $msbuildPath; Command = ""; Tool = "vs"; Framework = "net472" }
+    $buildTool = @{ Path = $msbuildPath; Command = ""; Tool = "vs"; Framework = "net472"; ExcludePrereleaseVS = $excludePrereleaseVS }
   } else {
     Write-PipelineTelemetryError -Category 'InitializeToolset' -Message "Unexpected value of -msbuildEngine: '$msbuildEngine'."
     ExitWithExitCode 1

global.json

@@ -11,6 +11,6 @@
     }
   },
   "msbuild-sdks": {
-    "Microsoft.DotNet.Arcade.Sdk": "6.0.0-beta.21253.2"
+    "Microsoft.DotNet.Arcade.Sdk": "6.0.0-beta.21274.6"
   }
 }