diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index 15a9b473..97f266d3 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -3,25 +3,25 @@ - + https://github.com/dotnet/arcade - e9fd640e1ec1890489ea66ad0f59e733448056da + f22435d0f9cc367e75e1875d93ef980be1be6b79 - + https://github.com/dotnet/arcade - e9fd640e1ec1890489ea66ad0f59e733448056da + f22435d0f9cc367e75e1875d93ef980be1be6b79 - + https://github.com/dotnet/arcade - e9fd640e1ec1890489ea66ad0f59e733448056da + f22435d0f9cc367e75e1875d93ef980be1be6b79 - + https://github.com/dotnet/arcade - e9fd640e1ec1890489ea66ad0f59e733448056da + f22435d0f9cc367e75e1875d93ef980be1be6b79 - + https://github.com/dotnet/arcade - e9fd640e1ec1890489ea66ad0f59e733448056da + f22435d0f9cc367e75e1875d93ef980be1be6b79 https://github.com/dotnet/arcade-services diff --git a/eng/common/build.ps1 b/eng/common/build.ps1 index 94a91c08..8943da24 100644 --- a/eng/common/build.ps1 +++ b/eng/common/build.ps1 @@ -25,6 +25,7 @@ Param( [switch] $prepareMachine, [string] $runtimeSourceFeed = '', [string] $runtimeSourceFeedKey = '', + [switch] $excludePrereleaseVS, [switch] $help, [Parameter(ValueFromRemainingArguments=$true)][String[]]$properties ) @@ -65,6 +66,7 @@ function Print-Usage() { Write-Host " -prepareMachine Prepare machine for CI run, clean up processes after build" Write-Host " -warnAsError Sets warnaserror msbuild parameter ('true' or 'false')" Write-Host " -msbuildEngine Msbuild engine to use to run build ('dotnet', 'vs', or unspecified)." + Write-Host " -excludePrereleaseVS Set to exclude build engines in prerelease versions of Visual Studio" Write-Host "" Write-Host "Command line arguments not listed above are passed thru to msbuild." diff --git a/eng/common/cross/build-android-rootfs.sh b/eng/common/cross/build-android-rootfs.sh index c29c8267..42516bbe 100755 --- a/eng/common/cross/build-android-rootfs.sh +++ b/eng/common/cross/build-android-rootfs.sh @@ -106,7 +106,6 @@ __AndroidPackages+=" libandroid-glob" __AndroidPackages+=" liblzma" __AndroidPackages+=" krb5" __AndroidPackages+=" openssl" -__AndroidPackages+=" openldap" for path in $(wget -qO- http://termux.net/dists/stable/main/binary-$__AndroidArch/Packages |\ grep -A15 "Package: \(${__AndroidPackages// /\\|}\)" | grep -v "static\|tool" | grep Filename); do diff --git a/eng/common/cross/build-rootfs.sh b/eng/common/cross/build-rootfs.sh index 81e641a5..591d8666 100755 --- a/eng/common/cross/build-rootfs.sh +++ b/eng/common/cross/build-rootfs.sh @@ -55,13 +55,11 @@ __UbuntuPackages+=" libcurl4-openssl-dev" __UbuntuPackages+=" libkrb5-dev" __UbuntuPackages+=" libssl-dev" __UbuntuPackages+=" zlib1g-dev" -__UbuntuPackages+=" libldap2-dev" __AlpinePackages+=" curl-dev" __AlpinePackages+=" krb5-dev" __AlpinePackages+=" openssl-dev" __AlpinePackages+=" zlib-dev" -__AlpinePackages+=" openldap-dev" __FreeBSDBase="12.1-RELEASE" __FreeBSDPkg="1.12.0" @@ -70,13 +68,15 @@ __FreeBSDPackages+=" icu" __FreeBSDPackages+=" libinotify" __FreeBSDPackages+=" lttng-ust" __FreeBSDPackages+=" krb5" -__FreeBSDPackages+=" libslapi-2.4" __IllumosPackages="icu-64.2nb2" __IllumosPackages+=" mit-krb5-1.16.2nb4" __IllumosPackages+=" openssl-1.1.1e" __IllumosPackages+=" zlib-1.2.11" -__IllumosPackages+=" openldap-client-2.4.49" + +# ML.NET dependencies +__UbuntuPackages+=" libomp5" +__UbuntuPackages+=" libomp-dev" __UseMirror=0 diff --git a/eng/common/generate-locproject.ps1 b/eng/common/generate-locproject.ps1 index 24c00b5b..de348a2e 100644 --- a/eng/common/generate-locproject.ps1 +++ b/eng/common/generate-locproject.ps1 @@ -14,7 +14,7 @@ $ErrorActionPreference = "Stop" Import-Module -Name (Join-Path $PSScriptRoot 'native\CommonLibrary.psm1') -$exclusionsFilePath = "$SourcesDirectory\Localize\LocExclusions.json" +$exclusionsFilePath = "$SourcesDirectory\eng\Localize\LocExclusions.json" $exclusions = @{ Exclusions = @() } if (Test-Path -Path $exclusionsFilePath) { @@ -92,14 +92,14 @@ Write-Host "LocProject.json generated:`n`n$json`n`n" Pop-Location if (!$UseCheckedInLocProjectJson) { - New-Item "$SourcesDirectory\Localize\LocProject.json" -Force # Need this to make sure the Localize directory is created - Set-Content "$SourcesDirectory\Localize\LocProject.json" $json + New-Item "$SourcesDirectory\eng\Localize\LocProject.json" -Force # Need this to make sure the Localize directory is created + Set-Content "$SourcesDirectory\eng\Localize\LocProject.json" $json } else { - New-Item "$SourcesDirectory\Localize\LocProject-generated.json" -Force # Need this to make sure the Localize directory is created - Set-Content "$SourcesDirectory\Localize\LocProject-generated.json" $json + New-Item "$SourcesDirectory\eng\Localize\LocProject-generated.json" -Force # Need this to make sure the Localize directory is created + Set-Content "$SourcesDirectory\eng\Localize\LocProject-generated.json" $json - if ((Get-FileHash "$SourcesDirectory\Localize\LocProject-generated.json").Hash -ne (Get-FileHash "$SourcesDirectory\Localize\LocProject.json").Hash) { + if ((Get-FileHash "$SourcesDirectory\eng\Localize\LocProject-generated.json").Hash -ne (Get-FileHash "$SourcesDirectory\eng\Localize\LocProject.json").Hash) { Write-PipelineTelemetryError -Category "OneLocBuild" -Message "Existing LocProject.json differs from generated LocProject.json. Download LocProject-generated.json and compare them." exit 1 diff --git a/eng/common/msbuild.ps1 b/eng/common/msbuild.ps1 index c6401230..eea19cd8 100644 --- a/eng/common/msbuild.ps1 +++ b/eng/common/msbuild.ps1 @@ -5,6 +5,7 @@ Param( [bool] $nodeReuse = $true, [switch] $ci, [switch] $prepareMachine, + [switch] $excludePrereleaseVS, [Parameter(ValueFromRemainingArguments=$true)][String[]]$extraArgs ) diff --git a/eng/common/post-build/sourcelink-validation.ps1 b/eng/common/post-build/sourcelink-validation.ps1 index 8c554729..85c89861 100644 --- a/eng/common/post-build/sourcelink-validation.ps1 +++ b/eng/common/post-build/sourcelink-validation.ps1 @@ -16,6 +16,8 @@ $global:RepoFiles = @{} # Maximum number of jobs to run in parallel $MaxParallelJobs = 16 +$MaxRetries = 5 + # Wait time between check for system load $SecondsBetweenLoadChecks = 10 @@ -29,7 +31,10 @@ $ValidatePackage = { # Ensure input file exist if (!(Test-Path $PackagePath)) { Write-Host "Input file does not exist: $PackagePath" - return 1 + return [pscustomobject]@{ + result = 1 + packagePath = $PackagePath + } } # Extensions for which we'll look for SourceLink information @@ -59,7 +64,10 @@ $ValidatePackage = { # We ignore resource DLLs if ($FileName.EndsWith('.resources.dll')) { - return + return [pscustomobject]@{ + result = 0 + packagePath = $PackagePath + } } [System.IO.Compression.ZipFileExtensions]::ExtractToFile($_, $TargetFile, $true) @@ -91,36 +99,49 @@ $ValidatePackage = { $Status = 200 $Cache = $using:RepoFiles - if ( !($Cache.ContainsKey($FilePath)) ) { - try { - $Uri = $Link -as [System.URI] - - # Only GitHub links are valid - if ($Uri.AbsoluteURI -ne $null -and ($Uri.Host -match 'github' -or $Uri.Host -match 'githubusercontent')) { - $Status = (Invoke-WebRequest -Uri $Link -UseBasicParsing -Method HEAD -TimeoutSec 5).StatusCode + $totalRetries = 0 + + while ($totalRetries -lt $using:MaxRetries) { + if ( !($Cache.ContainsKey($FilePath)) ) { + try { + $Uri = $Link -as [System.URI] + + # Only GitHub links are valid + if ($Uri.AbsoluteURI -ne $null -and ($Uri.Host -match 'github' -or $Uri.Host -match 'githubusercontent')) { + $Status = (Invoke-WebRequest -Uri $Link -UseBasicParsing -Method HEAD -TimeoutSec 5).StatusCode + } + else { + # If it's not a github link, we want to break out of the loop and not retry. + $Status = 0 + $totalRetries = $using:MaxRetries + } } - else { + catch { + Write-Host $_ $Status = 0 } } - catch { - write-host $_ - $Status = 0 - } - } - if ($Status -ne 200) { - if ($NumFailedLinks -eq 0) { - if ($FailedFiles.Value -eq 0) { - Write-Host + if ($Status -ne 200) { + $totalRetries++ + + if ($totalRetries -ge $using:MaxRetries) { + if ($NumFailedLinks -eq 0) { + if ($FailedFiles.Value -eq 0) { + Write-Host + } + + Write-Host "`tFile $RealPath has broken links:" + } + + Write-Host "`t`tFailed to retrieve $Link" + + $NumFailedLinks++ } - - Write-Host "`tFile $RealPath has broken links:" } - - Write-Host "`t`tFailed to retrieve $Link" - - $NumFailedLinks++ + else { + break + } } } } @@ -136,7 +157,7 @@ $ValidatePackage = { } } catch { - + Write-Host $_ } finally { $zip.Dispose() @@ -220,6 +241,7 @@ function ValidateSourceLinkLinks { # Process each NuGet package in parallel Get-ChildItem "$InputPath\*.symbols.nupkg" | ForEach-Object { + Write-Host "Starting $($_.FullName)" Start-Job -ScriptBlock $ValidatePackage -ArgumentList $_.FullName | Out-Null $NumJobs = @(Get-Job -State 'Running').Count @@ -267,6 +289,10 @@ function InstallSourcelinkCli { try { InstallSourcelinkCli + foreach ($Job in @(Get-Job)) { + Remove-Job -Id $Job.Id + } + ValidateSourceLinkLinks } catch { diff --git a/eng/common/post-build/symbols-validation.ps1 b/eng/common/post-build/symbols-validation.ps1 index 788321d7..a5af041b 100644 --- a/eng/common/post-build/symbols-validation.ps1 +++ b/eng/common/post-build/symbols-validation.ps1 @@ -114,6 +114,7 @@ $CountMissingSymbols = { $totalRetries = 0 while ($totalRetries -lt $using:MaxRetry) { + # Save the output and get diagnostic output $output = & $dotnetSymbolExe --symbols --modules $WindowsPdbVerificationParam $TargetServerParam $FullPath -o $SymbolsPath --diagnostics | Out-String @@ -132,20 +133,28 @@ $CountMissingSymbols = { elseif (Test-Path $SymbolPath) { return 'Module' } - elseif ($output.Contains("503 Service Unavailable")) { - # If we got a 503 error, we should retry. + else + { $totalRetries++ } - else { - return $null - } } return $null } - $SymbolsOnMSDL = & $FirstMatchingSymbolDescriptionOrDefault $FileName '--microsoft-symbol-server' $SymbolsPath $WindowsPdbVerificationParam - $SymbolsOnSymWeb = & $FirstMatchingSymbolDescriptionOrDefault $FileName '--internal-server' $SymbolsPath $WindowsPdbVerificationParam + $FileGuid = New-Guid + $ExpandedSymbolsPath = Join-Path -Path $SymbolsPath -ChildPath $FileGuid + + $SymbolsOnMSDL = & $FirstMatchingSymbolDescriptionOrDefault ` + -FullPath $FileName ` + -TargetServerParam '--microsoft-symbol-server' ` + -SymbolsPath "$ExpandedSymbolsPath-msdl" ` + -WindowsPdbVerificationParam $WindowsPdbVerificationParam + $SymbolsOnSymWeb = & $FirstMatchingSymbolDescriptionOrDefault ` + -FullPath $FileName ` + -TargetServerParam '--internal-server' ` + -SymbolsPath "$ExpandedSymbolsPath-symweb" ` + -WindowsPdbVerificationParam $WindowsPdbVerificationParam Write-Host -NoNewLine "`t Checking file " $FileName "... " @@ -208,6 +217,7 @@ function CheckSymbolsAvailable { Remove-Item $ExtractPath -Force -Recurse -ErrorAction SilentlyContinue } + $TotalPackages = 0 $TotalFailures = 0 $DupedSymbols = 0 @@ -230,6 +240,8 @@ function CheckSymbolsAvailable { return } + $TotalPackages++ + Start-Job -ScriptBlock $CountMissingSymbols -ArgumentList @($FullName,$WindowsPdbVerificationParam) | Out-Null $NumJobs = @(Get-Job -State 'Running').Count @@ -255,11 +267,11 @@ function CheckSymbolsAvailable { if ($TotalFailures -gt 0 -or $DupedSymbols -gt 0) { if ($TotalFailures -gt 0) { - Write-PipelineTelemetryError -Category 'CheckSymbols' -Message "Symbols missing for $TotalFailures packages" + Write-PipelineTelemetryError -Category 'CheckSymbols' -Message "Symbols missing for $TotalFailures/$TotalPackages packages" } if ($DupedSymbols -gt 0) { - Write-PipelineTelemetryError -Category 'CheckSymbols' -Message "$DupedSymbols packages had duplicated symbol files" + Write-PipelineTelemetryError -Category 'CheckSymbols' -Message "$DupedSymbols/$TotalPackages packages had duplicated symbol files and could not be extracted" } ExitWithExitCode 1 diff --git a/eng/common/templates/job/onelocbuild.yml b/eng/common/templates/job/onelocbuild.yml index b27d6faf..2acdd525 100644 --- a/eng/common/templates/job/onelocbuild.yml +++ b/eng/common/templates/job/onelocbuild.yml @@ -12,6 +12,7 @@ parameters: SourcesDirectory: $(Build.SourcesDirectory) CreatePr: true AutoCompletePr: false + UseLfLineEndings: true UseCheckedInLocProjectJson: false LanguageSet: VS_Main_Languages LclSource: lclFilesInRepo @@ -52,13 +53,14 @@ jobs: env: SYSTEM_ACCESSTOKEN: $(System.AccessToken) inputs: - locProj: Localize/LocProject.json + locProj: eng/Localize/LocProject.json outDir: $(Build.ArtifactStagingDirectory) lclSource: ${{ parameters.LclSource }} lclPackageId: ${{ parameters.LclPackageId }} isCreatePrSelected: ${{ parameters.CreatePr }} ${{ if eq(parameters.CreatePr, true) }}: isAutoCompletePrSelected: ${{ parameters.AutoCompletePr }} + isUseLfLineEndingsSelected: ${{ parameters.UseLfLineEndings }} packageSourceAuth: patAuth patVariable: ${{ parameters.CeapexPat }} ${{ if eq(parameters.RepoType, 'gitHub') }}: @@ -77,7 +79,7 @@ jobs: - task: PublishBuildArtifacts@1 displayName: Publish LocProject.json inputs: - PathtoPublish: '$(Build.SourcesDirectory)/Localize/' + PathtoPublish: '$(Build.SourcesDirectory)/eng/Localize/' PublishLocation: Container ArtifactName: Loc condition: ${{ parameters.condition }} \ No newline at end of file diff --git a/eng/common/templates/job/source-build.yml b/eng/common/templates/job/source-build.yml index aad41464..5023d36d 100644 --- a/eng/common/templates/job/source-build.yml +++ b/eng/common/templates/job/source-build.yml @@ -15,6 +15,9 @@ parameters: # nonPortable: false # Enables non-portable mode. This means a more specific RID (e.g. fedora.32-x64 rather than # linux-x64), and compiling against distro-provided packages rather than portable ones. + # skipPublishValidation: false + # Disables publishing validation. By default, a check is performed to ensure no packages are + # published by source-build. # container: '' # A container to use. Runs in docker. # pool: {} diff --git a/eng/common/templates/post-build/channels/generic-internal-channel.yml b/eng/common/templates/post-build/channels/generic-internal-channel.yml index 58fa9a35..8990dfc8 100644 --- a/eng/common/templates/post-build/channels/generic-internal-channel.yml +++ b/eng/common/templates/post-build/channels/generic-internal-channel.yml @@ -40,6 +40,9 @@ stages: pool: vmImage: 'windows-2019' steps: + - script: echo "##vso[task.logissue type=warning]Going forward, v2 Arcade publishing is no longer supported. Please read https://github.com/dotnet/arcade/blob/main/Documentation/CorePackages/Publishing.md for details, then contact dnceng if you have further questions." + displayName: Warn about v2 Arcade Publishing Usage + # This is necessary whenever we want to publish/restore to an AzDO private feed - task: NuGetAuthenticate@0 displayName: 'Authenticate to AzDO Feeds' @@ -110,6 +113,9 @@ stages: pool: vmImage: 'windows-2019' steps: + - script: echo "##vso[task.logissue type=warning]Going forward, v2 Arcade publishing is no longer supported. Please read https://github.com/dotnet/arcade/blob/main/Documentation/CorePackages/Publishing.md for details, then contact dnceng if you have further questions." + displayName: Warn about v2 Arcade Publishing Usage + - task: DownloadBuildArtifacts@0 displayName: Download Build Assets continueOnError: true diff --git a/eng/common/templates/post-build/channels/generic-public-channel.yml b/eng/common/templates/post-build/channels/generic-public-channel.yml index b50c0b3b..3220c6a4 100644 --- a/eng/common/templates/post-build/channels/generic-public-channel.yml +++ b/eng/common/templates/post-build/channels/generic-public-channel.yml @@ -42,6 +42,9 @@ stages: pool: vmImage: 'windows-2019' steps: + - script: echo "##vso[task.logissue type=warning]Going forward, v2 Arcade publishing is no longer supported. Please read https://github.com/dotnet/arcade/blob/main/Documentation/CorePackages/Publishing.md for details, then contact dnceng if you have further questions." + displayName: Warn about v2 Arcade Publishing Usage + - task: DownloadBuildArtifacts@0 displayName: Download Build Assets continueOnError: true @@ -109,6 +112,9 @@ stages: pool: vmImage: 'windows-2019' steps: + - script: echo "##vso[task.logissue type=warning]Going forward, v2 Arcade publishing is no longer supported. Please read https://github.com/dotnet/arcade/blob/main/Documentation/CorePackages/Publishing.md for details, then contact dnceng if you have further questions." + displayName: Warn about v2 Arcade Publishing Usage + - task: DownloadBuildArtifacts@0 displayName: Download Build Assets continueOnError: true diff --git a/eng/common/templates/steps/source-build.yml b/eng/common/templates/steps/source-build.yml index 65ee5992..e20637ed 100644 --- a/eng/common/templates/steps/source-build.yml +++ b/eng/common/templates/steps/source-build.yml @@ -34,9 +34,14 @@ steps: targetRidArgs='/p:TargetRid=${{ parameters.platform.targetRID }}' fi + publishArgs= + if [ '${{ parameters.platform.skipPublishValidation }}' != 'true' ]; then + publishArgs='--publish' + fi + ${{ coalesce(parameters.platform.buildScript, './build.sh') }} --ci \ --configuration $buildConfig \ - --restore --build --pack --publish -bl \ + --restore --build --pack $publishArgs -bl \ $officialBuildArgs \ $targetRidArgs \ /p:SourceBuildNonPortable=${{ parameters.platform.nonPortable }} \ diff --git a/eng/common/tools.ps1 b/eng/common/tools.ps1 index d52467ee..2d8a74f7 100644 --- a/eng/common/tools.ps1 +++ b/eng/common/tools.ps1 @@ -48,6 +48,9 @@ # True to use global NuGet cache instead of restoring packages to repository-local directory. [bool]$useGlobalNuGetCache = if (Test-Path variable:useGlobalNuGetCache) { $useGlobalNuGetCache } else { !$ci } +# True to exclude prerelease versions Visual Studio during build +[bool]$excludePrereleaseVS = if (Test-Path variable:excludePrereleaseVS) { $excludePrereleaseVS } else { $false } + # An array of names of processes to stop on script exit if prepareMachine is true. $processesToStopOnExit = if (Test-Path variable:processesToStopOnExit) { $processesToStopOnExit } else { @('msbuild', 'dotnet', 'vbcscompiler') } @@ -463,7 +466,11 @@ function LocateVisualStudio([object]$vsRequirements = $null){ } if (!$vsRequirements) { $vsRequirements = $GlobalJson.tools.vs } - $args = @('-latest', '-prerelease', '-format', 'json', '-requires', 'Microsoft.Component.MSBuild', '-products', '*') + $args = @('-latest', '-format', 'json', '-requires', 'Microsoft.Component.MSBuild', '-products', '*') + + if (!$excludePrereleaseVS) { + $args += '-prerelease' + } if (Get-Member -InputObject $vsRequirements -Name 'version') { $args += '-version' @@ -489,7 +496,13 @@ function LocateVisualStudio([object]$vsRequirements = $null){ function InitializeBuildTool() { if (Test-Path variable:global:_BuildTool) { - return $global:_BuildTool + # If the requested msbuild parameters do not match, clear the cached variables. + if($global:_BuildTool.Contains('ExcludePrereleaseVS') -and $global:_BuildTool.ExcludePrereleaseVS -ne $excludePrereleaseVS) { + Remove-Item variable:global:_BuildTool + Remove-Item variable:global:_MSBuildExe + } else { + return $global:_BuildTool + } } if (-not $msbuildEngine) { @@ -517,7 +530,7 @@ function InitializeBuildTool() { ExitWithExitCode 1 } - $buildTool = @{ Path = $msbuildPath; Command = ""; Tool = "vs"; Framework = "net472" } + $buildTool = @{ Path = $msbuildPath; Command = ""; Tool = "vs"; Framework = "net472"; ExcludePrereleaseVS = $excludePrereleaseVS } } else { Write-PipelineTelemetryError -Category 'InitializeToolset' -Message "Unexpected value of -msbuildEngine: '$msbuildEngine'." ExitWithExitCode 1 diff --git a/global.json b/global.json index c2bb0def..f31f485e 100644 --- a/global.json +++ b/global.json @@ -11,6 +11,6 @@ } }, "msbuild-sdks": { - "Microsoft.DotNet.Arcade.Sdk": "6.0.0-beta.21253.2" + "Microsoft.DotNet.Arcade.Sdk": "6.0.0-beta.21274.6" } }