From 40252471501025f82ef366b4b599377f15bb930e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?K=C3=A9vin=20Chalet?= Date: Wed, 18 Nov 2020 02:36:01 +0100 Subject: [PATCH] Bump Arcade to 5.0.0-beta.20510.1 --- eng/Version.Details.xml | 8 +-- eng/common/cross/arm64/tizen-build-rootfs.sh | 0 eng/common/cross/arm64/tizen-fetch.sh | 0 .../dotnet-install-scripts/dotnet-install.sh | 0 eng/common/internal/Directory.Build.props | 2 +- eng/common/native/find-native-compiler.sh | 0 eng/common/performance/performance-setup.sh | 2 +- eng/common/post-build/publish-using-darc.ps1 | 8 +-- .../post-build/sourcelink-validation.ps1 | 26 ++++++-- eng/common/post-build/symbols-validation.ps1 | 7 +- eng/common/sdk-task.ps1 | 2 +- eng/common/templates/job/job.yml | 12 +++- eng/common/templates/job/source-build.yml | 49 ++++++++++++++ eng/common/templates/jobs/jobs.yml | 16 +++++ eng/common/templates/jobs/source-build.yml | 48 ++++++++++++++ .../channels/generic-internal-channel.yml | 4 +- .../channels/generic-public-channel.yml | 2 +- .../templates/post-build/post-build.yml | 33 +++++----- .../templates/steps/perf-send-to-helix.yml | 36 +++------- eng/common/templates/steps/publish-logs.yml | 27 ++++---- eng/common/templates/steps/send-to-helix.yml | 2 +- eng/common/templates/steps/source-build.yml | 66 +++++++++++++++++++ eng/common/tools.ps1 | 47 +++++++++---- eng/common/tools.sh | 2 + global.json | 4 +- 25 files changed, 306 insertions(+), 97 deletions(-) mode change 100644 => 100755 eng/common/cross/arm64/tizen-build-rootfs.sh mode change 100644 => 100755 eng/common/cross/arm64/tizen-fetch.sh mode change 100644 => 100755 eng/common/dotnet-install-scripts/dotnet-install.sh mode change 100644 => 100755 eng/common/native/find-native-compiler.sh create mode 100644 eng/common/templates/job/source-build.yml create mode 100644 eng/common/templates/jobs/source-build.yml create mode 100644 eng/common/templates/steps/source-build.yml diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index 25d14ef0..42fa203b 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -5,14 +5,14 @@ - + https://github.com/dotnet/arcade - 3b720900ac55e9cc3675584c41da6460d6193f37 + 6813f5aa511a7a4498fa217a54219b5704a01f83 - + https://github.com/dotnet/arcade - 3b720900ac55e9cc3675584c41da6460d6193f37 + 6813f5aa511a7a4498fa217a54219b5704a01f83 diff --git a/eng/common/cross/arm64/tizen-build-rootfs.sh b/eng/common/cross/arm64/tizen-build-rootfs.sh old mode 100644 new mode 100755 diff --git a/eng/common/cross/arm64/tizen-fetch.sh b/eng/common/cross/arm64/tizen-fetch.sh old mode 100644 new mode 100755 diff --git a/eng/common/dotnet-install-scripts/dotnet-install.sh b/eng/common/dotnet-install-scripts/dotnet-install.sh old mode 100644 new mode 100755 diff --git a/eng/common/internal/Directory.Build.props b/eng/common/internal/Directory.Build.props index e33179ef..dbf99d82 100644 --- a/eng/common/internal/Directory.Build.props +++ b/eng/common/internal/Directory.Build.props @@ -1,4 +1,4 @@ - + diff --git a/eng/common/native/find-native-compiler.sh b/eng/common/native/find-native-compiler.sh old mode 100644 new mode 100755 diff --git a/eng/common/performance/performance-setup.sh b/eng/common/performance/performance-setup.sh index 806e56c6..99d1b7bc 100755 --- a/eng/common/performance/performance-setup.sh +++ b/eng/common/performance/performance-setup.sh @@ -201,7 +201,6 @@ if [[ "$internal" == true ]]; then fi if [[ "$mono_dotnet" != "" ]] && [[ "$monointerpreter" == "false" ]]; then - configurations="$configurations LLVM=$llvm MonoInterpreter=$monointerpreter MonoAOT=$monoaot" extra_benchmark_dotnet_arguments="$extra_benchmark_dotnet_arguments --category-exclusion-filter NoMono" fi @@ -211,6 +210,7 @@ if [[ "$wasm_runtime_loc" != "" ]]; then fi if [[ "$mono_dotnet" != "" ]] && [[ "$monointerpreter" == "true" ]]; then + configurations="$configurations LLVM=$llvm MonoInterpreter=$monointerpreter MonoAOT=$monoaot" extra_benchmark_dotnet_arguments="$extra_benchmark_dotnet_arguments --category-exclusion-filter NoInterpreter NoMono" fi diff --git a/eng/common/post-build/publish-using-darc.ps1 b/eng/common/post-build/publish-using-darc.ps1 index f044afe8..650b13b0 100644 --- a/eng/common/post-build/publish-using-darc.ps1 +++ b/eng/common/post-build/publish-using-darc.ps1 @@ -8,7 +8,7 @@ param( [Parameter(Mandatory=$false)][string] $EnableSourceLinkValidation, [Parameter(Mandatory=$false)][string] $EnableSigningValidation, [Parameter(Mandatory=$false)][string] $EnableNugetValidation, - [Parameter(Mandatory=$true)][string] $PublishInstallersAndChecksums, + [Parameter(Mandatory=$false)][string] $PublishInstallersAndChecksums, [Parameter(Mandatory=$false)][string] $ArtifactsPublishingAdditionalParameters, [Parameter(Mandatory=$false)][string] $SigningValidationAdditionalParameters ) @@ -16,7 +16,7 @@ param( try { . $PSScriptRoot\post-build-utils.ps1 # Hard coding darc version till the next arcade-services roll out, cos this version has required API changes for darc add-build-to-channel - . $PSScriptRoot\..\darc-init.ps1 -darcVersion "1.1.0-beta.20418.1" + $darc = Get-Darc "1.1.0-beta.20418.1" $optionalParams = [System.Collections.ArrayList]::new() @@ -29,7 +29,7 @@ try { $optionalParams.Add("--no-wait") | Out-Null } - if ("true" -eq $PublishInstallersAndChecksums) { + if ("false" -ne $PublishInstallersAndChecksums) { $optionalParams.Add("--publish-installers-and-checksums") | Out-Null } @@ -50,7 +50,7 @@ try { } } - & darc add-build-to-channel ` + & $darc add-build-to-channel ` --id $buildId ` --publishing-infra-version $PublishingInfraVersion ` --default-channels ` diff --git a/eng/common/post-build/sourcelink-validation.ps1 b/eng/common/post-build/sourcelink-validation.ps1 index cdd1bba4..c7e7ae67 100644 --- a/eng/common/post-build/sourcelink-validation.ps1 +++ b/eng/common/post-build/sourcelink-validation.ps1 @@ -144,11 +144,27 @@ $ValidatePackage = { if ($FailedFiles -eq 0) { Write-Host 'Passed.' - return 0 + return [pscustomobject]@{ + result = 0 + packagePath = $PackagePath + } } else { Write-PipelineTelemetryError -Category 'SourceLink' -Message "$PackagePath has broken SourceLink links." - return 1 + return [pscustomobject]@{ + result = 1 + packagePath = $PackagePath + } + } +} + +function CheckJobResult( + $result, + $packagePath, + [ref]$ValidationFailures) { + if ($jobResult.result -ne '0') { + Write-PipelineTelemetryError -Category 'SourceLink' -Message "$packagePath has broken SourceLink links." + $ValidationFailures.Value++ } } @@ -211,10 +227,8 @@ function ValidateSourceLinkLinks { } foreach ($Job in @(Get-Job -State 'Completed')) { - $jobResult = Receive-Job -Id $Job.Id - if ($jobResult -ne '0') { - $ValidationFailures++ - } + $jobResult = Wait-Job -Id $Job.Id | Receive-Job + CheckJobResult $jobResult.result $jobResult.packagePath ([ref]$ValidationFailures) Remove-Job -Id $Job.Id } } diff --git a/eng/common/post-build/symbols-validation.ps1 b/eng/common/post-build/symbols-validation.ps1 index a36fa6f2..fcc6019b 100644 --- a/eng/common/post-build/symbols-validation.ps1 +++ b/eng/common/post-build/symbols-validation.ps1 @@ -141,11 +141,6 @@ $CountMissingSymbols = { if ($using:Clean) { Remove-Item $ExtractPath -Recurse -Force } - - if ($MissingSymbols -ne 0) - { - Write-PipelineTelemetryError -Category 'CheckSymbols' -Message "Missing symbols for $MissingSymbols modules in the package $PackagePath" - } Pop-Location @@ -165,6 +160,7 @@ function CheckJobResult( $DupedSymbols.Value++ } elseif ($jobResult.result -ne '0') { + Write-PipelineTelemetryError -Category 'CheckSymbols' -Message "Missing symbols for $result modules in the package $packagePath" $TotalFailures.Value++ } } @@ -201,7 +197,6 @@ function CheckSymbolsAvailable { Start-Job -ScriptBlock $CountMissingSymbols -ArgumentList $FullName | Out-Null $NumJobs = @(Get-Job -State 'Running').Count - Write-Host $NumJobs while ($NumJobs -ge $MaxParallelJobs) { Write-Host "There are $NumJobs validation jobs running right now. Waiting $SecondsBetweenLoadChecks seconds to check again." diff --git a/eng/common/sdk-task.ps1 b/eng/common/sdk-task.ps1 index e159c6f1..f55c43c6 100644 --- a/eng/common/sdk-task.ps1 +++ b/eng/common/sdk-task.ps1 @@ -64,7 +64,7 @@ try { $GlobalJson.tools | Add-Member -Name "vs" -Value (ConvertFrom-Json "{ `"version`": `"16.5`" }") -MemberType NoteProperty } if( -not ($GlobalJson.tools.PSObject.Properties.Name -match "xcopy-msbuild" )) { - $GlobalJson.tools | Add-Member -Name "xcopy-msbuild" -Value "16.5.0-alpha" -MemberType NoteProperty + $GlobalJson.tools | Add-Member -Name "xcopy-msbuild" -Value "16.8.0-preview3" -MemberType NoteProperty } if ($GlobalJson.tools."xcopy-msbuild".Trim() -ine "none") { $xcopyMSBuildToolsFolder = InitializeXCopyMSBuild $GlobalJson.tools."xcopy-msbuild" -install $true diff --git a/eng/common/templates/job/job.yml b/eng/common/templates/job/job.yml index e78ed9a1..8b81a7e5 100644 --- a/eng/common/templates/job/job.yml +++ b/eng/common/templates/job/job.yml @@ -204,7 +204,7 @@ jobs: - ${{ if eq(parameters.enablePublishTestResults, 'true') }}: - task: PublishTestResults@2 - displayName: Publish Test Results + displayName: Publish XUnit Test Results inputs: testResultsFormat: 'xUnit' testResultsFiles: '*.xml' @@ -213,6 +213,16 @@ jobs: mergeTestResults: ${{ parameters.mergeTestResults }} continueOnError: true condition: always() + - task: PublishTestResults@2 + displayName: Publish TRX Test Results + inputs: + testResultsFormat: 'VSTest' + testResultsFiles: '*.trx' + searchFolder: '$(Build.SourcesDirectory)/artifacts/TestResults/$(_BuildConfig)' + testRunTitle: ${{ coalesce(parameters.testRunTitle, parameters.name, '$(System.JobName)') }}-trx + mergeTestResults: ${{ parameters.mergeTestResults }} + continueOnError: true + condition: always() - ${{ if and(eq(parameters.enablePublishBuildAssets, true), ne(parameters.enablePublishUsingPipelines, 'true'), eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: - task: CopyFiles@2 diff --git a/eng/common/templates/job/source-build.yml b/eng/common/templates/job/source-build.yml new file mode 100644 index 00000000..9332f5ec --- /dev/null +++ b/eng/common/templates/job/source-build.yml @@ -0,0 +1,49 @@ +parameters: + # This template adds arcade-powered source-build to CI. The template produces a server job with a + # default ID 'Source_Build_Complete' to put in a dependency list if necessary. + + # Specifies the prefix for source-build jobs added to pipeline. Use this if disambiguation needed. + jobNamePrefix: 'Source_Build' + + # Defines the platform on which to run the job. By default, a linux-x64 machine, suitable for + # managed-only repositories. This is an object with these properties: + # + # name: '' + # The name of the job. This is included in the job ID. + # targetRID: '' + # The name of the target RID to use, instead of the one auto-detected by Arcade. + # nonPortable: false + # Enables non-portable mode. This means a more specific RID (e.g. fedora.32-x64 rather than + # linux-x64), and compiling against distro-provided packages rather than portable ones. + # container: '' + # A container to use. Runs in docker. + # pool: {} + # A pool to use. Runs directly on an agent. + # buildScript: '' + # Specifies the build script to invoke to perform the build in the repo. The default + # './build.sh' should work for typical Arcade repositories, but this is customizable for + # difficult situations. + # jobProperties: {} + # A list of job properties to inject at the top level, for potential extensibility beyond + # container and pool. + platform: {} + +jobs: +- job: ${{ parameters.jobNamePrefix }}_${{ parameters.platform.name }} + displayName: Source-Build (${{ parameters.platform.name }}) + + ${{ each property in parameters.platform.jobProperties }}: + ${{ property.key }}: ${{ property.value }} + + ${{ if ne(parameters.platform.container, '') }}: + container: ${{ parameters.platform.container }} + ${{ if ne(parameters.platform.pool, '') }}: + pool: ${{ parameters.platform.pool }} + + workspace: + clean: all + + steps: + - template: /eng/common/templates/steps/source-build.yml + parameters: + platform: ${{ parameters.platform }} diff --git a/eng/common/templates/jobs/jobs.yml b/eng/common/templates/jobs/jobs.yml index c08225a9..08845950 100644 --- a/eng/common/templates/jobs/jobs.yml +++ b/eng/common/templates/jobs/jobs.yml @@ -24,6 +24,13 @@ parameters: # if 'true', the build won't run any of the internal only steps, even if it is running in non-public projects. runAsPublic: false + # Optional: Enable running the source-build jobs to build repo from source + runSourceBuild: false + + # Optional: Parameters for source-build template. + # See /eng/common/templates/jobs/source-build.yml for options + sourceBuildParameters: [] + # Internal resources (telemetry, microbuild) can only be accessed from non-public projects, # and some (Microbuild) should only be applied to non-PR cases for internal builds. @@ -43,6 +50,13 @@ jobs: name: ${{ job.job }} +- ${{ if eq(parameters.runSourceBuild, true) }}: + - template: /eng/common/templates/jobs/source-build.yml + parameters: + allCompletedJobId: Source_Build_Complete + ${{ each parameter in parameters.sourceBuildParameters }}: + ${{ parameter.key }}: ${{ parameter.value }} + - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: - ${{ if or(eq(parameters.enablePublishBuildAssets, true), eq(parameters.artifacts.publish.manifests, 'true'), ne(parameters.artifacts.publish.manifests, '')) }}: - template: ../job/publish-build-assets.yml @@ -55,6 +69,8 @@ jobs: - ${{ if eq(parameters.publishBuildAssetsDependsOn, '') }}: - ${{ each job in parameters.jobs }}: - ${{ job.job }} + - ${{ if eq(parameters.runSourceBuild, true) }}: + - Source_Build_Complete pool: vmImage: vs2017-win2016 runAsPublic: ${{ parameters.runAsPublic }} diff --git a/eng/common/templates/jobs/source-build.yml b/eng/common/templates/jobs/source-build.yml new file mode 100644 index 00000000..f463011e --- /dev/null +++ b/eng/common/templates/jobs/source-build.yml @@ -0,0 +1,48 @@ +parameters: + # This template adds arcade-powered source-build to CI. A job is created for each platform, as + # well as an optional server job that completes when all platform jobs complete. + + # The name of the "join" job for all source-build platforms. If set to empty string, the job is + # not included. Existing repo pipelines can use this job depend on all source-build jobs + # completing without maintaining a separate list of every single job ID: just depend on this one + # server job. By default, not included. Recommended name if used: 'Source_Build_Complete'. + allCompletedJobId: '' + + # See /eng/common/templates/job/source-build.yml + jobNamePrefix: 'Source_Build' + + # If changed to true, causes this template to include the default platform for a managed-only + # repo. The exact Docker image used for this build will be provided by Arcade. This has some risk, + # but since the repo is supposed to be managed-only, the risk should be very low. + includeDefaultManagedPlatform: false + defaultManagedPlatform: + name: 'Managed' + container: 'mcr.microsoft.com/dotnet-buildtools/prereqs:centos-7-3e800f1-20190501005343' + + # Defines the platforms on which to run build jobs. One job is created for each platform, and the + # object in this array is sent to the job template as 'platform'. + platforms: [] + +jobs: + +- ${{ if ne(parameters.allCompletedJobId, '') }}: + - job: ${{ parameters.allCompletedJobId }} + displayName: Source-Build Complete + pool: server + dependsOn: + - ${{ each platform in parameters.platforms }}: + - ${{ parameters.jobNamePrefix }}_${{ platform.name }} + - ${{ if eq(parameters.includeDefaultManagedPlatform, true) }}: + - ${{ parameters.jobNamePrefix }}_${{ parameters.defaultManagedPlatform.name }} + +- ${{ each platform in parameters.platforms }}: + - template: /eng/common/templates/job/source-build.yml + parameters: + jobNamePrefix: ${{ parameters.jobNamePrefix }} + platform: ${{ platform }} + +- ${{ if eq(parameters.includeDefaultManagedPlatform, true) }}: + - template: /eng/common/templates/job/source-build.yml + parameters: + jobNamePrefix: ${{ parameters.jobNamePrefix }} + platform: ${{ parameters.defaultManagedPlatform }} diff --git a/eng/common/templates/post-build/channels/generic-internal-channel.yml b/eng/common/templates/post-build/channels/generic-internal-channel.yml index 59eb93a4..7ae52559 100644 --- a/eng/common/templates/post-build/channels/generic-internal-channel.yml +++ b/eng/common/templates/post-build/channels/generic-internal-channel.yml @@ -4,7 +4,7 @@ parameters: artifactsPublishingAdditionalParameters: '' dependsOn: - Validate - publishInstallersAndChecksums: false + publishInstallersAndChecksums: true symbolPublishingAdditionalParameters: '' stageName: '' channelName: '' @@ -158,7 +158,7 @@ stages: /p:BlobBasePath='$(Build.ArtifactStagingDirectory)/BlobArtifacts/' /p:PackageBasePath='$(Build.ArtifactStagingDirectory)/PackageArtifacts/' /p:Configuration=Release - /p:PublishInstallersAndChecksums=true + /p:PublishInstallersAndChecksums=${{ parameters.publishInstallersAndChecksums }} /p:ChecksumsTargetStaticFeed=$(InternalChecksumsBlobFeedUrl) /p:ChecksumsAzureAccountKey=$(InternalChecksumsBlobFeedKey) /p:InstallersTargetStaticFeed=$(InternalInstallersBlobFeedUrl) diff --git a/eng/common/templates/post-build/channels/generic-public-channel.yml b/eng/common/templates/post-build/channels/generic-public-channel.yml index 7e80a621..6cf39dbb 100644 --- a/eng/common/templates/post-build/channels/generic-public-channel.yml +++ b/eng/common/templates/post-build/channels/generic-public-channel.yml @@ -4,7 +4,7 @@ parameters: artifactsPublishingAdditionalParameters: '' dependsOn: - Validate - publishInstallersAndChecksums: false + publishInstallersAndChecksums: true symbolPublishingAdditionalParameters: '' stageName: '' channelName: '' diff --git a/eng/common/templates/post-build/post-build.yml b/eng/common/templates/post-build/post-build.yml index 630a99d4..0854e489 100644 --- a/eng/common/templates/post-build/post-build.yml +++ b/eng/common/templates/post-build/post-build.yml @@ -19,7 +19,7 @@ parameters: enableSigningValidation: true enableSymbolValidation: false enableNugetValidation: true - publishInstallersAndChecksums: false + publishInstallersAndChecksums: true SDLValidationParameters: enable: false continueOnError: false @@ -78,21 +78,22 @@ stages: BARBuildId: ${{ parameters.BARBuildId }} PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }} - - job: - displayName: Post-build Checks - dependsOn: setupMaestroVars - variables: - - name: TargetChannels - value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.TargetChannels'] ] - pool: - vmImage: 'windows-2019' - steps: - - task: PowerShell@2 - displayName: Maestro Channels Consistency - inputs: - filePath: $(Build.SourcesDirectory)/eng/common/post-build/check-channel-consistency.ps1 - arguments: -PromoteToChannels "$(TargetChannels)" - -AvailableChannelIds ${{parameters.NetEngLatestChannelId}},${{parameters.NetEngValidationChannelId}},${{parameters.NetDev5ChannelId}},${{parameters.NetDev6ChannelId}},${{parameters.GeneralTestingChannelId}},${{parameters.NETCoreToolingDevChannelId}},${{parameters.NETCoreToolingReleaseChannelId}},${{parameters.NETInternalToolingChannelId}},${{parameters.NETCoreExperimentalChannelId}},${{parameters.NetEngServicesIntChannelId}},${{parameters.NetEngServicesProdChannelId}},${{parameters.Net5Preview8ChannelId}},${{parameters.Net5RC1ChannelId}},${{parameters.Net5RC2ChannelId}},${{parameters.NetCoreSDK313xxChannelId}},${{parameters.NetCoreSDK313xxInternalChannelId}},${{parameters.NetCoreSDK314xxChannelId}},${{parameters.NetCoreSDK314xxInternalChannelId}},${{parameters.VS166ChannelId}},${{parameters.VS167ChannelId}},${{parameters.VS168ChannelId}},${{parameters.VSMasterChannelId}} + - ${{ if and(le(parameters.publishingInfraVersion, 2), eq(parameters.inline, 'true')) }}: + - job: + displayName: Post-build Checks + dependsOn: setupMaestroVars + variables: + - name: TargetChannels + value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.TargetChannels'] ] + pool: + vmImage: 'windows-2019' + steps: + - task: PowerShell@2 + displayName: Maestro Channels Consistency + inputs: + filePath: $(Build.SourcesDirectory)/eng/common/post-build/check-channel-consistency.ps1 + arguments: -PromoteToChannels "$(TargetChannels)" + -AvailableChannelIds ${{parameters.NetEngLatestChannelId}},${{parameters.NetEngValidationChannelId}},${{parameters.NetDev5ChannelId}},${{parameters.NetDev6ChannelId}},${{parameters.GeneralTestingChannelId}},${{parameters.NETCoreToolingDevChannelId}},${{parameters.NETCoreToolingReleaseChannelId}},${{parameters.NETInternalToolingChannelId}},${{parameters.NETCoreExperimentalChannelId}},${{parameters.NetEngServicesIntChannelId}},${{parameters.NetEngServicesProdChannelId}},${{parameters.Net5Preview8ChannelId}},${{parameters.Net5RC1ChannelId}},${{parameters.Net5RC2ChannelId}},${{parameters.NetCoreSDK313xxChannelId}},${{parameters.NetCoreSDK313xxInternalChannelId}},${{parameters.NetCoreSDK314xxChannelId}},${{parameters.NetCoreSDK314xxInternalChannelId}},${{parameters.VS166ChannelId}},${{parameters.VS167ChannelId}},${{parameters.VS168ChannelId}},${{parameters.VSMasterChannelId}} - job: displayName: NuGet Validation diff --git a/eng/common/templates/steps/perf-send-to-helix.yml b/eng/common/templates/steps/perf-send-to-helix.yml index e003fe2e..a468e92c 100644 --- a/eng/common/templates/steps/perf-send-to-helix.yml +++ b/eng/common/templates/steps/perf-send-to-helix.yml @@ -11,7 +11,7 @@ parameters: WorkItemDirectory: '' # optional -- a payload directory to zip up and send to Helix; requires WorkItemCommand; incompatible with XUnitProjects CorrelationPayloadDirectory: '' # optional -- a directory to zip up and send to Helix as a correlation payload IncludeDotNetCli: false # optional -- true will download a version of the .NET CLI onto the Helix machine as a correlation payload; requires DotNetCliPackageType and DotNetCliVersion - DotNetCliPackageType: '' # optional -- either 'sdk' or 'runtime'; determines whether the sdk or runtime will be sent to Helix; see https://raw.githubusercontent.com/dotnet/core/master/release-notes/releases.json + DotNetCliPackageType: '' # optional -- either 'sdk', 'runtime' or 'aspnetcore-runtime'; determines whether the sdk or runtime will be sent to Helix; see https://raw.githubusercontent.com/dotnet/core/master/release-notes/releases.json DotNetCliVersion: '' # optional -- version of the CLI to send to Helix; based on this: https://raw.githubusercontent.com/dotnet/core/master/release-notes/releases.json EnableXUnitReporter: false # optional -- true enables XUnit result reporting to Mission Control WaitForWorkItemCompletion: true # optional -- true will make the task wait until work items have been completed and fail the build if work items fail. False is "fire and forget." @@ -19,34 +19,18 @@ parameters: DisplayNamePrefix: 'Send job to Helix' # optional -- rename the beginning of the displayName of the steps in AzDO condition: succeeded() # optional -- condition for step to execute; defaults to succeeded() continueOnError: false # optional -- determines whether to continue the build if the step errors; defaults to false + osGroup: '' # required -- operating system for the job steps: - - powershell: $(Build.SourcesDirectory)\eng\common\msbuild.ps1 $(Build.SourcesDirectory)\eng\common\performance\${{ parameters.ProjectFile }} /restore /t:Test /bl:$(Build.SourcesDirectory)\artifacts\log\$env:BuildConfig\SendToHelix.binlog - displayName: ${{ parameters.DisplayNamePrefix }} (Windows) - env: - BuildConfig: $(_BuildConfig) - HelixSource: ${{ parameters.HelixSource }} - HelixType: ${{ parameters.HelixType }} - HelixBuild: ${{ parameters.HelixBuild }} - HelixTargetQueues: ${{ parameters.HelixTargetQueues }} - HelixAccessToken: ${{ parameters.HelixAccessToken }} - HelixPreCommands: ${{ parameters.HelixPreCommands }} - HelixPostCommands: ${{ parameters.HelixPostCommands }} - WorkItemDirectory: ${{ parameters.WorkItemDirectory }} - CorrelationPayloadDirectory: ${{ parameters.CorrelationPayloadDirectory }} - IncludeDotNetCli: ${{ parameters.IncludeDotNetCli }} - DotNetCliPackageType: ${{ parameters.DotNetCliPackageType }} - DotNetCliVersion: ${{ parameters.DotNetCliVersion }} - EnableXUnitReporter: ${{ parameters.EnableXUnitReporter }} - WaitForWorkItemCompletion: ${{ parameters.WaitForWorkItemCompletion }} - Creator: ${{ parameters.Creator }} - SYSTEM_ACCESSTOKEN: $(System.AccessToken) - condition: and(${{ parameters.condition }}, eq(variables['Agent.Os'], 'Windows_NT')) +- template: /eng/pipelines/common/templates/runtimes/send-to-helix-inner-step.yml + parameters: + osGroup: ${{ parameters.osGroup }} + sendParams: $(Build.SourcesDirectory)/eng/common/performance/${{ parameters.ProjectFile }} /restore /t:Test /bl:$(Build.SourcesDirectory)/artifacts/log/$(_BuildConfig)/SendToHelix.binlog + displayName: ${{ parameters.DisplayNamePrefix }} + condition: ${{ parameters.condition }} continueOnError: ${{ parameters.continueOnError }} - - script: $BUILD_SOURCESDIRECTORY/eng/common/msbuild.sh $BUILD_SOURCESDIRECTORY/eng/common/performance/${{ parameters.ProjectFile }} /restore /t:Test /bl:$BUILD_SOURCESDIRECTORY/artifacts/log/$BuildConfig/SendToHelix.binlog - displayName: ${{ parameters.DisplayNamePrefix }} (Unix) - env: + environment: BuildConfig: $(_BuildConfig) HelixSource: ${{ parameters.HelixSource }} HelixType: ${{ parameters.HelixType }} @@ -64,5 +48,3 @@ steps: WaitForWorkItemCompletion: ${{ parameters.WaitForWorkItemCompletion }} Creator: ${{ parameters.Creator }} SYSTEM_ACCESSTOKEN: $(System.AccessToken) - condition: and(${{ parameters.condition }}, ne(variables['Agent.Os'], 'Windows_NT')) - continueOnError: ${{ parameters.continueOnError }} diff --git a/eng/common/templates/steps/publish-logs.yml b/eng/common/templates/steps/publish-logs.yml index 428026d2..88f238f3 100644 --- a/eng/common/templates/steps/publish-logs.yml +++ b/eng/common/templates/steps/publish-logs.yml @@ -3,20 +3,21 @@ parameters: JobLabel: '' steps: -- task: CopyFiles@2 - displayName: Copy Logs to $(Build.StagingDirectory)\BuildLogs +- task: Powershell@2 + displayName: Prepare Binlogs to Upload inputs: - SourceFolder: $(Build.SourcesDirectory)\artifacts - Contents: | - **/*.log - **/*.binlog - TargetFolder: '$(Build.StagingDirectory)\BuildLogs' + targetType: inline + script: | + New-Item -ItemType Directory $(Build.SourcesDirectory)/PostBuildLogs/${{parameters.StageLabel}}/${{parameters.JobLabel}}/ + Move-Item -Path $(Build.SourcesDirectory)/artifacts/log/Debug/* $(Build.SourcesDirectory)/PostBuildLogs/${{parameters.StageLabel}}/${{parameters.JobLabel}}/ continueOnError: true - condition: succeededOrFailed() + condition: always() -- task: PublishPipelineArtifact@1 - displayName: Publish BuildLogs +- task: PublishBuildArtifacts@1 + displayName: Publish Logs inputs: - targetPath: '$(Build.StagingDirectory)\BuildLogs' - artifactName: ${{ parameters.JobLabel }} - condition: succeededOrFailed() + PathtoPublish: '$(Build.SourcesDirectory)/PostBuildLogs' + PublishLocation: Container + ArtifactName: PostBuildLogs + continueOnError: true + condition: always() diff --git a/eng/common/templates/steps/send-to-helix.yml b/eng/common/templates/steps/send-to-helix.yml index 5eceb487..bb5f1a92 100644 --- a/eng/common/templates/steps/send-to-helix.yml +++ b/eng/common/templates/steps/send-to-helix.yml @@ -18,7 +18,7 @@ parameters: XUnitRuntimeTargetFramework: '' # optional -- framework to use for the xUnit console runner XUnitRunnerVersion: '' # optional -- version of the xUnit nuget package you wish to use on Helix; required for XUnitProjects IncludeDotNetCli: false # optional -- true will download a version of the .NET CLI onto the Helix machine as a correlation payload; requires DotNetCliPackageType and DotNetCliVersion - DotNetCliPackageType: '' # optional -- either 'sdk' or 'runtime'; determines whether the sdk or runtime will be sent to Helix; see https://raw.githubusercontent.com/dotnet/core/master/release-notes/releases-index.json + DotNetCliPackageType: '' # optional -- either 'sdk', 'runtime' or 'aspnetcore-runtime'; determines whether the sdk or runtime will be sent to Helix; see https://raw.githubusercontent.com/dotnet/core/master/release-notes/releases-index.json DotNetCliVersion: '' # optional -- version of the CLI to send to Helix; based on this: https://raw.githubusercontent.com/dotnet/core/master/release-notes/releases-index.json EnableXUnitReporter: false # optional -- true enables XUnit result reporting to Mission Control WaitForWorkItemCompletion: true # optional -- true will make the task wait until work items have been completed and fail the build if work items fail. False is "fire and forget." diff --git a/eng/common/templates/steps/source-build.yml b/eng/common/templates/steps/source-build.yml new file mode 100644 index 00000000..8e336b7d --- /dev/null +++ b/eng/common/templates/steps/source-build.yml @@ -0,0 +1,66 @@ +parameters: + # This template adds arcade-powered source-build to CI. + + # This is a 'steps' template, and is intended for advanced scenarios where the existing build + # infra has a careful build methodology that must be followed. For example, a repo + # (dotnet/runtime) might choose to clone the GitHub repo only once and store it as a pipeline + # artifact for all subsequent jobs to use, to reduce dependence on a strong network connection to + # GitHub. Using this steps template leaves room for that infra to be included. + + # Defines the platform on which to run the steps. See 'eng/common/templates/job/source-build.yml' + # for details. The entire object is described in the 'job' template for simplicity, even though + # the usage of the properties on this object is split between the 'job' and 'steps' templates. + platform: {} + +steps: +# Build. Keep it self-contained for simple reusability. (No source-build-specific job variables.) +- script: | + set -x + df -h + + buildConfig=Release + # Check if AzDO substitutes in a build config from a variable, and use it if so. + if [ '$(_BuildConfig)' != '$''(_BuildConfig)' ]; then + buildConfig='$(_BuildConfig)' + fi + + officialBuildArgs= + if [ '${{ and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}' = 'True' ]; then + officialBuildArgs='/p:DotNetPublishUsingPipelines=true /p:OfficialBuildId=$(BUILD.BUILDNUMBER)' + fi + + targetRidArgs= + if [ '${{ parameters.platform.targetRID }}' != '' ]; then + targetRidArgs='/p:TargetRid=${{ parameters.platform.targetRID }}' + fi + + ${{ coalesce(parameters.platform.buildScript, './build.sh') }} --ci \ + --configuration $buildConfig \ + --restore --build --pack --publish \ + $officialBuildArgs \ + $targetRidArgs \ + /p:SourceBuildNonPortable=${{ parameters.platform.nonPortable }} \ + /p:ArcadeBuildFromSource=true + displayName: Build + +# Upload build logs for diagnosis. +- task: CopyFiles@2 + displayName: Prepare BuildLogs staging directory + inputs: + SourceFolder: '$(Build.SourcesDirectory)' + Contents: | + **/*.log + **/*.binlog + artifacts/source-build/self/prebuilt-report/** + TargetFolder: '$(Build.StagingDirectory)/BuildLogs' + CleanTargetFolder: true + continueOnError: true + condition: succeededOrFailed() + +- task: PublishPipelineArtifact@1 + displayName: Publish BuildLogs + inputs: + targetPath: '$(Build.StagingDirectory)/BuildLogs' + artifactName: BuildLogs_SourceBuild_${{ parameters.platform.name }}_Attempt$(System.JobAttempt) + continueOnError: true + condition: succeededOrFailed() diff --git a/eng/common/tools.ps1 b/eng/common/tools.ps1 index 1bf21552..9014e062 100644 --- a/eng/common/tools.ps1 +++ b/eng/common/tools.ps1 @@ -240,8 +240,8 @@ function GetDotNetInstallScript([string] $dotnetRoot) { return $installScript } -function InstallDotNetSdk([string] $dotnetRoot, [string] $version, [string] $architecture = '') { - InstallDotNet $dotnetRoot $version $architecture '' $false $runtimeSourceFeed $runtimeSourceFeedKey +function InstallDotNetSdk([string] $dotnetRoot, [string] $version, [string] $architecture = '', [switch] $noPath) { + InstallDotNet $dotnetRoot $version $architecture '' $false $runtimeSourceFeed $runtimeSourceFeedKey -noPath:$noPath } function InstallDotNet([string] $dotnetRoot, @@ -250,7 +250,8 @@ function InstallDotNet([string] $dotnetRoot, [string] $runtime = '', [bool] $skipNonVersionedFiles = $false, [string] $runtimeSourceFeed = '', - [string] $runtimeSourceFeedKey = '') { + [string] $runtimeSourceFeedKey = '', + [switch] $noPath) { $installScript = GetDotNetInstallScript $dotnetRoot $installParameters = @{ @@ -261,6 +262,7 @@ function InstallDotNet([string] $dotnetRoot, if ($architecture) { $installParameters.Architecture = $architecture } if ($runtime) { $installParameters.Runtime = $runtime } if ($skipNonVersionedFiles) { $installParameters.SkipNonVersionedFiles = $skipNonVersionedFiles } + if ($noPath) { $installParameters.NoPath = $True } try { & $installScript @installParameters @@ -310,9 +312,15 @@ function InitializeVisualStudioMSBuild([bool]$install, [object]$vsRequirements = return $global:_MSBuildExe } - $vsMinVersionReqdStr = '16.5' + # Minimum VS version to require. + $vsMinVersionReqdStr = '16.8' $vsMinVersionReqd = [Version]::new($vsMinVersionReqdStr) + # If the version of msbuild is going to be xcopied, + # use this version. Version matches a package here: + # https://dev.azure.com/dnceng/public/_packaging?_a=package&feed=dotnet-eng&package=RoslynTools.MSBuild&protocolType=NuGet&version=16.8.0-preview3&view=overview + $defaultXCopyMSBuildVersion = '16.8.0-preview3' + if (!$vsRequirements) { $vsRequirements = $GlobalJson.tools.vs } $vsMinVersionStr = if ($vsRequirements.version) { $vsRequirements.version } else { $vsMinVersionReqdStr } $vsMinVersion = [Version]::new($vsMinVersionStr) @@ -347,23 +355,28 @@ function InitializeVisualStudioMSBuild([bool]$install, [object]$vsRequirements = $xcopyMSBuildVersion = $GlobalJson.tools.'xcopy-msbuild' $vsMajorVersion = $xcopyMSBuildVersion.Split('.')[0] } else { - #if vs version provided in global.json is incompatible then use the default version for xcopy msbuild download + #if vs version provided in global.json is incompatible (too low) then use the default version for xcopy msbuild download if($vsMinVersion -lt $vsMinVersionReqd){ - Write-Host "Using xcopy-msbuild version of $vsMinVersionReqdStr.0-alpha since VS version $vsMinVersionStr provided in global.json is not compatible" - $vsMajorVersion = $vsMinVersionReqd.Major - $vsMinorVersion = $vsMinVersionReqd.Minor + Write-Host "Using xcopy-msbuild version of $defaultXCopyMSBuildVersion since VS version $vsMinVersionStr provided in global.json is not compatible" + $xcopyMSBuildVersion = $defaultXCopyMSBuildVersion } else{ + # If the VS version IS compatible, look for an xcopy msbuild package + # with a version matching VS. + # Note: If this version does not exist, then an explicit version of xcopy msbuild + # can be specified in global.json. This will be required for pre-release versions of msbuild. $vsMajorVersion = $vsMinVersion.Major $vsMinorVersion = $vsMinVersion.Minor + $xcopyMSBuildVersion = "$vsMajorVersion.$vsMinorVersion.0" } - - $xcopyMSBuildVersion = "$vsMajorVersion.$vsMinorVersion.0-alpha" } $vsInstallDir = $null if ($xcopyMSBuildVersion.Trim() -ine "none") { $vsInstallDir = InitializeXCopyMSBuild $xcopyMSBuildVersion $install + if ($vsInstallDir -eq $null) { + throw "Could not xcopy msbuild. Please check that package 'RoslynTools.MSBuild @ $xcopyMSBuildVersion' exists on feed 'dotnet-eng'." + } } if ($vsInstallDir -eq $null) { throw 'Unable to find Visual Studio that has required version and components installed' @@ -527,13 +540,15 @@ function GetDefaultMSBuildEngine() { function GetNuGetPackageCachePath() { if ($env:NUGET_PACKAGES -eq $null) { - # Use local cache on CI to ensure deterministic build, + # Use local cache on CI to ensure deterministic build. + # Avoid using the http cache as workaround for https://github.com/NuGet/Home/issues/3116 # use global cache in dev builds to avoid cost of downloading packages. # For directory normalization, see also: https://github.com/NuGet/Home/issues/7968 if ($useGlobalNuGetCache) { $env:NUGET_PACKAGES = Join-Path $env:UserProfile '.nuget\packages\' } else { $env:NUGET_PACKAGES = Join-Path $RepoRoot '.packages\' + $env:RESTORENOCACHE = $true } } @@ -716,6 +731,16 @@ function IsWindowsPlatform() { return [environment]::OSVersion.Platform -eq [PlatformID]::Win32NT } +function Get-Darc($version) { + $darcPath = "$TempDir\darc\$(New-Guid)" + if ($version -ne $null) { + & $PSScriptRoot\darc-init.ps1 -toolpath $darcPath -darcVersion $version | Out-Host + } else { + & $PSScriptRoot\darc-init.ps1 -toolpath $darcPath | Out-Host + } + return "$darcPath\darc.exe" +} + . $PSScriptRoot\pipeline-logging-functions.ps1 $RepoRoot = Resolve-Path (Join-Path $PSScriptRoot '..\..') diff --git a/eng/common/tools.sh b/eng/common/tools.sh index c722a058..b5d63cb1 100755 --- a/eng/common/tools.sh +++ b/eng/common/tools.sh @@ -312,12 +312,14 @@ function InitializeBuildTool { _InitializeBuildToolFramework="netcoreapp2.1" } +# Set RestoreNoCache as a workaround for https://github.com/NuGet/Home/issues/3116 function GetNuGetPackageCachePath { if [[ -z ${NUGET_PACKAGES:-} ]]; then if [[ "$use_global_nuget_cache" == true ]]; then export NUGET_PACKAGES="$HOME/.nuget/packages" else export NUGET_PACKAGES="$repo_root/.packages" + export RESTORENOCACHE=true fi fi diff --git a/global.json b/global.json index 4e51b9b1..44d95934 100644 --- a/global.json +++ b/global.json @@ -11,7 +11,7 @@ }, "msbuild-sdks": { - "Microsoft.DotNet.Arcade.Sdk": "5.0.0-beta.20426.1", - "Microsoft.DotNet.Helix.Sdk": "5.0.0-beta.20426.1" + "Microsoft.DotNet.Arcade.Sdk": "5.0.0-beta.20510.1", + "Microsoft.DotNet.Helix.Sdk": "5.0.0-beta.20510.1" } }