diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml
index 25d14ef0..42fa203b 100644
--- a/eng/Version.Details.xml
+++ b/eng/Version.Details.xml
@@ -5,14 +5,14 @@
-
+
https://github.com/dotnet/arcade
- 3b720900ac55e9cc3675584c41da6460d6193f37
+ 6813f5aa511a7a4498fa217a54219b5704a01f83
-
+
https://github.com/dotnet/arcade
- 3b720900ac55e9cc3675584c41da6460d6193f37
+ 6813f5aa511a7a4498fa217a54219b5704a01f83
diff --git a/eng/common/cross/arm64/tizen-build-rootfs.sh b/eng/common/cross/arm64/tizen-build-rootfs.sh
old mode 100644
new mode 100755
diff --git a/eng/common/cross/arm64/tizen-fetch.sh b/eng/common/cross/arm64/tizen-fetch.sh
old mode 100644
new mode 100755
diff --git a/eng/common/dotnet-install-scripts/dotnet-install.sh b/eng/common/dotnet-install-scripts/dotnet-install.sh
old mode 100644
new mode 100755
diff --git a/eng/common/internal/Directory.Build.props b/eng/common/internal/Directory.Build.props
index e33179ef..dbf99d82 100644
--- a/eng/common/internal/Directory.Build.props
+++ b/eng/common/internal/Directory.Build.props
@@ -1,4 +1,4 @@
-
+
diff --git a/eng/common/native/find-native-compiler.sh b/eng/common/native/find-native-compiler.sh
old mode 100644
new mode 100755
diff --git a/eng/common/performance/performance-setup.sh b/eng/common/performance/performance-setup.sh
index 806e56c6..99d1b7bc 100755
--- a/eng/common/performance/performance-setup.sh
+++ b/eng/common/performance/performance-setup.sh
@@ -201,7 +201,6 @@ if [[ "$internal" == true ]]; then
fi
if [[ "$mono_dotnet" != "" ]] && [[ "$monointerpreter" == "false" ]]; then
- configurations="$configurations LLVM=$llvm MonoInterpreter=$monointerpreter MonoAOT=$monoaot"
extra_benchmark_dotnet_arguments="$extra_benchmark_dotnet_arguments --category-exclusion-filter NoMono"
fi
@@ -211,6 +210,7 @@ if [[ "$wasm_runtime_loc" != "" ]]; then
fi
if [[ "$mono_dotnet" != "" ]] && [[ "$monointerpreter" == "true" ]]; then
+ configurations="$configurations LLVM=$llvm MonoInterpreter=$monointerpreter MonoAOT=$monoaot"
extra_benchmark_dotnet_arguments="$extra_benchmark_dotnet_arguments --category-exclusion-filter NoInterpreter NoMono"
fi
diff --git a/eng/common/post-build/publish-using-darc.ps1 b/eng/common/post-build/publish-using-darc.ps1
index f044afe8..650b13b0 100644
--- a/eng/common/post-build/publish-using-darc.ps1
+++ b/eng/common/post-build/publish-using-darc.ps1
@@ -8,7 +8,7 @@ param(
[Parameter(Mandatory=$false)][string] $EnableSourceLinkValidation,
[Parameter(Mandatory=$false)][string] $EnableSigningValidation,
[Parameter(Mandatory=$false)][string] $EnableNugetValidation,
- [Parameter(Mandatory=$true)][string] $PublishInstallersAndChecksums,
+ [Parameter(Mandatory=$false)][string] $PublishInstallersAndChecksums,
[Parameter(Mandatory=$false)][string] $ArtifactsPublishingAdditionalParameters,
[Parameter(Mandatory=$false)][string] $SigningValidationAdditionalParameters
)
@@ -16,7 +16,7 @@ param(
try {
. $PSScriptRoot\post-build-utils.ps1
# Hard coding darc version till the next arcade-services roll out, cos this version has required API changes for darc add-build-to-channel
- . $PSScriptRoot\..\darc-init.ps1 -darcVersion "1.1.0-beta.20418.1"
+ $darc = Get-Darc "1.1.0-beta.20418.1"
$optionalParams = [System.Collections.ArrayList]::new()
@@ -29,7 +29,7 @@ try {
$optionalParams.Add("--no-wait") | Out-Null
}
- if ("true" -eq $PublishInstallersAndChecksums) {
+ if ("false" -ne $PublishInstallersAndChecksums) {
$optionalParams.Add("--publish-installers-and-checksums") | Out-Null
}
@@ -50,7 +50,7 @@ try {
}
}
- & darc add-build-to-channel `
+ & $darc add-build-to-channel `
--id $buildId `
--publishing-infra-version $PublishingInfraVersion `
--default-channels `
diff --git a/eng/common/post-build/sourcelink-validation.ps1 b/eng/common/post-build/sourcelink-validation.ps1
index cdd1bba4..c7e7ae67 100644
--- a/eng/common/post-build/sourcelink-validation.ps1
+++ b/eng/common/post-build/sourcelink-validation.ps1
@@ -144,11 +144,27 @@ $ValidatePackage = {
if ($FailedFiles -eq 0) {
Write-Host 'Passed.'
- return 0
+ return [pscustomobject]@{
+ result = 0
+ packagePath = $PackagePath
+ }
}
else {
Write-PipelineTelemetryError -Category 'SourceLink' -Message "$PackagePath has broken SourceLink links."
- return 1
+ return [pscustomobject]@{
+ result = 1
+ packagePath = $PackagePath
+ }
+ }
+}
+
+function CheckJobResult(
+ $result,
+ $packagePath,
+ [ref]$ValidationFailures) {
+ if ($jobResult.result -ne '0') {
+ Write-PipelineTelemetryError -Category 'SourceLink' -Message "$packagePath has broken SourceLink links."
+ $ValidationFailures.Value++
}
}
@@ -211,10 +227,8 @@ function ValidateSourceLinkLinks {
}
foreach ($Job in @(Get-Job -State 'Completed')) {
- $jobResult = Receive-Job -Id $Job.Id
- if ($jobResult -ne '0') {
- $ValidationFailures++
- }
+ $jobResult = Wait-Job -Id $Job.Id | Receive-Job
+ CheckJobResult $jobResult.result $jobResult.packagePath ([ref]$ValidationFailures)
Remove-Job -Id $Job.Id
}
}
diff --git a/eng/common/post-build/symbols-validation.ps1 b/eng/common/post-build/symbols-validation.ps1
index a36fa6f2..fcc6019b 100644
--- a/eng/common/post-build/symbols-validation.ps1
+++ b/eng/common/post-build/symbols-validation.ps1
@@ -141,11 +141,6 @@ $CountMissingSymbols = {
if ($using:Clean) {
Remove-Item $ExtractPath -Recurse -Force
}
-
- if ($MissingSymbols -ne 0)
- {
- Write-PipelineTelemetryError -Category 'CheckSymbols' -Message "Missing symbols for $MissingSymbols modules in the package $PackagePath"
- }
Pop-Location
@@ -165,6 +160,7 @@ function CheckJobResult(
$DupedSymbols.Value++
}
elseif ($jobResult.result -ne '0') {
+ Write-PipelineTelemetryError -Category 'CheckSymbols' -Message "Missing symbols for $result modules in the package $packagePath"
$TotalFailures.Value++
}
}
@@ -201,7 +197,6 @@ function CheckSymbolsAvailable {
Start-Job -ScriptBlock $CountMissingSymbols -ArgumentList $FullName | Out-Null
$NumJobs = @(Get-Job -State 'Running').Count
- Write-Host $NumJobs
while ($NumJobs -ge $MaxParallelJobs) {
Write-Host "There are $NumJobs validation jobs running right now. Waiting $SecondsBetweenLoadChecks seconds to check again."
diff --git a/eng/common/sdk-task.ps1 b/eng/common/sdk-task.ps1
index e159c6f1..f55c43c6 100644
--- a/eng/common/sdk-task.ps1
+++ b/eng/common/sdk-task.ps1
@@ -64,7 +64,7 @@ try {
$GlobalJson.tools | Add-Member -Name "vs" -Value (ConvertFrom-Json "{ `"version`": `"16.5`" }") -MemberType NoteProperty
}
if( -not ($GlobalJson.tools.PSObject.Properties.Name -match "xcopy-msbuild" )) {
- $GlobalJson.tools | Add-Member -Name "xcopy-msbuild" -Value "16.5.0-alpha" -MemberType NoteProperty
+ $GlobalJson.tools | Add-Member -Name "xcopy-msbuild" -Value "16.8.0-preview3" -MemberType NoteProperty
}
if ($GlobalJson.tools."xcopy-msbuild".Trim() -ine "none") {
$xcopyMSBuildToolsFolder = InitializeXCopyMSBuild $GlobalJson.tools."xcopy-msbuild" -install $true
diff --git a/eng/common/templates/job/job.yml b/eng/common/templates/job/job.yml
index e78ed9a1..8b81a7e5 100644
--- a/eng/common/templates/job/job.yml
+++ b/eng/common/templates/job/job.yml
@@ -204,7 +204,7 @@ jobs:
- ${{ if eq(parameters.enablePublishTestResults, 'true') }}:
- task: PublishTestResults@2
- displayName: Publish Test Results
+ displayName: Publish XUnit Test Results
inputs:
testResultsFormat: 'xUnit'
testResultsFiles: '*.xml'
@@ -213,6 +213,16 @@ jobs:
mergeTestResults: ${{ parameters.mergeTestResults }}
continueOnError: true
condition: always()
+ - task: PublishTestResults@2
+ displayName: Publish TRX Test Results
+ inputs:
+ testResultsFormat: 'VSTest'
+ testResultsFiles: '*.trx'
+ searchFolder: '$(Build.SourcesDirectory)/artifacts/TestResults/$(_BuildConfig)'
+ testRunTitle: ${{ coalesce(parameters.testRunTitle, parameters.name, '$(System.JobName)') }}-trx
+ mergeTestResults: ${{ parameters.mergeTestResults }}
+ continueOnError: true
+ condition: always()
- ${{ if and(eq(parameters.enablePublishBuildAssets, true), ne(parameters.enablePublishUsingPipelines, 'true'), eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}:
- task: CopyFiles@2
diff --git a/eng/common/templates/job/source-build.yml b/eng/common/templates/job/source-build.yml
new file mode 100644
index 00000000..9332f5ec
--- /dev/null
+++ b/eng/common/templates/job/source-build.yml
@@ -0,0 +1,49 @@
+parameters:
+ # This template adds arcade-powered source-build to CI. The template produces a server job with a
+ # default ID 'Source_Build_Complete' to put in a dependency list if necessary.
+
+ # Specifies the prefix for source-build jobs added to pipeline. Use this if disambiguation needed.
+ jobNamePrefix: 'Source_Build'
+
+ # Defines the platform on which to run the job. By default, a linux-x64 machine, suitable for
+ # managed-only repositories. This is an object with these properties:
+ #
+ # name: ''
+ # The name of the job. This is included in the job ID.
+ # targetRID: ''
+ # The name of the target RID to use, instead of the one auto-detected by Arcade.
+ # nonPortable: false
+ # Enables non-portable mode. This means a more specific RID (e.g. fedora.32-x64 rather than
+ # linux-x64), and compiling against distro-provided packages rather than portable ones.
+ # container: ''
+ # A container to use. Runs in docker.
+ # pool: {}
+ # A pool to use. Runs directly on an agent.
+ # buildScript: ''
+ # Specifies the build script to invoke to perform the build in the repo. The default
+ # './build.sh' should work for typical Arcade repositories, but this is customizable for
+ # difficult situations.
+ # jobProperties: {}
+ # A list of job properties to inject at the top level, for potential extensibility beyond
+ # container and pool.
+ platform: {}
+
+jobs:
+- job: ${{ parameters.jobNamePrefix }}_${{ parameters.platform.name }}
+ displayName: Source-Build (${{ parameters.platform.name }})
+
+ ${{ each property in parameters.platform.jobProperties }}:
+ ${{ property.key }}: ${{ property.value }}
+
+ ${{ if ne(parameters.platform.container, '') }}:
+ container: ${{ parameters.platform.container }}
+ ${{ if ne(parameters.platform.pool, '') }}:
+ pool: ${{ parameters.platform.pool }}
+
+ workspace:
+ clean: all
+
+ steps:
+ - template: /eng/common/templates/steps/source-build.yml
+ parameters:
+ platform: ${{ parameters.platform }}
diff --git a/eng/common/templates/jobs/jobs.yml b/eng/common/templates/jobs/jobs.yml
index c08225a9..08845950 100644
--- a/eng/common/templates/jobs/jobs.yml
+++ b/eng/common/templates/jobs/jobs.yml
@@ -24,6 +24,13 @@ parameters:
# if 'true', the build won't run any of the internal only steps, even if it is running in non-public projects.
runAsPublic: false
+ # Optional: Enable running the source-build jobs to build repo from source
+ runSourceBuild: false
+
+ # Optional: Parameters for source-build template.
+ # See /eng/common/templates/jobs/source-build.yml for options
+ sourceBuildParameters: []
+
# Internal resources (telemetry, microbuild) can only be accessed from non-public projects,
# and some (Microbuild) should only be applied to non-PR cases for internal builds.
@@ -43,6 +50,13 @@ jobs:
name: ${{ job.job }}
+- ${{ if eq(parameters.runSourceBuild, true) }}:
+ - template: /eng/common/templates/jobs/source-build.yml
+ parameters:
+ allCompletedJobId: Source_Build_Complete
+ ${{ each parameter in parameters.sourceBuildParameters }}:
+ ${{ parameter.key }}: ${{ parameter.value }}
+
- ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}:
- ${{ if or(eq(parameters.enablePublishBuildAssets, true), eq(parameters.artifacts.publish.manifests, 'true'), ne(parameters.artifacts.publish.manifests, '')) }}:
- template: ../job/publish-build-assets.yml
@@ -55,6 +69,8 @@ jobs:
- ${{ if eq(parameters.publishBuildAssetsDependsOn, '') }}:
- ${{ each job in parameters.jobs }}:
- ${{ job.job }}
+ - ${{ if eq(parameters.runSourceBuild, true) }}:
+ - Source_Build_Complete
pool:
vmImage: vs2017-win2016
runAsPublic: ${{ parameters.runAsPublic }}
diff --git a/eng/common/templates/jobs/source-build.yml b/eng/common/templates/jobs/source-build.yml
new file mode 100644
index 00000000..f463011e
--- /dev/null
+++ b/eng/common/templates/jobs/source-build.yml
@@ -0,0 +1,48 @@
+parameters:
+ # This template adds arcade-powered source-build to CI. A job is created for each platform, as
+ # well as an optional server job that completes when all platform jobs complete.
+
+ # The name of the "join" job for all source-build platforms. If set to empty string, the job is
+ # not included. Existing repo pipelines can use this job depend on all source-build jobs
+ # completing without maintaining a separate list of every single job ID: just depend on this one
+ # server job. By default, not included. Recommended name if used: 'Source_Build_Complete'.
+ allCompletedJobId: ''
+
+ # See /eng/common/templates/job/source-build.yml
+ jobNamePrefix: 'Source_Build'
+
+ # If changed to true, causes this template to include the default platform for a managed-only
+ # repo. The exact Docker image used for this build will be provided by Arcade. This has some risk,
+ # but since the repo is supposed to be managed-only, the risk should be very low.
+ includeDefaultManagedPlatform: false
+ defaultManagedPlatform:
+ name: 'Managed'
+ container: 'mcr.microsoft.com/dotnet-buildtools/prereqs:centos-7-3e800f1-20190501005343'
+
+ # Defines the platforms on which to run build jobs. One job is created for each platform, and the
+ # object in this array is sent to the job template as 'platform'.
+ platforms: []
+
+jobs:
+
+- ${{ if ne(parameters.allCompletedJobId, '') }}:
+ - job: ${{ parameters.allCompletedJobId }}
+ displayName: Source-Build Complete
+ pool: server
+ dependsOn:
+ - ${{ each platform in parameters.platforms }}:
+ - ${{ parameters.jobNamePrefix }}_${{ platform.name }}
+ - ${{ if eq(parameters.includeDefaultManagedPlatform, true) }}:
+ - ${{ parameters.jobNamePrefix }}_${{ parameters.defaultManagedPlatform.name }}
+
+- ${{ each platform in parameters.platforms }}:
+ - template: /eng/common/templates/job/source-build.yml
+ parameters:
+ jobNamePrefix: ${{ parameters.jobNamePrefix }}
+ platform: ${{ platform }}
+
+- ${{ if eq(parameters.includeDefaultManagedPlatform, true) }}:
+ - template: /eng/common/templates/job/source-build.yml
+ parameters:
+ jobNamePrefix: ${{ parameters.jobNamePrefix }}
+ platform: ${{ parameters.defaultManagedPlatform }}
diff --git a/eng/common/templates/post-build/channels/generic-internal-channel.yml b/eng/common/templates/post-build/channels/generic-internal-channel.yml
index 59eb93a4..7ae52559 100644
--- a/eng/common/templates/post-build/channels/generic-internal-channel.yml
+++ b/eng/common/templates/post-build/channels/generic-internal-channel.yml
@@ -4,7 +4,7 @@ parameters:
artifactsPublishingAdditionalParameters: ''
dependsOn:
- Validate
- publishInstallersAndChecksums: false
+ publishInstallersAndChecksums: true
symbolPublishingAdditionalParameters: ''
stageName: ''
channelName: ''
@@ -158,7 +158,7 @@ stages:
/p:BlobBasePath='$(Build.ArtifactStagingDirectory)/BlobArtifacts/'
/p:PackageBasePath='$(Build.ArtifactStagingDirectory)/PackageArtifacts/'
/p:Configuration=Release
- /p:PublishInstallersAndChecksums=true
+ /p:PublishInstallersAndChecksums=${{ parameters.publishInstallersAndChecksums }}
/p:ChecksumsTargetStaticFeed=$(InternalChecksumsBlobFeedUrl)
/p:ChecksumsAzureAccountKey=$(InternalChecksumsBlobFeedKey)
/p:InstallersTargetStaticFeed=$(InternalInstallersBlobFeedUrl)
diff --git a/eng/common/templates/post-build/channels/generic-public-channel.yml b/eng/common/templates/post-build/channels/generic-public-channel.yml
index 7e80a621..6cf39dbb 100644
--- a/eng/common/templates/post-build/channels/generic-public-channel.yml
+++ b/eng/common/templates/post-build/channels/generic-public-channel.yml
@@ -4,7 +4,7 @@ parameters:
artifactsPublishingAdditionalParameters: ''
dependsOn:
- Validate
- publishInstallersAndChecksums: false
+ publishInstallersAndChecksums: true
symbolPublishingAdditionalParameters: ''
stageName: ''
channelName: ''
diff --git a/eng/common/templates/post-build/post-build.yml b/eng/common/templates/post-build/post-build.yml
index 630a99d4..0854e489 100644
--- a/eng/common/templates/post-build/post-build.yml
+++ b/eng/common/templates/post-build/post-build.yml
@@ -19,7 +19,7 @@ parameters:
enableSigningValidation: true
enableSymbolValidation: false
enableNugetValidation: true
- publishInstallersAndChecksums: false
+ publishInstallersAndChecksums: true
SDLValidationParameters:
enable: false
continueOnError: false
@@ -78,21 +78,22 @@ stages:
BARBuildId: ${{ parameters.BARBuildId }}
PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }}
- - job:
- displayName: Post-build Checks
- dependsOn: setupMaestroVars
- variables:
- - name: TargetChannels
- value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.TargetChannels'] ]
- pool:
- vmImage: 'windows-2019'
- steps:
- - task: PowerShell@2
- displayName: Maestro Channels Consistency
- inputs:
- filePath: $(Build.SourcesDirectory)/eng/common/post-build/check-channel-consistency.ps1
- arguments: -PromoteToChannels "$(TargetChannels)"
- -AvailableChannelIds ${{parameters.NetEngLatestChannelId}},${{parameters.NetEngValidationChannelId}},${{parameters.NetDev5ChannelId}},${{parameters.NetDev6ChannelId}},${{parameters.GeneralTestingChannelId}},${{parameters.NETCoreToolingDevChannelId}},${{parameters.NETCoreToolingReleaseChannelId}},${{parameters.NETInternalToolingChannelId}},${{parameters.NETCoreExperimentalChannelId}},${{parameters.NetEngServicesIntChannelId}},${{parameters.NetEngServicesProdChannelId}},${{parameters.Net5Preview8ChannelId}},${{parameters.Net5RC1ChannelId}},${{parameters.Net5RC2ChannelId}},${{parameters.NetCoreSDK313xxChannelId}},${{parameters.NetCoreSDK313xxInternalChannelId}},${{parameters.NetCoreSDK314xxChannelId}},${{parameters.NetCoreSDK314xxInternalChannelId}},${{parameters.VS166ChannelId}},${{parameters.VS167ChannelId}},${{parameters.VS168ChannelId}},${{parameters.VSMasterChannelId}}
+ - ${{ if and(le(parameters.publishingInfraVersion, 2), eq(parameters.inline, 'true')) }}:
+ - job:
+ displayName: Post-build Checks
+ dependsOn: setupMaestroVars
+ variables:
+ - name: TargetChannels
+ value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.TargetChannels'] ]
+ pool:
+ vmImage: 'windows-2019'
+ steps:
+ - task: PowerShell@2
+ displayName: Maestro Channels Consistency
+ inputs:
+ filePath: $(Build.SourcesDirectory)/eng/common/post-build/check-channel-consistency.ps1
+ arguments: -PromoteToChannels "$(TargetChannels)"
+ -AvailableChannelIds ${{parameters.NetEngLatestChannelId}},${{parameters.NetEngValidationChannelId}},${{parameters.NetDev5ChannelId}},${{parameters.NetDev6ChannelId}},${{parameters.GeneralTestingChannelId}},${{parameters.NETCoreToolingDevChannelId}},${{parameters.NETCoreToolingReleaseChannelId}},${{parameters.NETInternalToolingChannelId}},${{parameters.NETCoreExperimentalChannelId}},${{parameters.NetEngServicesIntChannelId}},${{parameters.NetEngServicesProdChannelId}},${{parameters.Net5Preview8ChannelId}},${{parameters.Net5RC1ChannelId}},${{parameters.Net5RC2ChannelId}},${{parameters.NetCoreSDK313xxChannelId}},${{parameters.NetCoreSDK313xxInternalChannelId}},${{parameters.NetCoreSDK314xxChannelId}},${{parameters.NetCoreSDK314xxInternalChannelId}},${{parameters.VS166ChannelId}},${{parameters.VS167ChannelId}},${{parameters.VS168ChannelId}},${{parameters.VSMasterChannelId}}
- job:
displayName: NuGet Validation
diff --git a/eng/common/templates/steps/perf-send-to-helix.yml b/eng/common/templates/steps/perf-send-to-helix.yml
index e003fe2e..a468e92c 100644
--- a/eng/common/templates/steps/perf-send-to-helix.yml
+++ b/eng/common/templates/steps/perf-send-to-helix.yml
@@ -11,7 +11,7 @@ parameters:
WorkItemDirectory: '' # optional -- a payload directory to zip up and send to Helix; requires WorkItemCommand; incompatible with XUnitProjects
CorrelationPayloadDirectory: '' # optional -- a directory to zip up and send to Helix as a correlation payload
IncludeDotNetCli: false # optional -- true will download a version of the .NET CLI onto the Helix machine as a correlation payload; requires DotNetCliPackageType and DotNetCliVersion
- DotNetCliPackageType: '' # optional -- either 'sdk' or 'runtime'; determines whether the sdk or runtime will be sent to Helix; see https://raw.githubusercontent.com/dotnet/core/master/release-notes/releases.json
+ DotNetCliPackageType: '' # optional -- either 'sdk', 'runtime' or 'aspnetcore-runtime'; determines whether the sdk or runtime will be sent to Helix; see https://raw.githubusercontent.com/dotnet/core/master/release-notes/releases.json
DotNetCliVersion: '' # optional -- version of the CLI to send to Helix; based on this: https://raw.githubusercontent.com/dotnet/core/master/release-notes/releases.json
EnableXUnitReporter: false # optional -- true enables XUnit result reporting to Mission Control
WaitForWorkItemCompletion: true # optional -- true will make the task wait until work items have been completed and fail the build if work items fail. False is "fire and forget."
@@ -19,34 +19,18 @@ parameters:
DisplayNamePrefix: 'Send job to Helix' # optional -- rename the beginning of the displayName of the steps in AzDO
condition: succeeded() # optional -- condition for step to execute; defaults to succeeded()
continueOnError: false # optional -- determines whether to continue the build if the step errors; defaults to false
+ osGroup: '' # required -- operating system for the job
steps:
- - powershell: $(Build.SourcesDirectory)\eng\common\msbuild.ps1 $(Build.SourcesDirectory)\eng\common\performance\${{ parameters.ProjectFile }} /restore /t:Test /bl:$(Build.SourcesDirectory)\artifacts\log\$env:BuildConfig\SendToHelix.binlog
- displayName: ${{ parameters.DisplayNamePrefix }} (Windows)
- env:
- BuildConfig: $(_BuildConfig)
- HelixSource: ${{ parameters.HelixSource }}
- HelixType: ${{ parameters.HelixType }}
- HelixBuild: ${{ parameters.HelixBuild }}
- HelixTargetQueues: ${{ parameters.HelixTargetQueues }}
- HelixAccessToken: ${{ parameters.HelixAccessToken }}
- HelixPreCommands: ${{ parameters.HelixPreCommands }}
- HelixPostCommands: ${{ parameters.HelixPostCommands }}
- WorkItemDirectory: ${{ parameters.WorkItemDirectory }}
- CorrelationPayloadDirectory: ${{ parameters.CorrelationPayloadDirectory }}
- IncludeDotNetCli: ${{ parameters.IncludeDotNetCli }}
- DotNetCliPackageType: ${{ parameters.DotNetCliPackageType }}
- DotNetCliVersion: ${{ parameters.DotNetCliVersion }}
- EnableXUnitReporter: ${{ parameters.EnableXUnitReporter }}
- WaitForWorkItemCompletion: ${{ parameters.WaitForWorkItemCompletion }}
- Creator: ${{ parameters.Creator }}
- SYSTEM_ACCESSTOKEN: $(System.AccessToken)
- condition: and(${{ parameters.condition }}, eq(variables['Agent.Os'], 'Windows_NT'))
+- template: /eng/pipelines/common/templates/runtimes/send-to-helix-inner-step.yml
+ parameters:
+ osGroup: ${{ parameters.osGroup }}
+ sendParams: $(Build.SourcesDirectory)/eng/common/performance/${{ parameters.ProjectFile }} /restore /t:Test /bl:$(Build.SourcesDirectory)/artifacts/log/$(_BuildConfig)/SendToHelix.binlog
+ displayName: ${{ parameters.DisplayNamePrefix }}
+ condition: ${{ parameters.condition }}
continueOnError: ${{ parameters.continueOnError }}
- - script: $BUILD_SOURCESDIRECTORY/eng/common/msbuild.sh $BUILD_SOURCESDIRECTORY/eng/common/performance/${{ parameters.ProjectFile }} /restore /t:Test /bl:$BUILD_SOURCESDIRECTORY/artifacts/log/$BuildConfig/SendToHelix.binlog
- displayName: ${{ parameters.DisplayNamePrefix }} (Unix)
- env:
+ environment:
BuildConfig: $(_BuildConfig)
HelixSource: ${{ parameters.HelixSource }}
HelixType: ${{ parameters.HelixType }}
@@ -64,5 +48,3 @@ steps:
WaitForWorkItemCompletion: ${{ parameters.WaitForWorkItemCompletion }}
Creator: ${{ parameters.Creator }}
SYSTEM_ACCESSTOKEN: $(System.AccessToken)
- condition: and(${{ parameters.condition }}, ne(variables['Agent.Os'], 'Windows_NT'))
- continueOnError: ${{ parameters.continueOnError }}
diff --git a/eng/common/templates/steps/publish-logs.yml b/eng/common/templates/steps/publish-logs.yml
index 428026d2..88f238f3 100644
--- a/eng/common/templates/steps/publish-logs.yml
+++ b/eng/common/templates/steps/publish-logs.yml
@@ -3,20 +3,21 @@ parameters:
JobLabel: ''
steps:
-- task: CopyFiles@2
- displayName: Copy Logs to $(Build.StagingDirectory)\BuildLogs
+- task: Powershell@2
+ displayName: Prepare Binlogs to Upload
inputs:
- SourceFolder: $(Build.SourcesDirectory)\artifacts
- Contents: |
- **/*.log
- **/*.binlog
- TargetFolder: '$(Build.StagingDirectory)\BuildLogs'
+ targetType: inline
+ script: |
+ New-Item -ItemType Directory $(Build.SourcesDirectory)/PostBuildLogs/${{parameters.StageLabel}}/${{parameters.JobLabel}}/
+ Move-Item -Path $(Build.SourcesDirectory)/artifacts/log/Debug/* $(Build.SourcesDirectory)/PostBuildLogs/${{parameters.StageLabel}}/${{parameters.JobLabel}}/
continueOnError: true
- condition: succeededOrFailed()
+ condition: always()
-- task: PublishPipelineArtifact@1
- displayName: Publish BuildLogs
+- task: PublishBuildArtifacts@1
+ displayName: Publish Logs
inputs:
- targetPath: '$(Build.StagingDirectory)\BuildLogs'
- artifactName: ${{ parameters.JobLabel }}
- condition: succeededOrFailed()
+ PathtoPublish: '$(Build.SourcesDirectory)/PostBuildLogs'
+ PublishLocation: Container
+ ArtifactName: PostBuildLogs
+ continueOnError: true
+ condition: always()
diff --git a/eng/common/templates/steps/send-to-helix.yml b/eng/common/templates/steps/send-to-helix.yml
index 5eceb487..bb5f1a92 100644
--- a/eng/common/templates/steps/send-to-helix.yml
+++ b/eng/common/templates/steps/send-to-helix.yml
@@ -18,7 +18,7 @@ parameters:
XUnitRuntimeTargetFramework: '' # optional -- framework to use for the xUnit console runner
XUnitRunnerVersion: '' # optional -- version of the xUnit nuget package you wish to use on Helix; required for XUnitProjects
IncludeDotNetCli: false # optional -- true will download a version of the .NET CLI onto the Helix machine as a correlation payload; requires DotNetCliPackageType and DotNetCliVersion
- DotNetCliPackageType: '' # optional -- either 'sdk' or 'runtime'; determines whether the sdk or runtime will be sent to Helix; see https://raw.githubusercontent.com/dotnet/core/master/release-notes/releases-index.json
+ DotNetCliPackageType: '' # optional -- either 'sdk', 'runtime' or 'aspnetcore-runtime'; determines whether the sdk or runtime will be sent to Helix; see https://raw.githubusercontent.com/dotnet/core/master/release-notes/releases-index.json
DotNetCliVersion: '' # optional -- version of the CLI to send to Helix; based on this: https://raw.githubusercontent.com/dotnet/core/master/release-notes/releases-index.json
EnableXUnitReporter: false # optional -- true enables XUnit result reporting to Mission Control
WaitForWorkItemCompletion: true # optional -- true will make the task wait until work items have been completed and fail the build if work items fail. False is "fire and forget."
diff --git a/eng/common/templates/steps/source-build.yml b/eng/common/templates/steps/source-build.yml
new file mode 100644
index 00000000..8e336b7d
--- /dev/null
+++ b/eng/common/templates/steps/source-build.yml
@@ -0,0 +1,66 @@
+parameters:
+ # This template adds arcade-powered source-build to CI.
+
+ # This is a 'steps' template, and is intended for advanced scenarios where the existing build
+ # infra has a careful build methodology that must be followed. For example, a repo
+ # (dotnet/runtime) might choose to clone the GitHub repo only once and store it as a pipeline
+ # artifact for all subsequent jobs to use, to reduce dependence on a strong network connection to
+ # GitHub. Using this steps template leaves room for that infra to be included.
+
+ # Defines the platform on which to run the steps. See 'eng/common/templates/job/source-build.yml'
+ # for details. The entire object is described in the 'job' template for simplicity, even though
+ # the usage of the properties on this object is split between the 'job' and 'steps' templates.
+ platform: {}
+
+steps:
+# Build. Keep it self-contained for simple reusability. (No source-build-specific job variables.)
+- script: |
+ set -x
+ df -h
+
+ buildConfig=Release
+ # Check if AzDO substitutes in a build config from a variable, and use it if so.
+ if [ '$(_BuildConfig)' != '$''(_BuildConfig)' ]; then
+ buildConfig='$(_BuildConfig)'
+ fi
+
+ officialBuildArgs=
+ if [ '${{ and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}' = 'True' ]; then
+ officialBuildArgs='/p:DotNetPublishUsingPipelines=true /p:OfficialBuildId=$(BUILD.BUILDNUMBER)'
+ fi
+
+ targetRidArgs=
+ if [ '${{ parameters.platform.targetRID }}' != '' ]; then
+ targetRidArgs='/p:TargetRid=${{ parameters.platform.targetRID }}'
+ fi
+
+ ${{ coalesce(parameters.platform.buildScript, './build.sh') }} --ci \
+ --configuration $buildConfig \
+ --restore --build --pack --publish \
+ $officialBuildArgs \
+ $targetRidArgs \
+ /p:SourceBuildNonPortable=${{ parameters.platform.nonPortable }} \
+ /p:ArcadeBuildFromSource=true
+ displayName: Build
+
+# Upload build logs for diagnosis.
+- task: CopyFiles@2
+ displayName: Prepare BuildLogs staging directory
+ inputs:
+ SourceFolder: '$(Build.SourcesDirectory)'
+ Contents: |
+ **/*.log
+ **/*.binlog
+ artifacts/source-build/self/prebuilt-report/**
+ TargetFolder: '$(Build.StagingDirectory)/BuildLogs'
+ CleanTargetFolder: true
+ continueOnError: true
+ condition: succeededOrFailed()
+
+- task: PublishPipelineArtifact@1
+ displayName: Publish BuildLogs
+ inputs:
+ targetPath: '$(Build.StagingDirectory)/BuildLogs'
+ artifactName: BuildLogs_SourceBuild_${{ parameters.platform.name }}_Attempt$(System.JobAttempt)
+ continueOnError: true
+ condition: succeededOrFailed()
diff --git a/eng/common/tools.ps1 b/eng/common/tools.ps1
index 1bf21552..9014e062 100644
--- a/eng/common/tools.ps1
+++ b/eng/common/tools.ps1
@@ -240,8 +240,8 @@ function GetDotNetInstallScript([string] $dotnetRoot) {
return $installScript
}
-function InstallDotNetSdk([string] $dotnetRoot, [string] $version, [string] $architecture = '') {
- InstallDotNet $dotnetRoot $version $architecture '' $false $runtimeSourceFeed $runtimeSourceFeedKey
+function InstallDotNetSdk([string] $dotnetRoot, [string] $version, [string] $architecture = '', [switch] $noPath) {
+ InstallDotNet $dotnetRoot $version $architecture '' $false $runtimeSourceFeed $runtimeSourceFeedKey -noPath:$noPath
}
function InstallDotNet([string] $dotnetRoot,
@@ -250,7 +250,8 @@ function InstallDotNet([string] $dotnetRoot,
[string] $runtime = '',
[bool] $skipNonVersionedFiles = $false,
[string] $runtimeSourceFeed = '',
- [string] $runtimeSourceFeedKey = '') {
+ [string] $runtimeSourceFeedKey = '',
+ [switch] $noPath) {
$installScript = GetDotNetInstallScript $dotnetRoot
$installParameters = @{
@@ -261,6 +262,7 @@ function InstallDotNet([string] $dotnetRoot,
if ($architecture) { $installParameters.Architecture = $architecture }
if ($runtime) { $installParameters.Runtime = $runtime }
if ($skipNonVersionedFiles) { $installParameters.SkipNonVersionedFiles = $skipNonVersionedFiles }
+ if ($noPath) { $installParameters.NoPath = $True }
try {
& $installScript @installParameters
@@ -310,9 +312,15 @@ function InitializeVisualStudioMSBuild([bool]$install, [object]$vsRequirements =
return $global:_MSBuildExe
}
- $vsMinVersionReqdStr = '16.5'
+ # Minimum VS version to require.
+ $vsMinVersionReqdStr = '16.8'
$vsMinVersionReqd = [Version]::new($vsMinVersionReqdStr)
+ # If the version of msbuild is going to be xcopied,
+ # use this version. Version matches a package here:
+ # https://dev.azure.com/dnceng/public/_packaging?_a=package&feed=dotnet-eng&package=RoslynTools.MSBuild&protocolType=NuGet&version=16.8.0-preview3&view=overview
+ $defaultXCopyMSBuildVersion = '16.8.0-preview3'
+
if (!$vsRequirements) { $vsRequirements = $GlobalJson.tools.vs }
$vsMinVersionStr = if ($vsRequirements.version) { $vsRequirements.version } else { $vsMinVersionReqdStr }
$vsMinVersion = [Version]::new($vsMinVersionStr)
@@ -347,23 +355,28 @@ function InitializeVisualStudioMSBuild([bool]$install, [object]$vsRequirements =
$xcopyMSBuildVersion = $GlobalJson.tools.'xcopy-msbuild'
$vsMajorVersion = $xcopyMSBuildVersion.Split('.')[0]
} else {
- #if vs version provided in global.json is incompatible then use the default version for xcopy msbuild download
+ #if vs version provided in global.json is incompatible (too low) then use the default version for xcopy msbuild download
if($vsMinVersion -lt $vsMinVersionReqd){
- Write-Host "Using xcopy-msbuild version of $vsMinVersionReqdStr.0-alpha since VS version $vsMinVersionStr provided in global.json is not compatible"
- $vsMajorVersion = $vsMinVersionReqd.Major
- $vsMinorVersion = $vsMinVersionReqd.Minor
+ Write-Host "Using xcopy-msbuild version of $defaultXCopyMSBuildVersion since VS version $vsMinVersionStr provided in global.json is not compatible"
+ $xcopyMSBuildVersion = $defaultXCopyMSBuildVersion
}
else{
+ # If the VS version IS compatible, look for an xcopy msbuild package
+ # with a version matching VS.
+ # Note: If this version does not exist, then an explicit version of xcopy msbuild
+ # can be specified in global.json. This will be required for pre-release versions of msbuild.
$vsMajorVersion = $vsMinVersion.Major
$vsMinorVersion = $vsMinVersion.Minor
+ $xcopyMSBuildVersion = "$vsMajorVersion.$vsMinorVersion.0"
}
-
- $xcopyMSBuildVersion = "$vsMajorVersion.$vsMinorVersion.0-alpha"
}
$vsInstallDir = $null
if ($xcopyMSBuildVersion.Trim() -ine "none") {
$vsInstallDir = InitializeXCopyMSBuild $xcopyMSBuildVersion $install
+ if ($vsInstallDir -eq $null) {
+ throw "Could not xcopy msbuild. Please check that package 'RoslynTools.MSBuild @ $xcopyMSBuildVersion' exists on feed 'dotnet-eng'."
+ }
}
if ($vsInstallDir -eq $null) {
throw 'Unable to find Visual Studio that has required version and components installed'
@@ -527,13 +540,15 @@ function GetDefaultMSBuildEngine() {
function GetNuGetPackageCachePath() {
if ($env:NUGET_PACKAGES -eq $null) {
- # Use local cache on CI to ensure deterministic build,
+ # Use local cache on CI to ensure deterministic build.
+ # Avoid using the http cache as workaround for https://github.com/NuGet/Home/issues/3116
# use global cache in dev builds to avoid cost of downloading packages.
# For directory normalization, see also: https://github.com/NuGet/Home/issues/7968
if ($useGlobalNuGetCache) {
$env:NUGET_PACKAGES = Join-Path $env:UserProfile '.nuget\packages\'
} else {
$env:NUGET_PACKAGES = Join-Path $RepoRoot '.packages\'
+ $env:RESTORENOCACHE = $true
}
}
@@ -716,6 +731,16 @@ function IsWindowsPlatform() {
return [environment]::OSVersion.Platform -eq [PlatformID]::Win32NT
}
+function Get-Darc($version) {
+ $darcPath = "$TempDir\darc\$(New-Guid)"
+ if ($version -ne $null) {
+ & $PSScriptRoot\darc-init.ps1 -toolpath $darcPath -darcVersion $version | Out-Host
+ } else {
+ & $PSScriptRoot\darc-init.ps1 -toolpath $darcPath | Out-Host
+ }
+ return "$darcPath\darc.exe"
+}
+
. $PSScriptRoot\pipeline-logging-functions.ps1
$RepoRoot = Resolve-Path (Join-Path $PSScriptRoot '..\..')
diff --git a/eng/common/tools.sh b/eng/common/tools.sh
index c722a058..b5d63cb1 100755
--- a/eng/common/tools.sh
+++ b/eng/common/tools.sh
@@ -312,12 +312,14 @@ function InitializeBuildTool {
_InitializeBuildToolFramework="netcoreapp2.1"
}
+# Set RestoreNoCache as a workaround for https://github.com/NuGet/Home/issues/3116
function GetNuGetPackageCachePath {
if [[ -z ${NUGET_PACKAGES:-} ]]; then
if [[ "$use_global_nuget_cache" == true ]]; then
export NUGET_PACKAGES="$HOME/.nuget/packages"
else
export NUGET_PACKAGES="$repo_root/.packages"
+ export RESTORENOCACHE=true
fi
fi
diff --git a/global.json b/global.json
index 4e51b9b1..44d95934 100644
--- a/global.json
+++ b/global.json
@@ -11,7 +11,7 @@
},
"msbuild-sdks": {
- "Microsoft.DotNet.Arcade.Sdk": "5.0.0-beta.20426.1",
- "Microsoft.DotNet.Helix.Sdk": "5.0.0-beta.20426.1"
+ "Microsoft.DotNet.Arcade.Sdk": "5.0.0-beta.20510.1",
+ "Microsoft.DotNet.Helix.Sdk": "5.0.0-beta.20510.1"
}
}