Browse Source

Bump Arcade to 5.0.0-beta

pull/938/head
Kévin Chalet 6 years ago
parent
commit
d795296489
  1. 2
      Directory.Build.targets
  2. 8
      eng/Version.Details.xml
  3. 158
      eng/common/CheckSymbols.ps1
  4. 83
      eng/common/PublishToPackageFeed.proj
  5. 82
      eng/common/PublishToSymbolServers.proj
  6. 141
      eng/common/SetupNugetSources.ps1
  7. 149
      eng/common/SetupNugetSources.sh
  8. 83
      eng/common/SigningValidation.proj
  9. 184
      eng/common/SourceLinkValidation.ps1
  10. 102
      eng/common/build.ps1
  11. 24
      eng/common/build.sh
  12. 41
      eng/common/cross/android/arm/toolchain.cmake
  13. 42
      eng/common/cross/android/arm64/toolchain.cmake
  14. 106
      eng/common/cross/build-android-rootfs.sh
  15. 100
      eng/common/cross/build-rootfs.sh
  16. 81
      eng/common/cross/toolchain.cmake
  17. 30
      eng/common/darc-init.ps1
  18. 30
      eng/common/darc-init.sh
  19. 23
      eng/common/dotnet-install.ps1
  20. 50
      eng/common/dotnet-install.sh
  21. 11
      eng/common/enable-cross-org-publishing.ps1
  22. 51
      eng/common/generate-graph-files.ps1
  23. 37
      eng/common/init-tools-native.ps1
  24. 50
      eng/common/init-tools-native.sh
  25. 28
      eng/common/internal-feed-operations.ps1
  26. 4
      eng/common/internal-feed-operations.sh
  27. 5
      eng/common/msbuild.ps1
  28. 10
      eng/common/native/common-library.sh
  29. 4
      eng/common/native/install-cmake-test.sh
  30. 4
      eng/common/native/install-cmake.sh
  31. 12
      eng/common/native/install-tool.ps1
  32. 21
      eng/common/performance/perfhelixpublish.proj
  33. 7
      eng/common/performance/performance-setup.ps1
  34. 6
      eng/common/performance/performance-setup.sh
  35. 70
      eng/common/pipeline-logging-functions.ps1
  36. 11
      eng/common/pipeline-logging-functions.sh
  37. 20
      eng/common/post-build/add-build-to-channel.ps1
  38. 25
      eng/common/post-build/check-channel-consistency.ps1
  39. 45
      eng/common/post-build/darc-gather-drop.ps1
  40. 11
      eng/common/post-build/nuget-validation.ps1
  41. 27
      eng/common/post-build/post-build-utils.ps1
  42. 26
      eng/common/post-build/setup-maestro-vars.ps1
  43. 42
      eng/common/post-build/sourcelink-validation.ps1
  44. 71
      eng/common/post-build/symbols-validation.ps1
  45. 87
      eng/common/post-build/trigger-subscriptions.ps1
  46. 19
      eng/common/sdk-task.ps1
  47. 180
      eng/common/sdl/execute-all-sdl-tools.ps1
  48. 97
      eng/common/sdl/extract-artifact-packages.ps1
  49. 42
      eng/common/sdl/init-sdl.ps1
  50. 94
      eng/common/sdl/push-gdn.ps1
  51. 88
      eng/common/sdl/run-sdl.ps1
  52. 52
      eng/common/templates/job/execute-sdl.yml
  53. 142
      eng/common/templates/job/job.yml
  54. 6
      eng/common/templates/job/publish-build-assets.yml
  55. 74
      eng/common/templates/jobs/jobs.yml
  56. 118
      eng/common/templates/post-build/channels/generic-internal-channel.yml
  57. 178
      eng/common/templates/post-build/channels/generic-public-channel.yml
  58. 95
      eng/common/templates/post-build/channels/netcore-3-tools-validation.yml
  59. 130
      eng/common/templates/post-build/channels/netcore-3-tools.yml
  60. 130
      eng/common/templates/post-build/channels/netcore-dev-31.yml
  61. 130
      eng/common/templates/post-build/channels/netcore-dev-5.yml
  62. 130
      eng/common/templates/post-build/channels/netcore-release-30.yml
  63. 130
      eng/common/templates/post-build/channels/netcore-release-31.yml
  64. 130
      eng/common/templates/post-build/channels/netcore-tools-latest.yml
  65. 95
      eng/common/templates/post-build/channels/netcore-tools-validation.yml
  66. 43
      eng/common/templates/post-build/common-variables.yml
  67. 23
      eng/common/templates/post-build/darc-gather-drop.yml
  68. 407
      eng/common/templates/post-build/post-build.yml
  69. 25
      eng/common/templates/post-build/promote-build.yml
  70. 67
      eng/common/templates/post-build/setup-maestro-vars.yml
  71. 4
      eng/common/templates/steps/add-build-to-channel.yml
  72. 23
      eng/common/templates/steps/publish-logs.yml
  73. 3
      eng/common/templates/steps/send-to-helix.yml
  74. 261
      eng/common/tools.ps1
  75. 108
      eng/common/tools.sh
  76. 4
      global.json

2
Directory.Build.targets

@ -26,7 +26,7 @@
</PropertyGroup> </PropertyGroup>
<ItemGroup> <ItemGroup>
<EmbeddedFiles Include="$(GeneratedAssemblyInfoFile)"/> <EmbeddedFiles Include="$(GeneratedAssemblyInfoFile)" />
</ItemGroup> </ItemGroup>
<!-- <!--

8
eng/Version.Details.xml

@ -5,14 +5,14 @@
</ProductDependencies> </ProductDependencies>
<ToolsetDependencies> <ToolsetDependencies>
<Dependency Name="Microsoft.DotNet.Arcade.Sdk" Version="1.0.0-beta.19517.3"> <Dependency Name="Microsoft.DotNet.Arcade.Sdk" Version="5.0.0-beta.20180.5">
<Uri>https://github.com/dotnet/arcade</Uri> <Uri>https://github.com/dotnet/arcade</Uri>
<Sha>a42a124635ce1a218309ecb31ec59d559cacb886</Sha> <Sha>09bb9d929120b402348c9a0e9c8c951e824059aa</Sha>
</Dependency> </Dependency>
<Dependency Name="Microsoft.DotNet.Helix.Sdk" Version="2.0.0-beta.19517.3"> <Dependency Name="Microsoft.DotNet.Helix.Sdk" Version="5.0.0-beta.20180.5">
<Uri>https://github.com/dotnet/arcade</Uri> <Uri>https://github.com/dotnet/arcade</Uri>
<Sha>a42a124635ce1a218309ecb31ec59d559cacb886</Sha> <Sha>09bb9d929120b402348c9a0e9c8c951e824059aa</Sha>
</Dependency> </Dependency>
</ToolsetDependencies> </ToolsetDependencies>

158
eng/common/CheckSymbols.ps1

@ -1,158 +0,0 @@
param(
[Parameter(Mandatory=$true)][string] $InputPath, # Full path to directory where NuGet packages to be checked are stored
[Parameter(Mandatory=$true)][string] $ExtractPath, # Full path to directory where the packages will be extracted during validation
[Parameter(Mandatory=$true)][string] $SymbolToolPath # Full path to directory where dotnet symbol-tool was installed
)
Add-Type -AssemblyName System.IO.Compression.FileSystem
function FirstMatchingSymbolDescriptionOrDefault {
param(
[string] $FullPath, # Full path to the module that has to be checked
[string] $TargetServerParam, # Parameter to pass to `Symbol Tool` indicating the server to lookup for symbols
[string] $SymbolsPath
)
$FileName = [System.IO.Path]::GetFileName($FullPath)
$Extension = [System.IO.Path]::GetExtension($FullPath)
# Those below are potential symbol files that the `dotnet symbol` might
# return. Which one will be returned depend on the type of file we are
# checking and which type of file was uploaded.
# The file itself is returned
$SymbolPath = $SymbolsPath + "\" + $FileName
# PDB file for the module
$PdbPath = $SymbolPath.Replace($Extension, ".pdb")
# PDB file for R2R module (created by crossgen)
$NGenPdb = $SymbolPath.Replace($Extension, ".ni.pdb")
# DBG file for a .so library
$SODbg = $SymbolPath.Replace($Extension, ".so.dbg")
# DWARF file for a .dylib
$DylibDwarf = $SymbolPath.Replace($Extension, ".dylib.dwarf")
.\dotnet-symbol.exe --symbols --modules --windows-pdbs $TargetServerParam $FullPath -o $SymbolsPath | Out-Null
if (Test-Path $PdbPath) {
return "PDB"
}
elseif (Test-Path $NGenPdb) {
return "NGen PDB"
}
elseif (Test-Path $SODbg) {
return "DBG for SO"
}
elseif (Test-Path $DylibDwarf) {
return "Dwarf for Dylib"
}
elseif (Test-Path $SymbolPath) {
return "Module"
}
else {
return $null
}
}
function CountMissingSymbols {
param(
[string] $PackagePath # Path to a NuGet package
)
# Ensure input file exist
if (!(Test-Path $PackagePath)) {
throw "Input file does not exist: $PackagePath"
}
# Extensions for which we'll look for symbols
$RelevantExtensions = @(".dll", ".exe", ".so", ".dylib")
# How many files are missing symbol information
$MissingSymbols = 0
$PackageId = [System.IO.Path]::GetFileNameWithoutExtension($PackagePath)
$PackageGuid = New-Guid
$ExtractPath = Join-Path -Path $ExtractPath -ChildPath $PackageGuid
$SymbolsPath = Join-Path -Path $ExtractPath -ChildPath "Symbols"
[System.IO.Compression.ZipFile]::ExtractToDirectory($PackagePath, $ExtractPath)
# Makes easier to reference `symbol tool`
Push-Location $SymbolToolPath
Get-ChildItem -Recurse $ExtractPath |
Where-Object {$RelevantExtensions -contains $_.Extension} |
ForEach-Object {
if ($_.FullName -Match "\\ref\\") {
Write-Host "`t Ignoring reference assembly file" $_.FullName
return
}
$SymbolsOnMSDL = FirstMatchingSymbolDescriptionOrDefault $_.FullName "--microsoft-symbol-server" $SymbolsPath
$SymbolsOnSymWeb = FirstMatchingSymbolDescriptionOrDefault $_.FullName "--internal-server" $SymbolsPath
Write-Host -NoNewLine "`t Checking file" $_.FullName "... "
if ($SymbolsOnMSDL -ne $null -and $SymbolsOnSymWeb -ne $null) {
Write-Host "Symbols found on MSDL (" $SymbolsOnMSDL ") and SymWeb (" $SymbolsOnSymWeb ")"
}
else {
$MissingSymbols++
if ($SymbolsOnMSDL -eq $null -and $SymbolsOnSymWeb -eq $null) {
Write-Host "No symbols found on MSDL or SymWeb!"
}
else {
if ($SymbolsOnMSDL -eq $null) {
Write-Host "No symbols found on MSDL!"
}
else {
Write-Host "No symbols found on SymWeb!"
}
}
}
}
Pop-Location
return $MissingSymbols
}
function CheckSymbolsAvailable {
if (Test-Path $ExtractPath) {
Remove-Item $ExtractPath -Force -Recurse -ErrorAction SilentlyContinue
}
Get-ChildItem "$InputPath\*.nupkg" |
ForEach-Object {
$FileName = $_.Name
# These packages from Arcade-Services include some native libraries that
# our current symbol uploader can't handle. Below is a workaround until
# we get issue: https://github.com/dotnet/arcade/issues/2457 sorted.
if ($FileName -Match "Microsoft\.DotNet\.Darc\.") {
Write-Host "Ignoring Arcade-services file: $FileName"
Write-Host
return
}
elseif ($FileName -Match "Microsoft\.DotNet\.Maestro\.Tasks\.") {
Write-Host "Ignoring Arcade-services file: $FileName"
Write-Host
return
}
Write-Host "Validating $FileName "
$Status = CountMissingSymbols "$InputPath\$FileName"
if ($Status -ne 0) {
Write-Error "Missing symbols for $Status modules in the package $FileName"
}
Write-Host
}
}
CheckSymbolsAvailable

83
eng/common/PublishToPackageFeed.proj

@ -1,83 +0,0 @@
<!-- Licensed to the .NET Foundation under one or more agreements. The .NET Foundation licenses this file to you under the MIT license. See the LICENSE file in the project root for more information. -->
<Project Sdk="Microsoft.NET.Sdk">
<!--
This MSBuild file is intended to be used as the body of the default
publishing release pipeline. The release pipeline will use this file
to invoke the PushToStaticFeed task that will read the build asset
manifest and publish the assets described in the manifest to
informed target feeds.
-->
<PropertyGroup>
<TargetFramework>netcoreapp2.1</TargetFramework>
</PropertyGroup>
<Import Project="$(MSBuildThisFileDirectory)DefaultVersions.props" Condition="Exists('$(MSBuildThisFileDirectory)DefaultVersions.props')" />
<!--
This won't be necessary once we solve this issue:
https://github.com/dotnet/arcade/issues/2266
-->
<Import Project="$(MSBuildThisFileDirectory)ArtifactsCategory.props" Condition="Exists('$(MSBuildThisFileDirectory)ArtifactsCategory.props')" />
<Import Project="$(NuGetPackageRoot)microsoft.dotnet.build.tasks.feed\$(MicrosoftDotNetBuildTasksFeedVersion)\build\Microsoft.DotNet.Build.Tasks.Feed.targets" />
<Target Name="PublishToFeed">
<Error Condition="'$(ArtifactsCategory)' == ''" Text="ArtifactsCategory: The artifacts' category produced by the build wasn't provided." />
<Error Condition="'$(AccountKeyToStaticFeed)' == ''" Text="AccountKeyToStaticFeed: Account key for target feed wasn't provided." />
<Error Condition="'$(ManifestsBasePath)' == ''" Text="Full path to asset manifests directory wasn't provided." />
<Error Condition="'$(BlobBasePath)' == '' AND '$(PackageBasePath)' == ''" Text="A valid full path to BlobBasePath of PackageBasePath is required." />
<ItemGroup>
<!-- Include all manifests found in the manifest folder. -->
<ManifestFiles Include="$(ManifestsBasePath)*.xml" />
</ItemGroup>
<Error Condition="'@(ManifestFiles)' == ''" Text="No manifest file was found in the provided path: $(ManifestsBasePath)" />
<!--
For now the type of packages being published will be informed for the whole build.
Eventually this will be specified on a per package basis:
TODO: https://github.com/dotnet/arcade/issues/2266
-->
<PropertyGroup>
<TargetStaticFeed Condition="'$(ArtifactsCategory.ToUpper())' == '.NETCORE'">https://dotnetfeed.blob.core.windows.net/dotnet-core/index.json</TargetStaticFeed>
<TargetStaticFeed Condition="'$(ArtifactsCategory.ToUpper())' == '.NETCOREVALIDATION'">https://dotnetfeed.blob.core.windows.net/arcade-validation/index.json</TargetStaticFeed>
<TargetStaticFeed Condition="'$(ArtifactsCategory.ToUpper())' == 'ASPNETCORE'">https://dotnetfeed.blob.core.windows.net/aspnet-aspnetcore/index.json</TargetStaticFeed>
<TargetStaticFeed Condition="'$(ArtifactsCategory.ToUpper())' == 'ASPNETCORETOOLING'">https://dotnetfeed.blob.core.windows.net/aspnet-aspnetcore-tooling/index.json</TargetStaticFeed>
<TargetStaticFeed Condition="'$(ArtifactsCategory.ToUpper())' == 'ENTITYFRAMEWORKCORE'">https://dotnetfeed.blob.core.windows.net/aspnet-entityframeworkcore/index.json</TargetStaticFeed>
<TargetStaticFeed Condition="'$(ArtifactsCategory.ToUpper())' == 'ASPNETEXTENSIONS'">https://dotnetfeed.blob.core.windows.net/aspnet-extensions/index.json</TargetStaticFeed>
<TargetStaticFeed Condition="'$(ArtifactsCategory.ToUpper())' == 'CORECLR'">https://dotnetfeed.blob.core.windows.net/dotnet-coreclr/index.json</TargetStaticFeed>
<TargetStaticFeed Condition="'$(ArtifactsCategory.ToUpper())' == 'CORESDK'">https://dotnetfeed.blob.core.windows.net/dotnet-sdk/index.json</TargetStaticFeed>
<TargetStaticFeed Condition="'$(ArtifactsCategory.ToUpper())' == 'TOOLSINTERNAL'">https://dotnetfeed.blob.core.windows.net/dotnet-tools-internal/index.json</TargetStaticFeed>
<TargetStaticFeed Condition="'$(ArtifactsCategory.ToUpper())' == 'TOOLSET'">https://dotnetfeed.blob.core.windows.net/dotnet-toolset/index.json</TargetStaticFeed>
<TargetStaticFeed Condition="'$(ArtifactsCategory.ToUpper())' == 'WINDOWSDESKTOP'">https://dotnetfeed.blob.core.windows.net/dotnet-windowsdesktop/index.json</TargetStaticFeed>
<TargetStaticFeed Condition="'$(ArtifactsCategory.ToUpper())' == 'NUGETCLIENT'">https://dotnetfeed.blob.core.windows.net/nuget-nugetclient/index.json</TargetStaticFeed>
<TargetStaticFeed Condition="'$(ArtifactsCategory.ToUpper())' == 'ASPNETENTITYFRAMEWORK6'">https://dotnetfeed.blob.core.windows.net/aspnet-entityframework6/index.json</TargetStaticFeed>
<TargetStaticFeed Condition="'$(ArtifactsCategory.ToUpper())' == 'ASPNETBLAZOR'">https://dotnetfeed.blob.core.windows.net/aspnet-blazor/index.json</TargetStaticFeed>
</PropertyGroup>
<Error
Condition="'$(TargetStaticFeed)' == ''"
Text="'$(ArtifactsCategory)' wasn't recognized as a valid artifact category. Valid categories are: '.NetCore' and '.NetCoreValidation'" />
<!-- Iterate publishing assets from each manifest file. -->
<PushArtifactsInManifestToFeed
ExpectedFeedUrl="$(TargetStaticFeed)"
AccountKey="$(AccountKeyToStaticFeed)"
BARBuildId="$(BARBuildId)"
MaestroApiEndpoint="$(MaestroApiEndpoint)"
BuildAssetRegistryToken="$(BuildAssetRegistryToken)"
Overwrite="$(OverrideAssetsWithSameName)"
PassIfExistingItemIdentical="$(PassIfExistingItemIdentical)"
MaxClients="$(MaxParallelUploads)"
UploadTimeoutInMinutes="$(MaxUploadTimeoutInMinutes)"
AssetManifestPath="%(ManifestFiles.Identity)"
BlobAssetsBasePath="$(BlobBasePath)"
PackageAssetsBasePath="$(PackageBasePath)"/>
</Target>
<ItemGroup>
<PackageReference Include="Microsoft.DotNet.Build.Tasks.Feed" Version="$(MicrosoftDotNetBuildTasksFeedVersion)" />
</ItemGroup>
</Project>

82
eng/common/PublishToSymbolServers.proj

@ -1,82 +0,0 @@
<!-- Licensed to the .NET Foundation under one or more agreements. The .NET Foundation licenses this file to you under the MIT license. See the LICENSE file in the project root for more information. -->
<Project Sdk="Microsoft.NET.Sdk">
<!--
This MSBuild file is intended to be used as the body of the default
publishing release pipeline. The release pipeline will use this file
to invoke the PublishSymbols tasks to publish symbols to MSDL and SymWeb.
Parameters:
- PDBArtifactsDirectory : Full path to directory containing PDB files to be published.
- BlobBasePath : Full path containing *.symbols.nupkg packages to be published.
- DotNetSymbolServerTokenMsdl : PAT to access MSDL.
- DotNetSymbolServerTokenSymWeb : PAT to access SymWeb.
- DotNetSymbolExpirationInDays : Expiration days for published packages. Default is 3650.
-->
<PropertyGroup>
<TargetFramework>netcoreapp2.1</TargetFramework>
</PropertyGroup>
<Import Project="$(NuGetPackageRoot)microsoft.symboluploader.build.task\$(SymbolUploaderVersion)\build\PublishSymbols.targets" />
<Target Name="PublishSymbols">
<ItemGroup>
<FilesToPublishToSymbolServer Include="$(PDBArtifactsDirectory)\*.pdb"/>
<PackagesToPublishToSymbolServer Include="$(BlobBasePath)\*.symbols.nupkg"/>
<!--
These packages from Arcade-Services include some native libraries that
our current symbol uploader can't handle. Below is a workaround until
we get issue: https://github.com/dotnet/arcade/issues/2457 sorted.
-->
<PackagesToPublishToSymbolServer Remove="$(BlobBasePath)\Microsoft.DotNet.Darc.*" />
<PackagesToPublishToSymbolServer Remove="$(BlobBasePath)\Microsoft.DotNet.Maestro.Tasks.*" />
</ItemGroup>
<PropertyGroup>
<DotNetSymbolExpirationInDays Condition="'$(DotNetSymbolExpirationInDays)' == ''">3650</DotNetSymbolExpirationInDays>
<PublishToSymbolServer>true</PublishToSymbolServer>
<PublishToSymbolServer Condition="'@(FilesToPublishToSymbolServer)' == '' and '@(PackagesToPublishToSymbolServer)' == ''">false</PublishToSymbolServer>
</PropertyGroup>
<Message
Importance="High"
Text="No symbol package(s) were found to publish."
Condition="$(PublishToSymbolServer) == false" />
<!-- Symbol Uploader: MSDL -->
<Message Importance="High" Text="Publishing symbol packages to MSDL ..." Condition="$(PublishToSymbolServer)" />
<PublishSymbols PackagesToPublish="@(PackagesToPublishToSymbolServer)"
FilesToPublish="@(FilesToPublishToSymbolServer)"
PersonalAccessToken="$(DotNetSymbolServerTokenMsdl)"
SymbolServerPath="https://microsoftpublicsymbols.artifacts.visualstudio.com/DefaultCollection"
ExpirationInDays="$(DotNetSymbolExpirationInDays)"
VerboseLogging="true"
DryRun="false"
ConvertPortablePdbsToWindowsPdbs="false"
PdbConversionTreatAsWarning=""
Condition="$(PublishToSymbolServer)"/>
<!--
Symbol Uploader: SymWeb
Watson, VS insertion testings and the typical internal dev usage require SymWeb.
Currently we need to call the task twice (https://github.com/dotnet/core-eng/issues/3489).
-->
<Message Importance="High" Text="Publishing symbol packages to SymWeb ..." Condition="$(PublishToSymbolServer)" />
<PublishSymbols PackagesToPublish="@(PackagesToPublishToSymbolServer)"
FilesToPublish="@(FilesToPublishToSymbolServer)"
PersonalAccessToken="$(DotNetSymbolServerTokenSymWeb)"
SymbolServerPath="https://microsoft.artifacts.visualstudio.com/DefaultCollection"
ExpirationInDays="$(DotNetSymbolExpirationInDays)"
VerboseLogging="true"
DryRun="false"
ConvertPortablePdbsToWindowsPdbs="false"
PdbConversionTreatAsWarning=""
Condition="$(PublishToSymbolServer)"/>
</Target>
<ItemGroup>
<PackageReference Include="Microsoft.SymbolUploader.Build.Task" Version="$(SymbolUploaderVersion)" />
</ItemGroup>
</Project>

141
eng/common/SetupNugetSources.ps1

@ -0,0 +1,141 @@
# This file is a temporary workaround for internal builds to be able to restore from private AzDO feeds.
# This file should be removed as part of this issue: https://github.com/dotnet/arcade/issues/4080
#
# What the script does is iterate over all package sources in the pointed NuGet.config and add a credential entry
# under <packageSourceCredentials> for each Maestro managed private feed. Two additional credential
# entries are also added for the two private static internal feeds: dotnet3-internal and dotnet3-internal-transport.
#
# This script needs to be called in every job that will restore packages and which the base repo has
# private AzDO feeds in the NuGet.config.
#
# See example YAML call for this script below. Note the use of the variable `$(dn-bot-dnceng-artifact-feeds-rw)`
# from the AzureDevOps-Artifact-Feeds-Pats variable group.
#
# - task: PowerShell@2
# displayName: Setup Private Feeds Credentials
# condition: eq(variables['Agent.OS'], 'Windows_NT')
# inputs:
# filePath: $(Build.SourcesDirectory)/eng/common/SetupNugetSources.ps1
# arguments: -ConfigFile $(Build.SourcesDirectory)/NuGet.config -Password $Env:Token
# env:
# Token: $(dn-bot-dnceng-artifact-feeds-rw)
[CmdletBinding()]
param (
[Parameter(Mandatory = $true)][string]$ConfigFile,
[Parameter(Mandatory = $true)][string]$Password
)
$ErrorActionPreference = "Stop"
Set-StrictMode -Version 2.0
[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12
. $PSScriptRoot\tools.ps1
# Add source entry to PackageSources
function AddPackageSource($sources, $SourceName, $SourceEndPoint, $creds, $Username, $Password) {
$packageSource = $sources.SelectSingleNode("add[@key='$SourceName']")
if ($packageSource -eq $null)
{
$packageSource = $doc.CreateElement("add")
$packageSource.SetAttribute("key", $SourceName)
$packageSource.SetAttribute("value", $SourceEndPoint)
$sources.AppendChild($packageSource) | Out-Null
}
else {
Write-Host "Package source $SourceName already present."
}
AddCredential -Creds $creds -Source $SourceName -Username $Username -Password $Password
}
# Add a credential node for the specified source
function AddCredential($creds, $source, $username, $password) {
# Looks for credential configuration for the given SourceName. Create it if none is found.
$sourceElement = $creds.SelectSingleNode($Source)
if ($sourceElement -eq $null)
{
$sourceElement = $doc.CreateElement($Source)
$creds.AppendChild($sourceElement) | Out-Null
}
# Add the <Username> node to the credential if none is found.
$usernameElement = $sourceElement.SelectSingleNode("add[@key='Username']")
if ($usernameElement -eq $null)
{
$usernameElement = $doc.CreateElement("add")
$usernameElement.SetAttribute("key", "Username")
$sourceElement.AppendChild($usernameElement) | Out-Null
}
$usernameElement.SetAttribute("value", $Username)
# Add the <ClearTextPassword> to the credential if none is found.
# Add it as a clear text because there is no support for encrypted ones in non-windows .Net SDKs.
# -> https://github.com/NuGet/Home/issues/5526
$passwordElement = $sourceElement.SelectSingleNode("add[@key='ClearTextPassword']")
if ($passwordElement -eq $null)
{
$passwordElement = $doc.CreateElement("add")
$passwordElement.SetAttribute("key", "ClearTextPassword")
$sourceElement.AppendChild($passwordElement) | Out-Null
}
$passwordElement.SetAttribute("value", $Password)
}
function InsertMaestroPrivateFeedCredentials($Sources, $Creds, $Password) {
$maestroPrivateSources = $Sources.SelectNodes("add[contains(@key,'darc-int')]")
Write-Host "Inserting credentials for $($maestroPrivateSources.Count) Maestro's private feeds."
ForEach ($PackageSource in $maestroPrivateSources) {
Write-Host "`tInserting credential for Maestro's feed:" $PackageSource.Key
AddCredential -Creds $creds -Source $PackageSource.Key -Username $Username -Password $Password
}
}
if (!(Test-Path $ConfigFile -PathType Leaf)) {
Write-PipelineTelemetryError -Category 'Build' -Message "Eng/common/SetupNugetSources.ps1 returned a non-zero exit code. Couldn't find the NuGet config file: $ConfigFile"
ExitWithExitCode 1
}
if (!$Password) {
Write-PipelineTelemetryError -Category 'Build' -Message 'Eng/common/SetupNugetSources.ps1 returned a non-zero exit code. Please supply a valid PAT'
ExitWithExitCode 1
}
# Load NuGet.config
$doc = New-Object System.Xml.XmlDocument
$filename = (Get-Item $ConfigFile).FullName
$doc.Load($filename)
# Get reference to <PackageSources> or create one if none exist already
$sources = $doc.DocumentElement.SelectSingleNode("packageSources")
if ($sources -eq $null) {
$sources = $doc.CreateElement("packageSources")
$doc.DocumentElement.AppendChild($sources) | Out-Null
}
# Looks for a <PackageSourceCredentials> node. Create it if none is found.
$creds = $doc.DocumentElement.SelectSingleNode("packageSourceCredentials")
if ($creds -eq $null) {
$creds = $doc.CreateElement("packageSourceCredentials")
$doc.DocumentElement.AppendChild($creds) | Out-Null
}
# Insert credential nodes for Maestro's private feeds
InsertMaestroPrivateFeedCredentials -Sources $sources -Creds $creds -Password $Password
$dotnet3Source = $sources.SelectSingleNode("add[@key='dotnet3']")
if ($dotnet3Source -ne $null) {
AddPackageSource -Sources $sources -SourceName "dotnet3-internal" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/_packaging/dotnet3-internal/nuget/v2" -Creds $creds -Username "dn-bot" -Password $Password
AddPackageSource -Sources $sources -SourceName "dotnet3-internal-transport" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/_packaging/dotnet3-internal-transport/nuget/v2" -Creds $creds -Username "dn-bot" -Password $Password
}
$dotnet31Source = $sources.SelectSingleNode("add[@key='dotnet3.1']")
if ($dotnet31Source -ne $null) {
AddPackageSource -Sources $sources -SourceName "dotnet3.1-internal" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/_packaging/dotnet3.1-internal/nuget/v2" -Creds $creds -Username "dn-bot" -Password $Password
AddPackageSource -Sources $sources -SourceName "dotnet3.1-internal-transport" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/_packaging/dotnet3.1-internal-transport/nuget/v2" -Creds $creds -Username "dn-bot" -Password $Password
}
$doc.Save($filename)

149
eng/common/SetupNugetSources.sh

@ -0,0 +1,149 @@
#!/usr/bin/env bash
# This file is a temporary workaround for internal builds to be able to restore from private AzDO feeds.
# This file should be removed as part of this issue: https://github.com/dotnet/arcade/issues/4080
#
# What the script does is iterate over all package sources in the pointed NuGet.config and add a credential entry
# under <packageSourceCredentials> for each Maestro's managed private feed. Two additional credential
# entries are also added for the two private static internal feeds: dotnet3-internal and dotnet3-internal-transport.
#
# This script needs to be called in every job that will restore packages and which the base repo has
# private AzDO feeds in the NuGet.config.
#
# See example YAML call for this script below. Note the use of the variable `$(dn-bot-dnceng-artifact-feeds-rw)`
# from the AzureDevOps-Artifact-Feeds-Pats variable group.
#
# - task: Bash@3
# displayName: Setup Private Feeds Credentials
# inputs:
# filePath: $(Build.SourcesDirectory)/eng/common/SetupNugetSources.sh
# arguments: $(Build.SourcesDirectory)/NuGet.config $Token
# condition: ne(variables['Agent.OS'], 'Windows_NT')
# env:
# Token: $(dn-bot-dnceng-artifact-feeds-rw)
ConfigFile=$1
CredToken=$2
NL='\n'
TB=' '
source="${BASH_SOURCE[0]}"
# resolve $source until the file is no longer a symlink
while [[ -h "$source" ]]; do
scriptroot="$( cd -P "$( dirname "$source" )" && pwd )"
source="$(readlink "$source")"
# if $source was a relative symlink, we need to resolve it relative to the path where the
# symlink file was located
[[ $source != /* ]] && source="$scriptroot/$source"
done
scriptroot="$( cd -P "$( dirname "$source" )" && pwd )"
. "$scriptroot/tools.sh"
if [ ! -f "$ConfigFile" ]; then
Write-PipelineTelemetryError -Category 'Build' "Error: Eng/common/SetupNugetSources.sh returned a non-zero exit code. Couldn't find the NuGet config file: $ConfigFile"
ExitWithExitCode 1
fi
if [ -z "$CredToken" ]; then
Write-PipelineTelemetryError -category 'Build' "Error: Eng/common/SetupNugetSources.sh returned a non-zero exit code. Please supply a valid PAT"
ExitWithExitCode 1
fi
if [[ `uname -s` == "Darwin" ]]; then
NL=$'\\\n'
TB=''
fi
# Ensure there is a <packageSources>...</packageSources> section.
grep -i "<packageSources>" $ConfigFile
if [ "$?" != "0" ]; then
echo "Adding <packageSources>...</packageSources> section."
ConfigNodeHeader="<configuration>"
PackageSourcesTemplate="${TB}<packageSources>${NL}${TB}</packageSources>"
sed -i.bak "s|$ConfigNodeHeader|$ConfigNodeHeader${NL}$PackageSourcesTemplate|" NuGet.config
fi
# Ensure there is a <packageSourceCredentials>...</packageSourceCredentials> section.
grep -i "<packageSourceCredentials>" $ConfigFile
if [ "$?" != "0" ]; then
echo "Adding <packageSourceCredentials>...</packageSourceCredentials> section."
PackageSourcesNodeFooter="</packageSources>"
PackageSourceCredentialsTemplate="${TB}<packageSourceCredentials>${NL}${TB}</packageSourceCredentials>"
sed -i.bak "s|$PackageSourcesNodeFooter|$PackageSourcesNodeFooter${NL}$PackageSourceCredentialsTemplate|" NuGet.config
fi
PackageSources=()
# Ensure dotnet3-internal and dotnet3-internal-transport are in the packageSources if the public dotnet3 feeds are present
grep -i "<add key=\"dotnet3\"" $ConfigFile
if [ "$?" == "0" ]; then
grep -i "<add key=\"dotnet3-internal\">" $ConfigFile
if [ "$?" != "0" ]; then
echo "Adding dotnet3-internal to the packageSources."
PackageSourcesNodeFooter="</packageSources>"
PackageSourceTemplate="${TB}<add key=\"dotnet3-internal\" value=\"https://pkgs.dev.azure.com/dnceng/_packaging/dotnet3-internal/nuget/v2\" />"
sed -i.bak "s|$PackageSourcesNodeFooter|$PackageSourceTemplate${NL}$PackageSourcesNodeFooter|" $ConfigFile
fi
PackageSources+=('dotnet3-internal')
grep -i "<add key=\"dotnet3-internal-transport\"" $ConfigFile
if [ "$?" != "0" ]; then
echo "Adding dotnet3-internal-transport to the packageSources."
PackageSourcesNodeFooter="</packageSources>"
PackageSourceTemplate="${TB}<add key=\"dotnet3-internal-transport\" value=\"https://pkgs.dev.azure.com/dnceng/_packaging/dotnet3-internal-transport/nuget/v2\" />"
sed -i.bak "s|$PackageSourcesNodeFooter|$PackageSourceTemplate${NL}$PackageSourcesNodeFooter|" $ConfigFile
fi
PackageSources+=('dotnet3-internal-transport')
fi
# Ensure dotnet3.1-internal and dotnet3.1-internal-transport are in the packageSources if the public dotnet3.1 feeds are present
grep -i "<add key=\"dotnet3.1\"" $ConfigFile
if [ "$?" == "0" ]; then
grep -i "<add key=\"dotnet3.1-internal\"" $ConfigFile
if [ "$?" != "0" ]; then
echo "Adding dotnet3.1-internal to the packageSources."
PackageSourcesNodeFooter="</packageSources>"
PackageSourceTemplate="${TB}<add key=\"dotnet3.1-internal\" value=\"https://pkgs.dev.azure.com/dnceng/_packaging/dotnet3.1-internal/nuget/v2\" />"
sed -i.bak "s|$PackageSourcesNodeFooter|$PackageSourceTemplate${NL}$PackageSourcesNodeFooter|" $ConfigFile
fi
PackageSources+=('dotnet3.1-internal')
grep -i "<add key=\"dotnet3.1-internal-transport\">" $ConfigFile
if [ "$?" != "0" ]; then
echo "Adding dotnet3.1-internal-transport to the packageSources."
PackageSourcesNodeFooter="</packageSources>"
PackageSourceTemplate="${TB}<add key=\"dotnet3.1-internal-transport\" value=\"https://pkgs.dev.azure.com/dnceng/_packaging/dotnet3.1-internal-transport/nuget/v2\" />"
sed -i.bak "s|$PackageSourcesNodeFooter|$PackageSourceTemplate${NL}$PackageSourcesNodeFooter|" $ConfigFile
fi
PackageSources+=('dotnet3.1-internal-transport')
fi
# I want things split line by line
PrevIFS=$IFS
IFS=$'\n'
PackageSources+="$IFS"
PackageSources+=$(grep -oh '"darc-int-[^"]*"' $ConfigFile | tr -d '"')
IFS=$PrevIFS
for FeedName in ${PackageSources[@]} ; do
# Check if there is no existing credential for this FeedName
grep -i "<$FeedName>" $ConfigFile
if [ "$?" != "0" ]; then
echo "Adding credentials for $FeedName."
PackageSourceCredentialsNodeFooter="</packageSourceCredentials>"
NewCredential="${TB}${TB}<$FeedName>${NL}<add key=\"Username\" value=\"dn-bot\" />${NL}<add key=\"ClearTextPassword\" value=\"$CredToken\" />${NL}</$FeedName>"
sed -i.bak "s|$PackageSourceCredentialsNodeFooter|$NewCredential${NL}$PackageSourceCredentialsNodeFooter|" $ConfigFile
fi
done

83
eng/common/SigningValidation.proj

@ -1,83 +0,0 @@
<!-- Licensed to the .NET Foundation under one or more agreements. The .NET Foundation licenses this file to you under the MIT license. See the LICENSE file in the project root for more information. -->
<Project Sdk="Microsoft.NET.Sdk">
<!--
This MSBuild file is intended to be used as the body of the default
publishing release pipeline. The release pipeline will use this file
to invoke the SignCheck tool to validate that packages about to
be published are correctly signed.
Parameters:
- PackageBasePath : Directory containing all files that need to be validated.
- SignCheckVersion : Version of SignCheck package to be used.
- SignValidationExclusionList : ItemGroup containing exclusion list to be forwarded to SignCheck.
- EnableJarSigningCheck : Whether .jar files should be validated.
- EnableStrongNameCheck : Whether strong name check should be performed.
-->
<PropertyGroup>
<TargetFramework>netcoreapp2.1</TargetFramework>
</PropertyGroup>
<!--
From 'Signing.props' we import $(SignValidationExclusionList)
-->
<Import Project="$(MSBuildThisFileDirectory)Signing.props" Condition="Exists('$(MSBuildThisFileDirectory)Signing.props')" />
<Target Name="ValidateSigning">
<PropertyGroup>
<SignCheckToolPath>$(NuGetPackageRoot)Microsoft.DotNet.SignCheck\$(SignCheckVersion)\tools\Microsoft.DotNet.SignCheck.exe</SignCheckToolPath>
<SignCheckInputDir>$(PackageBasePath)</SignCheckInputDir>
<SignCheckLog>signcheck.log</SignCheckLog>
<SignCheckErrorLog>signcheck.errors.log</SignCheckErrorLog>
<SignCheckExclusionsFile>signcheck.exclusions.txt</SignCheckExclusionsFile>
</PropertyGroup>
<ItemGroup>
<!--
Documentation for these arguments is available here:
https://github.com/dotnet/arcade/tree/master/src/SignCheck
-->
<SignCheckArgs Include="--recursive" />
<SignCheckArgs Include="--traverse-subfolders" />
<SignCheckArgs Include="--file-status AllFiles" />
<SignCheckArgs Include="--log-file $(SignCheckLog)" />
<SignCheckArgs Include="--error-log-file $(SignCheckErrorLog)" />
<SignCheckArgs Include="--input-files $(SignCheckInputDir)" />
<SignCheckArgs Include="--exclusions-file $(SignCheckExclusionsFile)" Condition="'@(SignValidationExclusionList)' != ''" />
<SignCheckArgs Include="--verify-jar" Condition="'$(EnableJarSigningCheck)' == 'true'" />
<SignCheckArgs Include="--verify-strongname" Condition="'$(EnableStrongNameCheck)' == 'true'" />
</ItemGroup>
<WriteLinesToFile
File="$(SignCheckExclusionsFile)"
Lines="@(SignValidationExclusionList)"
Condition="'@(SignValidationExclusionList)' != ''"
Overwrite="true"
Encoding="Unicode"/>
<!--
IgnoreExitCode='true' because the tool doesn't return '0' on success.
-->
<Exec
Command="&quot;$(SignCheckToolPath)&quot; @(SignCheckArgs, ' ')"
IgnoreExitCode='true'
ConsoleToMsBuild="false"
StandardErrorImportance="high" />
<Error
Text="Signing validation failed. Check $(SignCheckErrorLog) for more information."
Condition="Exists($(SignCheckErrorLog)) and '$([System.IO.File]::ReadAllText($(SignCheckErrorLog)))' != ''" />
<Message
Text="##vso[artifact.upload containerfolder=LogFiles;artifactname=LogFiles]{SignCheckErrorLog}"
Condition="Exists($(SignCheckErrorLog)) and '$([System.IO.File]::ReadAllText($(SignCheckErrorLog)))' != ''" />
</Target>
<ItemGroup>
<PackageReference Include="Microsoft.DotNet.SignCheck" Version="$(SignCheckVersion)" />
</ItemGroup>
</Project>

184
eng/common/SourceLinkValidation.ps1

@ -1,184 +0,0 @@
param(
[Parameter(Mandatory=$true)][string] $InputPath, # Full path to directory where Symbols.NuGet packages to be checked are stored
[Parameter(Mandatory=$true)][string] $ExtractPath, # Full path to directory where the packages will be extracted during validation
[Parameter(Mandatory=$true)][string] $SourceLinkToolPath, # Full path to directory where dotnet SourceLink CLI was installed
[Parameter(Mandatory=$true)][string] $GHRepoName, # GitHub name of the repo including the Org. E.g., dotnet/arcade
[Parameter(Mandatory=$true)][string] $GHCommit # GitHub commit SHA used to build the packages
)
# Cache/HashMap (File -> Exist flag) used to consult whether a file exist
# in the repository at a specific commit point. This is populated by inserting
# all files present in the repo at a specific commit point.
$global:RepoFiles = @{}
$ValidatePackage = {
param(
[string] $PackagePath # Full path to a Symbols.NuGet package
)
# Ensure input file exist
if (!(Test-Path $PackagePath)) {
throw "Input file does not exist: $PackagePath"
}
# Extensions for which we'll look for SourceLink information
# For now we'll only care about Portable & Embedded PDBs
$RelevantExtensions = @(".dll", ".exe", ".pdb")
Write-Host -NoNewLine "Validating" ([System.IO.Path]::GetFileName($PackagePath)) "... "
$PackageId = [System.IO.Path]::GetFileNameWithoutExtension($PackagePath)
$ExtractPath = Join-Path -Path $using:ExtractPath -ChildPath $PackageId
$FailedFiles = 0
Add-Type -AssemblyName System.IO.Compression.FileSystem
[System.IO.Directory]::CreateDirectory($ExtractPath);
$zip = [System.IO.Compression.ZipFile]::OpenRead($PackagePath)
$zip.Entries |
Where-Object {$RelevantExtensions -contains [System.IO.Path]::GetExtension($_.Name)} |
ForEach-Object {
$FileName = $_.FullName
$Extension = [System.IO.Path]::GetExtension($_.Name)
$FakeName = -Join((New-Guid), $Extension)
$TargetFile = Join-Path -Path $ExtractPath -ChildPath $FakeName
# We ignore resource DLLs
if ($FileName.EndsWith(".resources.dll")) {
return
}
[System.IO.Compression.ZipFileExtensions]::ExtractToFile($_, $TargetFile, $true)
$ValidateFile = {
param(
[string] $FullPath, # Full path to the module that has to be checked
[string] $RealPath,
[ref] $FailedFiles
)
# Makes easier to reference `sourcelink cli`
Push-Location $using:SourceLinkToolPath
$SourceLinkInfos = .\sourcelink.exe print-urls $FullPath | Out-String
if ($LASTEXITCODE -eq 0 -and -not ([string]::IsNullOrEmpty($SourceLinkInfos))) {
$NumFailedLinks = 0
# We only care about Http addresses
$Matches = (Select-String '(http[s]?)(:\/\/)([^\s,]+)' -Input $SourceLinkInfos -AllMatches).Matches
if ($Matches.Count -ne 0) {
$Matches.Value |
ForEach-Object {
$Link = $_
$CommitUrl = -Join("https://raw.githubusercontent.com/", $using:GHRepoName, "/", $using:GHCommit, "/")
$FilePath = $Link.Replace($CommitUrl, "")
$Status = 200
$Cache = $using:RepoFiles
if ( !($Cache.ContainsKey($FilePath)) ) {
try {
$Uri = $Link -as [System.URI]
# Only GitHub links are valid
if ($Uri.AbsoluteURI -ne $null -and $Uri.Host -match "github") {
$Status = (Invoke-WebRequest -Uri $Link -UseBasicParsing -Method HEAD -TimeoutSec 5).StatusCode
}
else {
$Status = 0
}
}
catch {
$Status = 0
}
}
if ($Status -ne 200) {
if ($NumFailedLinks -eq 0) {
if ($FailedFiles.Value -eq 0) {
Write-Host
}
Write-Host "`tFile $RealPath has broken links:"
}
Write-Host "`t`tFailed to retrieve $Link"
$NumFailedLinks++
}
}
}
if ($NumFailedLinks -ne 0) {
$FailedFiles.value++
$global:LASTEXITCODE = 1
}
}
Pop-Location
}
&$ValidateFile $TargetFile $FileName ([ref]$FailedFiles)
}
$zip.Dispose()
if ($FailedFiles -eq 0) {
Write-Host "Passed."
}
}
function ValidateSourceLinkLinks {
if (!($GHRepoName -Match "^[^\s\/]+/[^\s\/]+$")) {
Write-Host "GHRepoName should be in the format <org>/<repo>"
$global:LASTEXITCODE = 1
return
}
if (!($GHCommit -Match "^[0-9a-fA-F]{40}$")) {
Write-Host "GHCommit should be a 40 chars hexadecimal string"
$global:LASTEXITCODE = 1
return
}
$RepoTreeURL = -Join("https://api.github.com/repos/", $GHRepoName, "/git/trees/", $GHCommit, "?recursive=1")
$CodeExtensions = @(".cs", ".vb", ".fs", ".fsi", ".fsx", ".fsscript")
try {
# Retrieve the list of files in the repo at that particular commit point and store them in the RepoFiles hash
$Data = Invoke-WebRequest $RepoTreeURL | ConvertFrom-Json | Select-Object -ExpandProperty tree
foreach ($file in $Data) {
$Extension = [System.IO.Path]::GetExtension($file.path)
if ($CodeExtensions.Contains($Extension)) {
$RepoFiles[$file.path] = 1
}
}
}
catch {
Write-Host "Problems downloading the list of files from the repo. Url used: $RepoTreeURL"
$global:LASTEXITCODE = 1
return
}
if (Test-Path $ExtractPath) {
Remove-Item $ExtractPath -Force -Recurse -ErrorAction SilentlyContinue
}
# Process each NuGet package in parallel
$Jobs = @()
Get-ChildItem "$InputPath\*.symbols.nupkg" |
ForEach-Object {
$Jobs += Start-Job -ScriptBlock $ValidatePackage -ArgumentList $_.FullName
}
foreach ($Job in $Jobs) {
Wait-Job -Id $Job.Id | Receive-Job
}
}
Measure-Command { ValidateSourceLinkLinks }

102
eng/common/build.ps1

@ -18,6 +18,7 @@ Param(
[switch] $sign, [switch] $sign,
[switch] $pack, [switch] $pack,
[switch] $publish, [switch] $publish,
[switch] $clean,
[switch][Alias('bl')]$binaryLog, [switch][Alias('bl')]$binaryLog,
[switch] $ci, [switch] $ci,
[switch] $prepareMachine, [switch] $prepareMachine,
@ -25,49 +26,55 @@ Param(
[Parameter(ValueFromRemainingArguments=$true)][String[]]$properties [Parameter(ValueFromRemainingArguments=$true)][String[]]$properties
) )
. $PSScriptRoot\tools.ps1 # Unset 'Platform' environment variable to avoid unwanted collision in InstallDotNetCore.targets file
# some computer has this env var defined (e.g. Some HP)
if($env:Platform) {
$env:Platform=""
}
function Print-Usage() { function Print-Usage() {
Write-Host "Common settings:" Write-Host "Common settings:"
Write-Host " -configuration <value> Build configuration: 'Debug' or 'Release' (short: -c)" Write-Host " -configuration <value> Build configuration: 'Debug' or 'Release' (short: -c)"
Write-Host " -platform <value> Platform configuration: 'x86', 'x64' or any valid Platform value to pass to msbuild" Write-Host " -platform <value> Platform configuration: 'x86', 'x64' or any valid Platform value to pass to msbuild"
Write-Host " -verbosity <value> Msbuild verbosity: q[uiet], m[inimal], n[ormal], d[etailed], and diag[nostic] (short: -v)" Write-Host " -verbosity <value> Msbuild verbosity: q[uiet], m[inimal], n[ormal], d[etailed], and diag[nostic] (short: -v)"
Write-Host " -binaryLog Output binary log (short: -bl)" Write-Host " -binaryLog Output binary log (short: -bl)"
Write-Host " -help Print help and exit" Write-Host " -help Print help and exit"
Write-Host "" Write-Host ""
Write-Host "Actions:" Write-Host "Actions:"
Write-Host " -restore Restore dependencies (short: -r)" Write-Host " -restore Restore dependencies (short: -r)"
Write-Host " -build Build solution (short: -b)" Write-Host " -build Build solution (short: -b)"
Write-Host " -rebuild Rebuild solution" Write-Host " -rebuild Rebuild solution"
Write-Host " -deploy Deploy built VSIXes" Write-Host " -deploy Deploy built VSIXes"
Write-Host " -deployDeps Deploy dependencies (e.g. VSIXes for integration tests)" Write-Host " -deployDeps Deploy dependencies (e.g. VSIXes for integration tests)"
Write-Host " -test Run all unit tests in the solution (short: -t)" Write-Host " -test Run all unit tests in the solution (short: -t)"
Write-Host " -integrationTest Run all integration tests in the solution" Write-Host " -integrationTest Run all integration tests in the solution"
Write-Host " -performanceTest Run all performance tests in the solution" Write-Host " -performanceTest Run all performance tests in the solution"
Write-Host " -pack Package build outputs into NuGet packages and Willow components" Write-Host " -pack Package build outputs into NuGet packages and Willow components"
Write-Host " -sign Sign build outputs" Write-Host " -sign Sign build outputs"
Write-Host " -publish Publish artifacts (e.g. symbols)" Write-Host " -publish Publish artifacts (e.g. symbols)"
Write-Host "" Write-Host " -clean Clean the solution"
Write-Host ""
Write-Host "Advanced settings:"
Write-Host " -projects <value> Semi-colon delimited list of sln/proj's to build. Globbing is supported (*.sln)" Write-Host "Advanced settings:"
Write-Host " -ci Set when running on CI server" Write-Host " -projects <value> Semi-colon delimited list of sln/proj's to build. Globbing is supported (*.sln)"
Write-Host " -prepareMachine Prepare machine for CI run, clean up processes after build" Write-Host " -ci Set when running on CI server"
Write-Host " -warnAsError <value> Sets warnaserror msbuild parameter ('true' or 'false')" Write-Host " -prepareMachine Prepare machine for CI run, clean up processes after build"
Write-Host " -msbuildEngine <value> Msbuild engine to use to run build ('dotnet', 'vs', or unspecified)." Write-Host " -warnAsError <value> Sets warnaserror msbuild parameter ('true' or 'false')"
Write-Host "" Write-Host " -msbuildEngine <value> Msbuild engine to use to run build ('dotnet', 'vs', or unspecified)."
Write-Host ""
Write-Host "Command line arguments not listed above are passed thru to msbuild."
Write-Host "The above arguments can be shortened as much as to be unambiguous (e.g. -co for configuration, -t for test, etc.)." Write-Host "Command line arguments not listed above are passed thru to msbuild."
Write-Host "The above arguments can be shortened as much as to be unambiguous (e.g. -co for configuration, -t for test, etc.)."
} }
. $PSScriptRoot\tools.ps1
function InitializeCustomToolset { function InitializeCustomToolset {
if (-not $restore) { if (-not $restore) {
return return
} }
$script = Join-Path $EngRoot "restore-toolset.ps1" $script = Join-Path $EngRoot 'restore-toolset.ps1'
if (Test-Path $script) { if (Test-Path $script) {
. $script . $script
@ -78,8 +85,8 @@ function Build {
$toolsetBuildProj = InitializeToolset $toolsetBuildProj = InitializeToolset
InitializeCustomToolset InitializeCustomToolset
$bl = if ($binaryLog) { "/bl:" + (Join-Path $LogDir "Build.binlog") } else { "" } $bl = if ($binaryLog) { '/bl:' + (Join-Path $LogDir 'Build.binlog') } else { '' }
$platformArg = if ($platform) { "/p:Platform=$platform" } else { "" } $platformArg = if ($platform) { "/p:Platform=$platform" } else { '' }
if ($projects) { if ($projects) {
# Re-assign properties to a new variable because PowerShell doesn't let us append properties directly for unclear reasons. # Re-assign properties to a new variable because PowerShell doesn't let us append properties directly for unclear reasons.
@ -113,7 +120,15 @@ function Build {
} }
try { try {
if ($help -or (($null -ne $properties) -and ($properties.Contains("/help") -or $properties.Contains("/?")))) { if ($clean) {
if (Test-Path $ArtifactsDir) {
Remove-Item -Recurse -Force $ArtifactsDir
Write-Host 'Artifacts directory deleted.'
}
exit 0
}
if ($help -or (($null -ne $properties) -and ($properties.Contains('/help') -or $properties.Contains('/?')))) {
Print-Usage Print-Usage
exit 0 exit 0
} }
@ -123,14 +138,7 @@ try {
$nodeReuse = $false $nodeReuse = $false
} }
# Import custom tools configuration, if present in the repo. if ($restore) {
# Note: Import in global scope so that the script set top-level variables without qualification.
$configureToolsetScript = Join-Path $EngRoot "configure-toolset.ps1"
if (Test-Path $configureToolsetScript) {
. $configureToolsetScript
}
if (($restore) -and ($null -eq $env:DisableNativeToolsetInstalls)) {
InitializeNativeTools InitializeNativeTools
} }
@ -138,7 +146,7 @@ try {
} }
catch { catch {
Write-Host $_.ScriptStackTrace Write-Host $_.ScriptStackTrace
Write-PipelineTelemetryError -Category "InitializeToolset" -Message $_ Write-PipelineTelemetryError -Category 'InitializeToolset' -Message $_
ExitWithExitCode 1 ExitWithExitCode 1
} }

24
eng/common/build.sh

@ -26,6 +26,7 @@ usage()
echo " --pack Package build outputs into NuGet packages and Willow components" echo " --pack Package build outputs into NuGet packages and Willow components"
echo " --sign Sign build outputs" echo " --sign Sign build outputs"
echo " --publish Publish artifacts (e.g. symbols)" echo " --publish Publish artifacts (e.g. symbols)"
echo " --clean Clean the solution"
echo "" echo ""
echo "Advanced settings:" echo "Advanced settings:"
@ -62,6 +63,7 @@ publish=false
sign=false sign=false
public=false public=false
ci=false ci=false
clean=false
warn_as_error=true warn_as_error=true
node_reuse=true node_reuse=true
@ -82,6 +84,9 @@ while [[ $# > 0 ]]; do
usage usage
exit 0 exit 0
;; ;;
-clean)
clean=true
;;
-configuration|-c) -configuration|-c)
configuration=$2 configuration=$2
shift shift
@ -196,20 +201,15 @@ function Build {
ExitWithExitCode 0 ExitWithExitCode 0
} }
# Import custom tools configuration, if present in the repo. if [[ "$clean" == true ]]; then
configure_toolset_script="$eng_root/configure-toolset.sh" if [ -d "$artifacts_dir" ]; then
if [[ -a "$configure_toolset_script" ]]; then rm -rf $artifacts_dir
. "$configure_toolset_script" echo "Artifacts directory deleted."
fi fi
exit 0
# TODO: https://github.com/dotnet/arcade/issues/1468
# Temporary workaround to avoid breaking change.
# Remove once repos are updated.
if [[ -n "${useInstalledDotNetCli:-}" ]]; then
use_installed_dotnet_cli="$useInstalledDotNetCli"
fi fi
if [[ "$restore" == true && -z ${DisableNativeToolsetInstalls:-} ]]; then if [[ "$restore" == true ]]; then
InitializeNativeTools InitializeNativeTools
fi fi

41
eng/common/cross/android/arm/toolchain.cmake

@ -1,41 +0,0 @@
set(CROSS_NDK_TOOLCHAIN $ENV{ROOTFS_DIR}/../)
set(CROSS_ROOTFS ${CROSS_NDK_TOOLCHAIN}/sysroot)
set(CLR_CMAKE_PLATFORM_ANDROID "Android")
set(CMAKE_SYSTEM_NAME Linux)
set(CMAKE_SYSTEM_VERSION 1)
set(CMAKE_SYSTEM_PROCESSOR arm)
## Specify the toolchain
set(TOOLCHAIN "arm-linux-androideabi")
set(CMAKE_PREFIX_PATH ${CROSS_NDK_TOOLCHAIN})
set(TOOLCHAIN_PREFIX ${TOOLCHAIN}-)
find_program(CMAKE_C_COMPILER ${TOOLCHAIN_PREFIX}clang)
find_program(CMAKE_CXX_COMPILER ${TOOLCHAIN_PREFIX}clang++)
find_program(CMAKE_ASM_COMPILER ${TOOLCHAIN_PREFIX}clang)
find_program(CMAKE_AR ${TOOLCHAIN_PREFIX}ar)
find_program(CMAKE_LD ${TOOLCHAIN_PREFIX}ar)
find_program(CMAKE_OBJCOPY ${TOOLCHAIN_PREFIX}objcopy)
find_program(CMAKE_OBJDUMP ${TOOLCHAIN_PREFIX}objdump)
add_compile_options(--sysroot=${CROSS_ROOTFS})
add_compile_options(-fPIE)
add_compile_options(-mfloat-abi=soft)
include_directories(SYSTEM ${CROSS_NDK_TOOLCHAIN}/include/c++/4.9.x/)
include_directories(SYSTEM ${CROSS_NDK_TOOLCHAIN}/include/c++/4.9.x/arm-linux-androideabi/)
set(CROSS_LINK_FLAGS "${CROSS_LINK_FLAGS} -B ${CROSS_ROOTFS}/usr/lib/gcc/${TOOLCHAIN}")
set(CROSS_LINK_FLAGS "${CROSS_LINK_FLAGS} -L${CROSS_ROOTFS}/lib/${TOOLCHAIN}")
set(CROSS_LINK_FLAGS "${CROSS_LINK_FLAGS} --sysroot=${CROSS_ROOTFS}")
set(CROSS_LINK_FLAGS "${CROSS_LINK_FLAGS} -fPIE -pie")
set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} ${CROSS_LINK_FLAGS}" CACHE STRING "" FORCE)
set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} ${CROSS_LINK_FLAGS}" CACHE STRING "" FORCE)
set(CMAKE_MODULE_LINKER_FLAGS "${CMAKE_MODULE_LINKER_FLAGS} ${CROSS_LINK_FLAGS}" CACHE STRING "" FORCE)
set(CMAKE_FIND_ROOT_PATH "${CROSS_ROOTFS}")
set(CMAKE_FIND_ROOT_PATH_MODE_PROGRAM NEVER)
set(CMAKE_FIND_ROOT_PATH_MODE_LIBRARY ONLY)
set(CMAKE_FIND_ROOT_PATH_MODE_INCLUDE ONLY)
set(CMAKE_FIND_ROOT_PATH_MODE_PACKAGE ONLY)

42
eng/common/cross/android/arm64/toolchain.cmake

@ -1,42 +0,0 @@
set(CROSS_NDK_TOOLCHAIN $ENV{ROOTFS_DIR}/../)
set(CROSS_ROOTFS ${CROSS_NDK_TOOLCHAIN}/sysroot)
set(CLR_CMAKE_PLATFORM_ANDROID "Android")
set(CMAKE_SYSTEM_NAME Linux)
set(CMAKE_SYSTEM_VERSION 1)
set(CMAKE_SYSTEM_PROCESSOR aarch64)
## Specify the toolchain
set(TOOLCHAIN "aarch64-linux-android")
set(CMAKE_PREFIX_PATH ${CROSS_NDK_TOOLCHAIN})
set(TOOLCHAIN_PREFIX ${TOOLCHAIN}-)
find_program(CMAKE_C_COMPILER ${TOOLCHAIN_PREFIX}clang)
find_program(CMAKE_CXX_COMPILER ${TOOLCHAIN_PREFIX}clang++)
find_program(CMAKE_ASM_COMPILER ${TOOLCHAIN_PREFIX}clang)
find_program(CMAKE_AR ${TOOLCHAIN_PREFIX}ar)
find_program(CMAKE_LD ${TOOLCHAIN_PREFIX}ar)
find_program(CMAKE_OBJCOPY ${TOOLCHAIN_PREFIX}objcopy)
find_program(CMAKE_OBJDUMP ${TOOLCHAIN_PREFIX}objdump)
add_compile_options(--sysroot=${CROSS_ROOTFS})
add_compile_options(-fPIE)
## Needed for Android or bionic specific conditionals
add_compile_options(-D__ANDROID__)
add_compile_options(-D__BIONIC__)
set(CROSS_LINK_FLAGS "${CROSS_LINK_FLAGS} -B ${CROSS_ROOTFS}/usr/lib/gcc/${TOOLCHAIN}")
set(CROSS_LINK_FLAGS "${CROSS_LINK_FLAGS} -L${CROSS_ROOTFS}/lib/${TOOLCHAIN}")
set(CROSS_LINK_FLAGS "${CROSS_LINK_FLAGS} --sysroot=${CROSS_ROOTFS}")
set(CROSS_LINK_FLAGS "${CROSS_LINK_FLAGS} -fPIE -pie")
set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} ${CROSS_LINK_FLAGS}" CACHE STRING "" FORCE)
set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} ${CROSS_LINK_FLAGS}" CACHE STRING "" FORCE)
set(CMAKE_MODULE_LINKER_FLAGS "${CMAKE_MODULE_LINKER_FLAGS} ${CROSS_LINK_FLAGS}" CACHE STRING "" FORCE)
set(CMAKE_FIND_ROOT_PATH "${CROSS_ROOTFS}")
set(CMAKE_FIND_ROOT_PATH_MODE_PROGRAM NEVER)
set(CMAKE_FIND_ROOT_PATH_MODE_LIBRARY ONLY)
set(CMAKE_FIND_ROOT_PATH_MODE_INCLUDE ONLY)
set(CMAKE_FIND_ROOT_PATH_MODE_PACKAGE ONLY)

106
eng/common/cross/build-android-rootfs.sh

@ -1,6 +1,6 @@
#!/usr/bin/env bash #!/usr/bin/env bash
set -e set -e
__NDK_Version=r14 __NDK_Version=r21
usage() usage()
{ {
@ -16,11 +16,11 @@ usage()
echo. echo.
echo "By default, the NDK will be downloaded into the cross/android-rootfs/android-ndk-$__NDK_Version directory. If you already have an NDK installation," echo "By default, the NDK will be downloaded into the cross/android-rootfs/android-ndk-$__NDK_Version directory. If you already have an NDK installation,"
echo "you can set the NDK_DIR environment variable to have this script use that installation of the NDK." echo "you can set the NDK_DIR environment variable to have this script use that installation of the NDK."
echo "By default, this script will generate a file, android_platform, in the root of the ROOTFS_DIR directory that contains the RID for the supported and tested Android build: android.21-arm64. This file is to replace '/etc/os-release', which is not available for Android." echo "By default, this script will generate a file, android_platform, in the root of the ROOTFS_DIR directory that contains the RID for the supported and tested Android build: android.28-arm64. This file is to replace '/etc/os-release', which is not available for Android."
exit 1 exit 1
} }
__ApiLevel=21 # The minimum platform for arm64 is API level 21 __ApiLevel=28 # The minimum platform for arm64 is API level 21 but the minimum version that support glob(3) is 28. See $ANDROID_NDK/toolchains/llvm/prebuilt/linux-x86_64/sysroot/usr/include/glob.h
__BuildArch=arm64 __BuildArch=arm64
__AndroidArch=aarch64 __AndroidArch=aarch64
__AndroidToolchain=aarch64-linux-android __AndroidToolchain=aarch64-linux-android
@ -53,13 +53,20 @@ for i in "$@"
done done
# Obtain the location of the bash script to figure out where the root of the repo is. # Obtain the location of the bash script to figure out where the root of the repo is.
__CrossDir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" __ScriptBaseDir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
__Android_Cross_Dir="$__CrossDir/android-rootfs" __CrossDir="$__ScriptBaseDir/../../../.tools/android-rootfs"
__NDK_Dir="$__Android_Cross_Dir/android-ndk-$__NDK_Version"
__libunwind_Dir="$__Android_Cross_Dir/libunwind" if [[ ! -f "$__CrossDir" ]]; then
__lldb_Dir="$__Android_Cross_Dir/lldb" mkdir -p "$__CrossDir"
__ToolchainDir="$__Android_Cross_Dir/toolchain/$__BuildArch" fi
# Resolve absolute path to avoid `../` in build logs
__CrossDir="$( cd "$__CrossDir" && pwd )"
__NDK_Dir="$__CrossDir/android-ndk-$__NDK_Version"
__lldb_Dir="$__CrossDir/lldb"
__ToolchainDir="$__CrossDir/android-ndk-$__NDK_Version"
if [[ -n "$TOOLCHAIN_DIR" ]]; then if [[ -n "$TOOLCHAIN_DIR" ]]; then
__ToolchainDir=$TOOLCHAIN_DIR __ToolchainDir=$TOOLCHAIN_DIR
@ -78,60 +85,47 @@ echo "Target Toolchain location: $__ToolchainDir"
if [ ! -d $__NDK_Dir ]; then if [ ! -d $__NDK_Dir ]; then
echo Downloading the NDK into $__NDK_Dir echo Downloading the NDK into $__NDK_Dir
mkdir -p $__NDK_Dir mkdir -p $__NDK_Dir
wget -nv -nc --show-progress https://dl.google.com/android/repository/android-ndk-$__NDK_Version-linux-x86_64.zip -O $__Android_Cross_Dir/android-ndk-$__NDK_Version-linux-x86_64.zip wget -q --progress=bar:force:noscroll --show-progress https://dl.google.com/android/repository/android-ndk-$__NDK_Version-linux-x86_64.zip -O $__CrossDir/android-ndk-$__NDK_Version-linux-x86_64.zip
unzip -q $__Android_Cross_Dir/android-ndk-$__NDK_Version-linux-x86_64.zip -d $__Android_Cross_Dir unzip -q $__CrossDir/android-ndk-$__NDK_Version-linux-x86_64.zip -d $__CrossDir
fi fi
if [ ! -d $__lldb_Dir ]; then if [ ! -d $__lldb_Dir ]; then
mkdir -p $__lldb_Dir mkdir -p $__lldb_Dir
echo Downloading LLDB into $__lldb_Dir echo Downloading LLDB into $__lldb_Dir
wget -nv -nc --show-progress https://dl.google.com/android/repository/lldb-2.3.3614996-linux-x86_64.zip -O $__Android_Cross_Dir/lldb-2.3.3614996-linux-x86_64.zip wget -q --progress=bar:force:noscroll --show-progress https://dl.google.com/android/repository/lldb-2.3.3614996-linux-x86_64.zip -O $__CrossDir/lldb-2.3.3614996-linux-x86_64.zip
unzip -q $__Android_Cross_Dir/lldb-2.3.3614996-linux-x86_64.zip -d $__lldb_Dir unzip -q $__CrossDir/lldb-2.3.3614996-linux-x86_64.zip -d $__lldb_Dir
fi fi
# Create the RootFS for both arm64 as well as aarch echo "Download dependencies..."
rm -rf $__Android_Cross_Dir/toolchain __TmpDir=$__CrossDir/tmp/$__BuildArch/
mkdir -p "$__TmpDir"
echo Generating the $__BuildArch toolchain
$__NDK_Dir/build/tools/make_standalone_toolchain.py --arch $__BuildArch --api $__ApiLevel --install-dir $__ToolchainDir
# Install the required packages into the toolchain
# TODO: Add logic to get latest pkg version instead of specific version number
rm -rf $__Android_Cross_Dir/deb/
rm -rf $__Android_Cross_Dir/tmp
mkdir -p $__Android_Cross_Dir/deb/
mkdir -p $__Android_Cross_Dir/tmp/$arch/
wget -nv -nc http://termux.net/dists/stable/main/binary-$__AndroidArch/libicu_60.2_$__AndroidArch.deb -O $__Android_Cross_Dir/deb/libicu_60.2_$__AndroidArch.deb
wget -nv -nc http://termux.net/dists/stable/main/binary-$__AndroidArch/libicu-dev_60.2_$__AndroidArch.deb -O $__Android_Cross_Dir/deb/libicu-dev_60.2_$__AndroidArch.deb
wget -nv -nc http://termux.net/dists/stable/main/binary-$__AndroidArch/libandroid-glob-dev_0.4_$__AndroidArch.deb -O $__Android_Cross_Dir/deb/libandroid-glob-dev_0.4_$__AndroidArch.deb
wget -nv -nc http://termux.net/dists/stable/main/binary-$__AndroidArch/libandroid-glob_0.4_$__AndroidArch.deb -O $__Android_Cross_Dir/deb/libandroid-glob_0.4_$__AndroidArch.deb
wget -nv -nc http://termux.net/dists/stable/main/binary-$__AndroidArch/libandroid-support-dev_22_$__AndroidArch.deb -O $__Android_Cross_Dir/deb/libandroid-support-dev_22_$__AndroidArch.deb
wget -nv -nc http://termux.net/dists/stable/main/binary-$__AndroidArch/libandroid-support_22_$__AndroidArch.deb -O $__Android_Cross_Dir/deb/libandroid-support_22_$__AndroidArch.deb
wget -nv -nc http://termux.net/dists/stable/main/binary-$__AndroidArch/liblzma-dev_5.2.3_$__AndroidArch.deb -O $__Android_Cross_Dir/deb/liblzma-dev_5.2.3_$__AndroidArch.deb
wget -nv -nc http://termux.net/dists/stable/main/binary-$__AndroidArch/liblzma_5.2.3_$__AndroidArch.deb -O $__Android_Cross_Dir/deb/liblzma_5.2.3_$__AndroidArch.deb
wget -nv -nc http://termux.net/dists/stable/main/binary-$__AndroidArch/libunwind-dev_1.2.20170304_$__AndroidArch.deb -O $__Android_Cross_Dir/deb/libunwind-dev_1.2.20170304_$__AndroidArch.deb
wget -nv -nc http://termux.net/dists/stable/main/binary-$__AndroidArch/libunwind_1.2.20170304_$__AndroidArch.deb -O $__Android_Cross_Dir/deb/libunwind_1.2.20170304_$__AndroidArch.deb
echo Unpacking Termux packages
dpkg -x $__Android_Cross_Dir/deb/libicu_60.2_$__AndroidArch.deb $__Android_Cross_Dir/tmp/$__AndroidArch/
dpkg -x $__Android_Cross_Dir/deb/libicu-dev_60.2_$__AndroidArch.deb $__Android_Cross_Dir/tmp/$__AndroidArch/
dpkg -x $__Android_Cross_Dir/deb/libandroid-glob-dev_0.4_$__AndroidArch.deb $__Android_Cross_Dir/tmp/$__AndroidArch/
dpkg -x $__Android_Cross_Dir/deb/libandroid-glob_0.4_$__AndroidArch.deb $__Android_Cross_Dir/tmp/$__AndroidArch/
dpkg -x $__Android_Cross_Dir/deb/libandroid-support-dev_22_$__AndroidArch.deb $__Android_Cross_Dir/tmp/$__AndroidArch/
dpkg -x $__Android_Cross_Dir/deb/libandroid-support_22_$__AndroidArch.deb $__Android_Cross_Dir/tmp/$__AndroidArch/
dpkg -x $__Android_Cross_Dir/deb/liblzma-dev_5.2.3_$__AndroidArch.deb $__Android_Cross_Dir/tmp/$__AndroidArch/
dpkg -x $__Android_Cross_Dir/deb/liblzma_5.2.3_$__AndroidArch.deb $__Android_Cross_Dir/tmp/$__AndroidArch/
dpkg -x $__Android_Cross_Dir/deb/libunwind-dev_1.2.20170304_$__AndroidArch.deb $__Android_Cross_Dir/tmp/$__AndroidArch/
dpkg -x $__Android_Cross_Dir/deb/libunwind_1.2.20170304_$__AndroidArch.deb $__Android_Cross_Dir/tmp/$__AndroidArch/
cp -R $__Android_Cross_Dir/tmp/$__AndroidArch/data/data/com.termux/files/usr/* $__ToolchainDir/sysroot/usr/
# Generate platform file for build.sh script to assign to __DistroRid # combined dependencies for coreclr, installer and libraries
echo "Generating platform file..." __AndroidPackages="libicu"
__AndroidPackages+=" libandroid-glob"
__AndroidPackages+=" liblzma"
__AndroidPackages+=" krb5"
__AndroidPackages+=" openssl"
echo "RID=android.21-arm64" > $__ToolchainDir/sysroot/android_platform for path in $(wget -qO- http://termux.net/dists/stable/main/binary-$__AndroidArch/Packages |\
echo Now run: grep -A15 "Package: \(${__AndroidPackages// /\\|}\)" | grep -v "static\|tool" | grep Filename); do
echo CONFIG_DIR=\`realpath cross/android/$__BuildArch\` ROOTFS_DIR=\`realpath $__ToolchainDir/sysroot\` ./build.sh cross $__BuildArch skipgenerateversion skipnuget cmakeargs -DENABLE_LLDBPLUGIN=0
if [[ "$path" != "Filename:" ]]; then
echo "Working on: $path"
wget -qO- http://termux.net/$path | dpkg -x - "$__TmpDir"
fi
done
cp -R "$__TmpDir/data/data/com.termux/files/usr/"* "$__ToolchainDir/sysroot/usr/"
# Generate platform file for build.sh script to assign to __DistroRid
echo "Generating platform file..."
echo "RID=android.${__ApiLevel}-${__BuildArch}" > $__ToolchainDir/sysroot/android_platform
echo "Now to build coreclr, libraries and installers; run:"
echo ROOTFS_DIR=\$\(realpath $__ToolchainDir/sysroot\) ./build.sh --cross --arch $__BuildArch \
--subsetCategory coreclr
echo ROOTFS_DIR=\$\(realpath $__ToolchainDir/sysroot\) ./build.sh --cross --arch $__BuildArch \
--subsetCategory libraries
echo ROOTFS_DIR=\$\(realpath $__ToolchainDir/sysroot\) ./build.sh --cross --arch $__BuildArch \
--subsetCategory installer

100
eng/common/cross/build-rootfs.sh

@ -2,15 +2,16 @@
usage() usage()
{ {
echo "Usage: $0 [BuildArch] [LinuxCodeName] [lldbx.y] [--skipunmount] --rootfsdir <directory>]" echo "Usage: $0 [BuildArch] [CodeName] [lldbx.y] [--skipunmount] --rootfsdir <directory>]"
echo "BuildArch can be: arm(default), armel, arm64, x86" echo "BuildArch can be: arm(default), armel, arm64, x86"
echo "LinuxCodeName - optional, Code name for Linux, can be: trusty, xenial(default), zesty, bionic, alpine. If BuildArch is armel, LinuxCodeName is jessie(default) or tizen." echo "CodeName - optional, Code name for Linux, can be: trusty, xenial(default), zesty, bionic, alpine. If BuildArch is armel, LinuxCodeName is jessie(default) or tizen."
echo "lldbx.y - optional, LLDB version, can be: lldb3.9(default), lldb4.0, lldb5.0, lldb6.0 no-lldb. Ignored for alpine" echo " for FreeBSD can be: freebsd11 or freebsd12."
echo "lldbx.y - optional, LLDB version, can be: lldb3.9(default), lldb4.0, lldb5.0, lldb6.0 no-lldb. Ignored for alpine and FReeBSD"
echo "--skipunmount - optional, will skip the unmount of rootfs folder." echo "--skipunmount - optional, will skip the unmount of rootfs folder."
exit 1 exit 1
} }
__LinuxCodeName=xenial __CodeName=xenial
__CrossDir=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd ) __CrossDir=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )
__InitialDir=$PWD __InitialDir=$PWD
__BuildArch=arm __BuildArch=arm
@ -25,8 +26,9 @@ __UbuntuPackages="build-essential"
__AlpinePackages="alpine-base" __AlpinePackages="alpine-base"
__AlpinePackages+=" build-base" __AlpinePackages+=" build-base"
__AlpinePackages+=" linux-headers" __AlpinePackages+=" linux-headers"
__AlpinePackages+=" lldb-dev" __AlpinePackagesEdgeTesting=" lldb-dev"
__AlpinePackages+=" llvm-dev" __AlpinePackagesEdgeMain=" llvm9-libs"
__AlpinePackagesEdgeMain+=" python3"
# symlinks fixer # symlinks fixer
__UbuntuPackages+=" symlinks" __UbuntuPackages+=" symlinks"
@ -52,6 +54,15 @@ __AlpinePackages+=" krb5-dev"
__AlpinePackages+=" openssl-dev" __AlpinePackages+=" openssl-dev"
__AlpinePackages+=" zlib-dev" __AlpinePackages+=" zlib-dev"
__FreeBSDBase="12.1-RELEASE"
__FreeBSDPkg="1.10.5"
__FreeBSDPackages="libunwind"
__FreeBSDPackages+=" icu"
__FreeBSDPackages+=" libinotify"
__FreeBSDPackages+=" lttng-ust"
__FreeBSDPackages+=" llvm-90"
__FreeBSDPackages+=" krb5"
__UnprocessedBuildArgs= __UnprocessedBuildArgs=
while :; do while :; do
if [ $# -le 0 ]; then if [ $# -le 0 ]; then
@ -80,7 +91,7 @@ while :; do
__BuildArch=armel __BuildArch=armel
__UbuntuArch=armel __UbuntuArch=armel
__UbuntuRepo="http://ftp.debian.org/debian/" __UbuntuRepo="http://ftp.debian.org/debian/"
__LinuxCodeName=jessie __CodeName=jessie
;; ;;
x86) x86)
__BuildArch=x86 __BuildArch=x86
@ -109,36 +120,36 @@ while :; do
unset __LLDB_Package unset __LLDB_Package
;; ;;
trusty) # Ubuntu 14.04 trusty) # Ubuntu 14.04
if [ "$__LinuxCodeName" != "jessie" ]; then if [ "$__CodeName" != "jessie" ]; then
__LinuxCodeName=trusty __CodeName=trusty
fi fi
;; ;;
xenial) # Ubuntu 16.04 xenial) # Ubuntu 16.04
if [ "$__LinuxCodeName" != "jessie" ]; then if [ "$__CodeName" != "jessie" ]; then
__LinuxCodeName=xenial __CodeName=xenial
fi fi
;; ;;
zesty) # Ubuntu 17.04 zesty) # Ubuntu 17.04
if [ "$__LinuxCodeName" != "jessie" ]; then if [ "$__CodeName" != "jessie" ]; then
__LinuxCodeName=zesty __CodeName=zesty
fi fi
;; ;;
bionic) # Ubuntu 18.04 bionic) # Ubuntu 18.04
if [ "$__LinuxCodeName" != "jessie" ]; then if [ "$__CodeName" != "jessie" ]; then
__LinuxCodeName=bionic __CodeName=bionic
fi fi
;; ;;
jessie) # Debian 8 jessie) # Debian 8
__LinuxCodeName=jessie __CodeName=jessie
__UbuntuRepo="http://ftp.debian.org/debian/" __UbuntuRepo="http://ftp.debian.org/debian/"
;; ;;
stretch) # Debian 9 stretch) # Debian 9
__LinuxCodeName=stretch __CodeName=stretch
__UbuntuRepo="http://ftp.debian.org/debian/" __UbuntuRepo="http://ftp.debian.org/debian/"
__LLDB_Package="liblldb-6.0-dev" __LLDB_Package="liblldb-6.0-dev"
;; ;;
buster) # Debian 10 buster) # Debian 10
__LinuxCodeName=buster __CodeName=buster
__UbuntuRepo="http://ftp.debian.org/debian/" __UbuntuRepo="http://ftp.debian.org/debian/"
__LLDB_Package="liblldb-6.0-dev" __LLDB_Package="liblldb-6.0-dev"
;; ;;
@ -148,14 +159,22 @@ while :; do
usage; usage;
exit 1; exit 1;
fi fi
__LinuxCodeName= __CodeName=
__UbuntuRepo= __UbuntuRepo=
__Tizen=tizen __Tizen=tizen
;; ;;
alpine) alpine)
__LinuxCodeName=alpine __CodeName=alpine
__UbuntuRepo= __UbuntuRepo=
;; ;;
freebsd11)
__FreeBSDBase="11.3-RELEASE"
;&
freebsd12)
__CodeName=freebsd
__BuildArch=x64
__SkipUnmount=1
;;
--skipunmount) --skipunmount)
__SkipUnmount=1 __SkipUnmount=1
;; ;;
@ -191,25 +210,50 @@ if [ -d "$__RootfsDir" ]; then
rm -rf $__RootfsDir rm -rf $__RootfsDir
fi fi
if [[ "$__LinuxCodeName" == "alpine" ]]; then if [[ "$__CodeName" == "alpine" ]]; then
__ApkToolsVersion=2.9.1 __ApkToolsVersion=2.9.1
__AlpineVersion=3.7 __AlpineVersion=3.9
__ApkToolsDir=$(mktemp -d) __ApkToolsDir=$(mktemp -d)
wget https://github.com/alpinelinux/apk-tools/releases/download/v$__ApkToolsVersion/apk-tools-$__ApkToolsVersion-x86_64-linux.tar.gz -P $__ApkToolsDir wget https://github.com/alpinelinux/apk-tools/releases/download/v$__ApkToolsVersion/apk-tools-$__ApkToolsVersion-x86_64-linux.tar.gz -P $__ApkToolsDir
tar -xf $__ApkToolsDir/apk-tools-$__ApkToolsVersion-x86_64-linux.tar.gz -C $__ApkToolsDir tar -xf $__ApkToolsDir/apk-tools-$__ApkToolsVersion-x86_64-linux.tar.gz -C $__ApkToolsDir
mkdir -p $__RootfsDir/usr/bin mkdir -p $__RootfsDir/usr/bin
cp -v /usr/bin/qemu-$__QEMUArch-static $__RootfsDir/usr/bin cp -v /usr/bin/qemu-$__QEMUArch-static $__RootfsDir/usr/bin
$__ApkToolsDir/apk-tools-$__ApkToolsVersion/apk \ $__ApkToolsDir/apk-tools-$__ApkToolsVersion/apk \
-X http://dl-cdn.alpinelinux.org/alpine/v$__AlpineVersion/main \ -X http://dl-cdn.alpinelinux.org/alpine/v$__AlpineVersion/main \
-X http://dl-cdn.alpinelinux.org/alpine/v$__AlpineVersion/community \ -X http://dl-cdn.alpinelinux.org/alpine/v$__AlpineVersion/community \
-X http://dl-cdn.alpinelinux.org/alpine/edge/testing \
-X http://dl-cdn.alpinelinux.org/alpine/edge/main \
-U --allow-untrusted --root $__RootfsDir --arch $__AlpineArch --initdb \ -U --allow-untrusted --root $__RootfsDir --arch $__AlpineArch --initdb \
add $__AlpinePackages add $__AlpinePackages
$__ApkToolsDir/apk-tools-$__ApkToolsVersion/apk \
-X http://dl-cdn.alpinelinux.org/alpine/edge/main \
-U --allow-untrusted --root $__RootfsDir --arch $__AlpineArch --initdb \
add $__AlpinePackagesEdgeMain
$__ApkToolsDir/apk-tools-$__ApkToolsVersion/apk \
-X http://dl-cdn.alpinelinux.org/alpine/edge/testing \
-U --allow-untrusted --root $__RootfsDir --arch $__AlpineArch --initdb \
add $__AlpinePackagesEdgeTesting
rm -r $__ApkToolsDir rm -r $__ApkToolsDir
elif [[ -n $__LinuxCodeName ]]; then elif [[ "$__CodeName" == "freebsd" ]]; then
qemu-debootstrap --arch $__UbuntuArch $__LinuxCodeName $__RootfsDir $__UbuntuRepo mkdir -p $__RootfsDir/usr/local/etc
cp $__CrossDir/$__BuildArch/sources.list.$__LinuxCodeName $__RootfsDir/etc/apt/sources.list wget -O - https://download.freebsd.org/ftp/releases/amd64/${__FreeBSDBase}/base.txz | tar -C $__RootfsDir -Jxf - ./lib ./usr/lib ./usr/libdata ./usr/include ./usr/share/keys ./etc ./bin/freebsd-version
# For now, ask for 11 ABI even on 12. This can be revisited later.
echo "ABI = \"FreeBSD:11:amd64\"; FINGERPRINTS = \"${__RootfsDir}/usr/share/keys\"; REPOS_DIR = [\"${__RootfsDir}/etc/pkg\"]; REPO_AUTOUPDATE = NO; RUN_SCRIPTS = NO;" > ${__RootfsDir}/usr/local/etc/pkg.conf
echo "FreeBSD: { url: "pkg+http://pkg.FreeBSD.org/\${ABI}/quarterly", mirror_type: \"srv\", signature_type: \"fingerprints\", fingerprints: \"${__RootfsDir}/usr/share/keys/pkg\", enabled: yes }" > ${__RootfsDir}/etc/pkg/FreeBSD.conf
mkdir -p $__RootfsDir/tmp
# get and build package manager
wget -O - https://github.com/freebsd/pkg/archive/${__FreeBSDPkg}.tar.gz | tar -C $__RootfsDir/tmp -zxf -
cd $__RootfsDir/tmp/pkg-${__FreeBSDPkg}
./autogen.sh && ./configure --prefix=$__RootfsDir/host && make install
rm -rf $__RootfsDir/tmp/pkg-${__FreeBSDPkg}
# install packages we need.
$__RootfsDir/host/sbin/pkg -r $__RootfsDir -C $__RootfsDir/usr/local/etc/pkg.conf update
$__RootfsDir/host/sbin/pkg -r $__RootfsDir -C $__RootfsDir/usr/local/etc/pkg.conf install --yes $__FreeBSDPackages
elif [[ -n $__CodeName ]]; then
qemu-debootstrap --arch $__UbuntuArch $__CodeName $__RootfsDir $__UbuntuRepo
cp $__CrossDir/$__BuildArch/sources.list.$__CodeName $__RootfsDir/etc/apt/sources.list
chroot $__RootfsDir apt-get update chroot $__RootfsDir apt-get update
chroot $__RootfsDir apt-get -f -y install chroot $__RootfsDir apt-get -f -y install
chroot $__RootfsDir apt-get -y install $__UbuntuPackages chroot $__RootfsDir apt-get -y install $__UbuntuPackages
@ -219,7 +263,7 @@ elif [[ -n $__LinuxCodeName ]]; then
umount $__RootfsDir/* umount $__RootfsDir/*
fi fi
if [[ "$__BuildArch" == "arm" && "$__LinuxCodeName" == "trusty" ]]; then if [[ "$__BuildArch" == "arm" && "$__CodeName" == "trusty" ]]; then
pushd $__RootfsDir pushd $__RootfsDir
patch -p1 < $__CrossDir/$__BuildArch/trusty.patch patch -p1 < $__CrossDir/$__BuildArch/trusty.patch
patch -p1 < $__CrossDir/$__BuildArch/trusty-lttng-2.4.patch patch -p1 < $__CrossDir/$__BuildArch/trusty-lttng-2.4.patch

81
eng/common/cross/toolchain.cmake

@ -31,6 +31,10 @@ else()
message(FATAL_ERROR "Arch is ${TARGET_ARCH_NAME}. Only armel, arm, arm64 and x86 are supported!") message(FATAL_ERROR "Arch is ${TARGET_ARCH_NAME}. Only armel, arm, arm64 and x86 are supported!")
endif() endif()
if(DEFINED ENV{TOOLCHAIN})
set(TOOLCHAIN $ENV{TOOLCHAIN})
endif()
# Specify include paths # Specify include paths
if(TARGET_ARCH_NAME STREQUAL "armel") if(TARGET_ARCH_NAME STREQUAL "armel")
if(DEFINED TIZEN_TOOLCHAIN) if(DEFINED TIZEN_TOOLCHAIN)
@ -39,50 +43,47 @@ if(TARGET_ARCH_NAME STREQUAL "armel")
endif() endif()
endif() endif()
# add_compile_param - adds only new options without duplicates. if("$ENV{__DistroRid}" MATCHES "android.*")
# arg0 - list with result options, arg1 - list with new options. if(TARGET_ARCH_NAME STREQUAL "arm")
# arg2 - optional argument, quick summary string for optional using CACHE FORCE mode. set(ANDROID_ABI armeabi-v7a)
macro(add_compile_param) elseif(TARGET_ARCH_NAME STREQUAL "arm64")
if(NOT ${ARGC} MATCHES "^(2|3)$") set(ANDROID_ABI arm64-v8a)
message(FATAL_ERROR "Wrong using add_compile_param! Two or three parameters must be given! See add_compile_param description.")
endif()
foreach(OPTION ${ARGV1})
if(NOT ${ARGV0} MATCHES "${OPTION}($| )")
set(${ARGV0} "${${ARGV0}} ${OPTION}")
if(${ARGC} EQUAL "3") # CACHE FORCE mode
set(${ARGV0} "${${ARGV0}}" CACHE STRING "${ARGV2}" FORCE)
endif()
endif() endif()
endforeach()
endmacro() # extract platform number required by the NDK's toolchain
string(REGEX REPLACE ".*\\.([0-9]+)-.*" "\\1" ANDROID_PLATFORM "$ENV{__DistroRid}")
set(ANDROID_TOOLCHAIN clang)
set(FEATURE_EVENT_TRACE 0) # disable event trace as there is no lttng-ust package in termux repository
set(CMAKE_SYSTEM_LIBRARY_PATH "${CROSS_ROOTFS}/usr/lib")
set(CMAKE_SYSTEM_INCLUDE_PATH "${CROSS_ROOTFS}/usr/include")
# include official NDK toolchain script
include(${CROSS_ROOTFS}/../build/cmake/android.toolchain.cmake)
else()
set(CMAKE_SYSROOT "${CROSS_ROOTFS}")
set(CMAKE_C_COMPILER_EXTERNAL_TOOLCHAIN "${CROSS_ROOTFS}/usr")
set(CMAKE_CXX_COMPILER_EXTERNAL_TOOLCHAIN "${CROSS_ROOTFS}/usr")
set(CMAKE_ASM_COMPILER_EXTERNAL_TOOLCHAIN "${CROSS_ROOTFS}/usr")
endif()
# Specify link flags # Specify link flags
add_compile_param(CROSS_LINK_FLAGS "--sysroot=${CROSS_ROOTFS}")
add_compile_param(CROSS_LINK_FLAGS "--gcc-toolchain=${CROSS_ROOTFS}/usr")
add_compile_param(CROSS_LINK_FLAGS "--target=${TOOLCHAIN}")
add_compile_param(CROSS_LINK_FLAGS "-fuse-ld=gold")
if(TARGET_ARCH_NAME STREQUAL "armel") if(TARGET_ARCH_NAME STREQUAL "armel")
if(DEFINED TIZEN_TOOLCHAIN) # For Tizen only if(DEFINED TIZEN_TOOLCHAIN) # For Tizen only
add_compile_param(CROSS_LINK_FLAGS "-B${CROSS_ROOTFS}/usr/lib/gcc/${TIZEN_TOOLCHAIN}") add_link_options("-B${CROSS_ROOTFS}/usr/lib/gcc/${TIZEN_TOOLCHAIN}")
add_compile_param(CROSS_LINK_FLAGS "-L${CROSS_ROOTFS}/lib") add_link_options("-L${CROSS_ROOTFS}/lib")
add_compile_param(CROSS_LINK_FLAGS "-L${CROSS_ROOTFS}/usr/lib") add_link_options("-L${CROSS_ROOTFS}/usr/lib")
add_compile_param(CROSS_LINK_FLAGS "-L${CROSS_ROOTFS}/usr/lib/gcc/${TIZEN_TOOLCHAIN}") add_link_options("-L${CROSS_ROOTFS}/usr/lib/gcc/${TIZEN_TOOLCHAIN}")
endif() endif()
elseif(TARGET_ARCH_NAME STREQUAL "x86") elseif(TARGET_ARCH_NAME STREQUAL "x86")
add_compile_param(CROSS_LINK_FLAGS "-m32") add_link_options(-m32)
endif() endif()
add_compile_param(CMAKE_EXE_LINKER_FLAGS "${CROSS_LINK_FLAGS}" "TOOLCHAIN_EXE_LINKER_FLAGS")
add_compile_param(CMAKE_SHARED_LINKER_FLAGS "${CROSS_LINK_FLAGS}" "TOOLCHAIN_EXE_LINKER_FLAGS")
add_compile_param(CMAKE_MODULE_LINKER_FLAGS "${CROSS_LINK_FLAGS}" "TOOLCHAIN_EXE_LINKER_FLAGS")
# Specify compile options # Specify compile options
add_compile_options("--sysroot=${CROSS_ROOTFS}")
add_compile_options("--target=${TOOLCHAIN}")
add_compile_options("--gcc-toolchain=${CROSS_ROOTFS}/usr")
if(TARGET_ARCH_NAME MATCHES "^(arm|armel|arm64)$") if(TARGET_ARCH_NAME MATCHES "^(arm|armel|arm64)$" AND NOT "$ENV{__DistroRid}" MATCHES "android.*")
set(CMAKE_C_COMPILER_TARGET ${TOOLCHAIN}) set(CMAKE_C_COMPILER_TARGET ${TOOLCHAIN})
set(CMAKE_CXX_COMPILER_TARGET ${TOOLCHAIN}) set(CMAKE_CXX_COMPILER_TARGET ${TOOLCHAIN})
set(CMAKE_ASM_COMPILER_TARGET ${TOOLCHAIN}) set(CMAKE_ASM_COMPILER_TARGET ${TOOLCHAIN})
@ -90,7 +91,17 @@ endif()
if(TARGET_ARCH_NAME MATCHES "^(arm|armel)$") if(TARGET_ARCH_NAME MATCHES "^(arm|armel)$")
add_compile_options(-mthumb) add_compile_options(-mthumb)
add_compile_options(-mfpu=vfpv3) if (NOT DEFINED CLR_ARM_FPU_TYPE)
set (CLR_ARM_FPU_TYPE vfpv3)
endif (NOT DEFINED CLR_ARM_FPU_TYPE)
add_compile_options (-mfpu=${CLR_ARM_FPU_TYPE})
if (NOT DEFINED CLR_ARM_FPU_CAPABILITY)
set (CLR_ARM_FPU_CAPABILITY 0x7)
endif (NOT DEFINED CLR_ARM_FPU_CAPABILITY)
add_definitions (-DCLR_ARM_FPU_CAPABILITY=${CLR_ARM_FPU_CAPABILITY})
if(TARGET_ARCH_NAME STREQUAL "armel") if(TARGET_ARCH_NAME STREQUAL "armel")
add_compile_options(-mfloat-abi=softfp) add_compile_options(-mfloat-abi=softfp)
if(DEFINED TIZEN_TOOLCHAIN) if(DEFINED TIZEN_TOOLCHAIN)
@ -103,7 +114,7 @@ elseif(TARGET_ARCH_NAME STREQUAL "x86")
add_compile_options(-Wno-error=unused-command-line-argument) add_compile_options(-Wno-error=unused-command-line-argument)
endif() endif()
# Set LLDB include and library paths # Set LLDB include and library paths for builds that need lldb.
if(TARGET_ARCH_NAME MATCHES "^(arm|armel|x86)$") if(TARGET_ARCH_NAME MATCHES "^(arm|armel|x86)$")
if(TARGET_ARCH_NAME STREQUAL "x86") if(TARGET_ARCH_NAME STREQUAL "x86")
set(LLVM_CROSS_DIR "$ENV{LLVM_CROSS_HOME}") set(LLVM_CROSS_DIR "$ENV{LLVM_CROSS_HOME}")
@ -131,7 +142,7 @@ if(TARGET_ARCH_NAME MATCHES "^(arm|armel|x86)$")
endif() endif()
endif() endif()
set(CMAKE_FIND_ROOT_PATH "${CROSS_ROOTFS}")
set(CMAKE_FIND_ROOT_PATH_MODE_PROGRAM NEVER) set(CMAKE_FIND_ROOT_PATH_MODE_PROGRAM NEVER)
set(CMAKE_FIND_ROOT_PATH_MODE_LIBRARY ONLY) set(CMAKE_FIND_ROOT_PATH_MODE_LIBRARY ONLY)
set(CMAKE_FIND_ROOT_PATH_MODE_INCLUDE ONLY) set(CMAKE_FIND_ROOT_PATH_MODE_INCLUDE ONLY)

30
eng/common/darc-init.ps1

@ -1,13 +1,14 @@
param ( param (
$darcVersion = $null, $darcVersion = $null,
$versionEndpoint = "https://maestro-prod.westus2.cloudapp.azure.com/api/assets/darc-version?api-version=2019-01-16", $versionEndpoint = 'https://maestro-prod.westus2.cloudapp.azure.com/api/assets/darc-version?api-version=2019-01-16',
$verbosity = "m" $verbosity = 'minimal',
$toolpath = $null
) )
. $PSScriptRoot\tools.ps1 . $PSScriptRoot\tools.ps1
function InstallDarcCli ($darcVersion) { function InstallDarcCli ($darcVersion, $toolpath) {
$darcCliPackageName = "microsoft.dotnet.darc" $darcCliPackageName = 'microsoft.dotnet.darc'
$dotnetRoot = InitializeDotNetCli -install:$true $dotnetRoot = InitializeDotNetCli -install:$true
$dotnet = "$dotnetRoot\dotnet.exe" $dotnet = "$dotnetRoot\dotnet.exe"
@ -23,11 +24,24 @@ function InstallDarcCli ($darcVersion) {
$darcVersion = $(Invoke-WebRequest -Uri $versionEndpoint -UseBasicParsing).Content $darcVersion = $(Invoke-WebRequest -Uri $versionEndpoint -UseBasicParsing).Content
} }
$arcadeServicesSource = 'https://dotnetfeed.blob.core.windows.net/dotnet-core/index.json' $arcadeServicesSource = 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-eng/nuget/v3/index.json'
Write-Host "Installing Darc CLI version $darcVersion..." Write-Host "Installing Darc CLI version $darcVersion..."
Write-Host "You may need to restart your command window if this is the first dotnet tool you have installed." Write-Host 'You may need to restart your command window if this is the first dotnet tool you have installed.'
& "$dotnet" tool install $darcCliPackageName --version $darcVersion --add-source "$arcadeServicesSource" -v $verbosity -g if (-not $toolpath) {
Write-Host "'$dotnet' tool install $darcCliPackageName --version $darcVersion --add-source '$arcadeServicesSource' -v $verbosity -g"
& "$dotnet" tool install $darcCliPackageName --version $darcVersion --add-source "$arcadeServicesSource" -v $verbosity -g
}else {
Write-Host "'$dotnet' tool install $darcCliPackageName --version $darcVersion --add-source '$arcadeServicesSource' -v $verbosity --tool-path '$toolpath'"
& "$dotnet" tool install $darcCliPackageName --version $darcVersion --add-source "$arcadeServicesSource" -v $verbosity --tool-path "$toolpath"
}
} }
InstallDarcCli $darcVersion try {
InstallDarcCli $darcVersion $toolpath
}
catch {
Write-Host $_.ScriptStackTrace
Write-PipelineTelemetryError -Category 'Darc' -Message $_
ExitWithExitCode 1
}

30
eng/common/darc-init.sh

@ -2,8 +2,8 @@
source="${BASH_SOURCE[0]}" source="${BASH_SOURCE[0]}"
darcVersion='' darcVersion=''
versionEndpoint="https://maestro-prod.westus2.cloudapp.azure.com/api/assets/darc-version?api-version=2019-01-16" versionEndpoint='https://maestro-prod.westus2.cloudapp.azure.com/api/assets/darc-version?api-version=2019-01-16'
verbosity=m verbosity='minimal'
while [[ $# > 0 ]]; do while [[ $# > 0 ]]; do
opt="$(echo "$1" | awk '{print tolower($0)}')" opt="$(echo "$1" | awk '{print tolower($0)}')"
@ -20,6 +20,10 @@ while [[ $# > 0 ]]; do
verbosity=$2 verbosity=$2
shift shift
;; ;;
--toolpath)
toolpath=$2
shift
;;
*) *)
echo "Invalid argument: $1" echo "Invalid argument: $1"
usage usage
@ -52,17 +56,27 @@ function InstallDarcCli {
InitializeDotNetCli InitializeDotNetCli
local dotnet_root=$_InitializeDotNetCli local dotnet_root=$_InitializeDotNetCli
local uninstall_command=`$dotnet_root/dotnet tool uninstall $darc_cli_package_name -g` if [ -z "$toolpath" ]; then
local tool_list=$($dotnet_root/dotnet tool list -g) local tool_list=$($dotnet_root/dotnet tool list -g)
if [[ $tool_list = *$darc_cli_package_name* ]]; then if [[ $tool_list = *$darc_cli_package_name* ]]; then
echo $($dotnet_root/dotnet tool uninstall $darc_cli_package_name -g) echo $($dotnet_root/dotnet tool uninstall $darc_cli_package_name -g)
fi
else
local tool_list=$($dotnet_root/dotnet tool list --tool-path "$toolpath")
if [[ $tool_list = *$darc_cli_package_name* ]]; then
echo $($dotnet_root/dotnet tool uninstall $darc_cli_package_name --tool-path "$toolpath")
fi
fi fi
local arcadeServicesSource="https://dotnetfeed.blob.core.windows.net/dotnet-core/index.json" local arcadeServicesSource="https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools/nuget/v3/index.json"
echo "Installing Darc CLI version $darcVersion..." echo "Installing Darc CLI version $darcVersion..."
echo "You may need to restart your command shell if this is the first dotnet tool you have installed." echo "You may need to restart your command shell if this is the first dotnet tool you have installed."
echo $($dotnet_root/dotnet tool install $darc_cli_package_name --version $darcVersion --add-source "$arcadeServicesSource" -v $verbosity -g) if [ -z "$toolpath" ]; then
echo $($dotnet_root/dotnet tool install $darc_cli_package_name --version $darcVersion --add-source "$arcadeServicesSource" -v $verbosity -g)
else
echo $($dotnet_root/dotnet tool install $darc_cli_package_name --version $darcVersion --add-source "$arcadeServicesSource" -v $verbosity --tool-path "$toolpath")
fi
} }
InstallDarcCli InstallDarcCli

23
eng/common/dotnet-install.ps1

@ -1,26 +1,27 @@
[CmdletBinding(PositionalBinding=$false)] [CmdletBinding(PositionalBinding=$false)]
Param( Param(
[string] $verbosity = "minimal", [string] $verbosity = 'minimal',
[string] $architecture = "", [string] $architecture = '',
[string] $version = "Latest", [string] $version = 'Latest',
[string] $runtime = "dotnet" [string] $runtime = 'dotnet',
[string] $RuntimeSourceFeed = '',
[string] $RuntimeSourceFeedKey = ''
) )
. $PSScriptRoot\tools.ps1 . $PSScriptRoot\tools.ps1
$dotnetRoot = Join-Path $RepoRoot ".dotnet" $dotnetRoot = Join-Path $RepoRoot '.dotnet'
$installdir = $dotnetRoot $installdir = $dotnetRoot
try { try {
if ($architecture -and $architecture.Trim() -eq "x86") { if ($architecture -and $architecture.Trim() -eq 'x86') {
$installdir = Join-Path $installdir "x86" $installdir = Join-Path $installdir 'x86'
} }
InstallDotNet $installdir $version $architecture $runtime $true InstallDotNet $installdir $version $architecture $runtime $true -RuntimeSourceFeed $RuntimeSourceFeed -RuntimeSourceFeedKey $RuntimeSourceFeedKey
} }
catch { catch {
Write-Host $_
Write-Host $_.Exception
Write-Host $_.ScriptStackTrace Write-Host $_.ScriptStackTrace
Write-PipelineTelemetryError -Category 'InitializeToolset' -Message $_
ExitWithExitCode 1 ExitWithExitCode 1
} }

50
eng/common/dotnet-install.sh

@ -11,9 +11,13 @@ while [[ -h "$source" ]]; do
done done
scriptroot="$( cd -P "$( dirname "$source" )" && pwd )" scriptroot="$( cd -P "$( dirname "$source" )" && pwd )"
. "$scriptroot/tools.sh"
version='Latest' version='Latest'
architecture='' architecture=''
runtime='dotnet' runtime='dotnet'
runtimeSourceFeed=''
runtimeSourceFeedKey=''
while [[ $# > 0 ]]; do while [[ $# > 0 ]]; do
opt="$(echo "$1" | awk '{print tolower($0)}')" opt="$(echo "$1" | awk '{print tolower($0)}')"
case "$opt" in case "$opt" in
@ -29,20 +33,56 @@ while [[ $# > 0 ]]; do
shift shift
runtime="$1" runtime="$1"
;; ;;
-runtimesourcefeed)
shift
runtimeSourceFeed="$1"
;;
-runtimesourcefeedkey)
shift
runtimeSourceFeedKey="$1"
;;
*) *)
echo "Invalid argument: $1" Write-PipelineTelemetryError -Category 'Build' -Message "Invalid argument: $1"
usage
exit 1 exit 1
;; ;;
esac esac
shift shift
done done
. "$scriptroot/tools.sh" # Use uname to determine what the CPU is.
cpuname=$(uname -p)
# Some Linux platforms report unknown for platform, but the arch for machine.
if [[ "$cpuname" == "unknown" ]]; then
cpuname=$(uname -m)
fi
case $cpuname in
aarch64)
buildarch=arm64
;;
amd64|x86_64)
buildarch=x64
;;
armv*l)
buildarch=arm
;;
i686)
buildarch=x86
;;
*)
echo "Unknown CPU $cpuname detected, treating it as x64"
buildarch=x64
;;
esac
dotnetRoot="$repo_root/.dotnet" dotnetRoot="$repo_root/.dotnet"
InstallDotNet $dotnetRoot $version "$architecture" $runtime true || { if [[ $architecture != "" ]] && [[ $architecture != $buildarch ]]; then
dotnetRoot="$dotnetRoot/$architecture"
fi
InstallDotNet $dotnetRoot $version "$architecture" $runtime true $runtimeSourceFeed $runtimeSourceFeedKey || {
local exit_code=$? local exit_code=$?
echo "dotnet-install.sh failed (exit code '$exit_code')." >&2 Write-PipelineTelemetryError -Category 'InitializeToolset' -Message "dotnet-install.sh failed (exit code '$exit_code')." >&2
ExitWithExitCode $exit_code ExitWithExitCode $exit_code
} }

11
eng/common/enable-cross-org-publishing.ps1

@ -2,5 +2,12 @@ param(
[string] $token [string] $token
) )
Write-Host "##vso[task.setvariable variable=VSS_NUGET_ACCESSTOKEN]$token"
Write-Host "##vso[task.setvariable variable=VSS_NUGET_URI_PREFIXES]https://dnceng.pkgs.visualstudio.com/;https://pkgs.dev.azure.com/dnceng/;https://devdiv.pkgs.visualstudio.com/;https://pkgs.dev.azure.com/devdiv/" . $PSScriptRoot\pipeline-logging-functions.ps1
# Write-PipelineSetVariable will no-op if a variable named $ci is not defined
# Since this script is only ever called in AzDO builds, just universally set it
$ci = $true
Write-PipelineSetVariable -Name 'VSS_NUGET_ACCESSTOKEN' -Value $token -IsMultiJobVariable $false
Write-PipelineSetVariable -Name 'VSS_NUGET_URI_PREFIXES' -Value 'https://dnceng.pkgs.visualstudio.com/;https://pkgs.dev.azure.com/dnceng/;https://devdiv.pkgs.visualstudio.com/;https://pkgs.dev.azure.com/devdiv/' -IsMultiJobVariable $false

51
eng/common/generate-graph-files.ps1

@ -3,39 +3,39 @@ Param(
[Parameter(Mandatory=$true)][string] $gitHubPat, # GitHub personal access token from https://github.com/settings/tokens (no auth scopes needed) [Parameter(Mandatory=$true)][string] $gitHubPat, # GitHub personal access token from https://github.com/settings/tokens (no auth scopes needed)
[Parameter(Mandatory=$true)][string] $azdoPat, # Azure Dev Ops tokens from https://dev.azure.com/dnceng/_details/security/tokens (code read scope needed) [Parameter(Mandatory=$true)][string] $azdoPat, # Azure Dev Ops tokens from https://dev.azure.com/dnceng/_details/security/tokens (code read scope needed)
[Parameter(Mandatory=$true)][string] $outputFolder, # Where the graphviz.txt file will be created [Parameter(Mandatory=$true)][string] $outputFolder, # Where the graphviz.txt file will be created
[string] $darcVersion = '1.1.0-beta.19175.6', # darc's version [string] $darcVersion, # darc's version
[string] $graphvizVersion = '2.38', # GraphViz version [string] $graphvizVersion = '2.38', # GraphViz version
[switch] $includeToolset # Whether the graph should include toolset dependencies or not. i.e. arcade, optimization. For more about [switch] $includeToolset # Whether the graph should include toolset dependencies or not. i.e. arcade, optimization. For more about
# toolset dependencies see https://github.com/dotnet/arcade/blob/master/Documentation/Darc.md#toolset-vs-product-dependencies # toolset dependencies see https://github.com/dotnet/arcade/blob/master/Documentation/Darc.md#toolset-vs-product-dependencies
) )
$ErrorActionPreference = "Stop"
. $PSScriptRoot\tools.ps1
Import-Module -Name (Join-Path $PSScriptRoot "native\CommonLibrary.psm1")
function CheckExitCode ([string]$stage) function CheckExitCode ([string]$stage)
{ {
$exitCode = $LASTEXITCODE $exitCode = $LASTEXITCODE
if ($exitCode -ne 0) { if ($exitCode -ne 0) {
Write-Host "Something failed in stage: '$stage'. Check for errors above. Exiting now..." Write-PipelineTelemetryError -Category 'Arcade' -Message "Something failed in stage: '$stage'. Check for errors above. Exiting now..."
ExitWithExitCode $exitCode ExitWithExitCode $exitCode
} }
} }
try { try {
$ErrorActionPreference = 'Stop'
. $PSScriptRoot\tools.ps1
Import-Module -Name (Join-Path $PSScriptRoot 'native\CommonLibrary.psm1')
Push-Location $PSScriptRoot Push-Location $PSScriptRoot
Write-Host "Installing darc..." Write-Host 'Installing darc...'
. .\darc-init.ps1 -darcVersion $darcVersion . .\darc-init.ps1 -darcVersion $darcVersion
CheckExitCode "Running darc-init" CheckExitCode 'Running darc-init'
$engCommonBaseDir = Join-Path $PSScriptRoot "native\" $engCommonBaseDir = Join-Path $PSScriptRoot 'native\'
$graphvizInstallDir = CommonLibrary\Get-NativeInstallDirectory $graphvizInstallDir = CommonLibrary\Get-NativeInstallDirectory
$nativeToolBaseUri = "https://netcorenativeassets.blob.core.windows.net/resource-packages/external" $nativeToolBaseUri = 'https://netcorenativeassets.blob.core.windows.net/resource-packages/external'
$installBin = Join-Path $graphvizInstallDir "bin" $installBin = Join-Path $graphvizInstallDir 'bin'
Write-Host "Installing dot..." Write-Host 'Installing dot...'
.\native\install-tool.ps1 -ToolName graphviz -InstallPath $installBin -BaseUri $nativeToolBaseUri -CommonLibraryDirectory $engCommonBaseDir -Version $graphvizVersion -Verbose .\native\install-tool.ps1 -ToolName graphviz -InstallPath $installBin -BaseUri $nativeToolBaseUri -CommonLibraryDirectory $engCommonBaseDir -Version $graphvizVersion -Verbose
$darcExe = "$env:USERPROFILE\.dotnet\tools" $darcExe = "$env:USERPROFILE\.dotnet\tools"
@ -51,37 +51,36 @@ try {
$graphVizImageFilePath = "$outputFolder\graph.png" $graphVizImageFilePath = "$outputFolder\graph.png"
$normalGraphFilePath = "$outputFolder\graph-full.txt" $normalGraphFilePath = "$outputFolder\graph-full.txt"
$flatGraphFilePath = "$outputFolder\graph-flat.txt" $flatGraphFilePath = "$outputFolder\graph-flat.txt"
$baseOptions = @( "--github-pat", "$gitHubPat", "--azdev-pat", "$azdoPat", "--password", "$barToken" ) $baseOptions = @( '--github-pat', "$gitHubPat", '--azdev-pat', "$azdoPat", '--password', "$barToken" )
if ($includeToolset) { if ($includeToolset) {
Write-Host "Toolsets will be included in the graph..." Write-Host 'Toolsets will be included in the graph...'
$baseOptions += @( "--include-toolset" ) $baseOptions += @( '--include-toolset' )
} }
Write-Host "Generating standard dependency graph..." Write-Host 'Generating standard dependency graph...'
& "$darcExe" get-dependency-graph @baseOptions --output-file $normalGraphFilePath & "$darcExe" get-dependency-graph @baseOptions --output-file $normalGraphFilePath
CheckExitCode "Generating normal dependency graph" CheckExitCode 'Generating normal dependency graph'
Write-Host "Generating flat dependency graph and graphviz file..." Write-Host 'Generating flat dependency graph and graphviz file...'
& "$darcExe" get-dependency-graph @baseOptions --flat --coherency --graphviz $graphVizFilePath --output-file $flatGraphFilePath & "$darcExe" get-dependency-graph @baseOptions --flat --coherency --graphviz $graphVizFilePath --output-file $flatGraphFilePath
CheckExitCode "Generating flat and graphviz dependency graph" CheckExitCode 'Generating flat and graphviz dependency graph'
Write-Host "Generating graph image $graphVizFilePath" Write-Host "Generating graph image $graphVizFilePath"
$dotFilePath = Join-Path $installBin "graphviz\$graphvizVersion\release\bin\dot.exe" $dotFilePath = Join-Path $installBin "graphviz\$graphvizVersion\release\bin\dot.exe"
& "$dotFilePath" -Tpng -o"$graphVizImageFilePath" "$graphVizFilePath" & "$dotFilePath" -Tpng -o"$graphVizImageFilePath" "$graphVizFilePath"
CheckExitCode "Generating graphviz image" CheckExitCode 'Generating graphviz image'
Write-Host "'$graphVizFilePath', '$flatGraphFilePath', '$normalGraphFilePath' and '$graphVizImageFilePath' created!" Write-Host "'$graphVizFilePath', '$flatGraphFilePath', '$normalGraphFilePath' and '$graphVizImageFilePath' created!"
} }
catch { catch {
if (!$includeToolset) { if (!$includeToolset) {
Write-Host "This might be a toolset repo which includes only toolset dependencies. " -NoNewline -ForegroundColor Yellow Write-Host 'This might be a toolset repo which includes only toolset dependencies. ' -NoNewline -ForegroundColor Yellow
Write-Host "Since -includeToolset is not set there is no graph to create. Include -includeToolset and try again..." -ForegroundColor Yellow Write-Host 'Since -includeToolset is not set there is no graph to create. Include -includeToolset and try again...' -ForegroundColor Yellow
} }
Write-Host $_
Write-Host $_.Exception
Write-Host $_.ScriptStackTrace Write-Host $_.ScriptStackTrace
Write-PipelineTelemetryError -Category 'Arcade' -Message $_
ExitWithExitCode 1 ExitWithExitCode 1
} finally { } finally {
Pop-Location Pop-Location
} }

37
eng/common/init-tools-native.ps1

@ -35,7 +35,7 @@ File path to global.json file
#> #>
[CmdletBinding(PositionalBinding=$false)] [CmdletBinding(PositionalBinding=$false)]
Param ( Param (
[string] $BaseUri = "https://netcorenativeassets.blob.core.windows.net/resource-packages/external", [string] $BaseUri = 'https://netcorenativeassets.blob.core.windows.net/resource-packages/external',
[string] $InstallDirectory, [string] $InstallDirectory,
[switch] $Clean = $False, [switch] $Clean = $False,
[switch] $Force = $False, [switch] $Force = $False,
@ -45,26 +45,27 @@ Param (
) )
if (!$GlobalJsonFile) { if (!$GlobalJsonFile) {
$GlobalJsonFile = Join-Path (Get-Item $PSScriptRoot).Parent.Parent.FullName "global.json" $GlobalJsonFile = Join-Path (Get-Item $PSScriptRoot).Parent.Parent.FullName 'global.json'
} }
Set-StrictMode -version 2.0 Set-StrictMode -version 2.0
$ErrorActionPreference="Stop" $ErrorActionPreference='Stop'
Import-Module -Name (Join-Path $PSScriptRoot "native\CommonLibrary.psm1") . $PSScriptRoot\pipeline-logging-functions.ps1
Import-Module -Name (Join-Path $PSScriptRoot 'native\CommonLibrary.psm1')
try { try {
# Define verbose switch if undefined # Define verbose switch if undefined
$Verbose = $VerbosePreference -Eq "Continue" $Verbose = $VerbosePreference -Eq 'Continue'
$EngCommonBaseDir = Join-Path $PSScriptRoot "native\" $EngCommonBaseDir = Join-Path $PSScriptRoot 'native\'
$NativeBaseDir = $InstallDirectory $NativeBaseDir = $InstallDirectory
if (!$NativeBaseDir) { if (!$NativeBaseDir) {
$NativeBaseDir = CommonLibrary\Get-NativeInstallDirectory $NativeBaseDir = CommonLibrary\Get-NativeInstallDirectory
} }
$Env:CommonLibrary_NativeInstallDir = $NativeBaseDir $Env:CommonLibrary_NativeInstallDir = $NativeBaseDir
$InstallBin = Join-Path $NativeBaseDir "bin" $InstallBin = Join-Path $NativeBaseDir 'bin'
$InstallerPath = Join-Path $EngCommonBaseDir "install-tool.ps1" $InstallerPath = Join-Path $EngCommonBaseDir 'install-tool.ps1'
# Process tools list # Process tools list
Write-Host "Processing $GlobalJsonFile" Write-Host "Processing $GlobalJsonFile"
@ -74,7 +75,7 @@ try {
} }
$NativeTools = Get-Content($GlobalJsonFile) -Raw | $NativeTools = Get-Content($GlobalJsonFile) -Raw |
ConvertFrom-Json | ConvertFrom-Json |
Select-Object -Expand "native-tools" -ErrorAction SilentlyContinue Select-Object -Expand 'native-tools' -ErrorAction SilentlyContinue
if ($NativeTools) { if ($NativeTools) {
$NativeTools.PSObject.Properties | ForEach-Object { $NativeTools.PSObject.Properties | ForEach-Object {
$ToolName = $_.Name $ToolName = $_.Name
@ -112,18 +113,21 @@ try {
} }
$toolInstallationFailure = $true $toolInstallationFailure = $true
} else { } else {
Write-Error $errMsg # We cannot change this to Write-PipelineTelemetryError because of https://github.com/dotnet/arcade/issues/4482
Write-Host $errMsg
exit 1 exit 1
} }
} }
} }
if ((Get-Variable 'toolInstallationFailure' -ErrorAction 'SilentlyContinue') -and $toolInstallationFailure) { if ((Get-Variable 'toolInstallationFailure' -ErrorAction 'SilentlyContinue') -and $toolInstallationFailure) {
# We cannot change this to Write-PipelineTelemetryError because of https://github.com/dotnet/arcade/issues/4482
Write-Host 'Native tools bootstrap failed'
exit 1 exit 1
} }
} }
else { else {
Write-Host "No native tools defined in global.json" Write-Host 'No native tools defined in global.json'
exit 0 exit 0
} }
@ -131,17 +135,18 @@ try {
exit 0 exit 0
} }
if (Test-Path $InstallBin) { if (Test-Path $InstallBin) {
Write-Host "Native tools are available from" (Convert-Path -Path $InstallBin) Write-Host 'Native tools are available from ' (Convert-Path -Path $InstallBin)
Write-Host "##vso[task.prependpath]$(Convert-Path -Path $InstallBin)" Write-Host "##vso[task.prependpath]$(Convert-Path -Path $InstallBin)"
return $InstallBin
} }
else { else {
Write-Error "Native tools install directory does not exist, installation failed" Write-PipelineTelemetryError -Category 'NativeToolsBootstrap' -Message 'Native tools install directory does not exist, installation failed'
exit 1 exit 1
} }
exit 0 exit 0
} }
catch { catch {
Write-Host $_ Write-Host $_.ScriptStackTrace
Write-Host $_.Exception Write-PipelineTelemetryError -Category 'NativeToolsBootstrap' -Message $_
exit 1 ExitWithExitCode 1
} }

50
eng/common/init-tools-native.sh

@ -12,6 +12,7 @@ retry_wait_time_seconds=30
global_json_file="$(dirname "$(dirname "${scriptroot}")")/global.json" global_json_file="$(dirname "$(dirname "${scriptroot}")")/global.json"
declare -A native_assets declare -A native_assets
. $scriptroot/pipeline-logging-functions.sh
. $scriptroot/native/common-library.sh . $scriptroot/native/common-library.sh
while (($# > 0)); do while (($# > 0)); do
@ -33,6 +34,14 @@ while (($# > 0)); do
force=true force=true
shift 1 shift 1
;; ;;
--donotabortonfailure)
donotabortonfailure=true
shift 1
;;
--donotdisplaywarnings)
donotdisplaywarnings=true
shift 1
;;
--downloadretries) --downloadretries)
download_retries=$2 download_retries=$2
shift 2 shift 2
@ -51,6 +60,8 @@ while (($# > 0)); do
echo " - (default) %USERPROFILE%/.netcoreeng/native" echo " - (default) %USERPROFILE%/.netcoreeng/native"
echo "" echo ""
echo " --clean Switch specifying not to install anything, but cleanup native asset folders" echo " --clean Switch specifying not to install anything, but cleanup native asset folders"
echo " --donotabortonfailure Switch specifiying whether to abort native tools installation on failure"
echo " --donotdisplaywarnings Switch specifiying whether to display warnings during native tools installation on failure"
echo " --force Clean and then install tools" echo " --force Clean and then install tools"
echo " --help Print help and exit" echo " --help Print help and exit"
echo "" echo ""
@ -91,6 +102,7 @@ if [[ -z $install_directory ]]; then
fi fi
install_bin="${native_base_dir}/bin" install_bin="${native_base_dir}/bin"
installed_any=false
ReadGlobalJsonNativeTools ReadGlobalJsonNativeTools
@ -102,8 +114,8 @@ else
for tool in "${!native_assets[@]}" for tool in "${!native_assets[@]}"
do do
tool_version=${native_assets[$tool]} tool_version=${native_assets[$tool]}
installer_name="install-$tool.sh" installer_path="$native_installer_dir/install-$tool.sh"
installer_command="$native_installer_dir/$installer_name" installer_command="$installer_path"
installer_command+=" --baseuri $base_uri" installer_command+=" --baseuri $base_uri"
installer_command+=" --installpath $install_bin" installer_command+=" --installpath $install_bin"
installer_command+=" --version $tool_version" installer_command+=" --version $tool_version"
@ -117,11 +129,29 @@ else
installer_command+=" --clean" installer_command+=" --clean"
fi fi
$installer_command if [[ -a $installer_path ]]; then
$installer_command
if [[ $? != 0 ]]; then if [[ $? != 0 ]]; then
echo "Execution Failed" >&2 if [[ $donotabortonfailure = true ]]; then
exit 1 if [[ $donotdisplaywarnings != true ]]; then
Write-PipelineTelemetryError -category 'NativeToolsBootstrap' "Execution Failed"
fi
else
Write-PipelineTelemetryError -category 'NativeToolsBootstrap' "Execution Failed"
exit 1
fi
else
$installed_any = true
fi
else
if [[ $donotabortonfailure == true ]]; then
if [[ $donotdisplaywarnings != true ]]; then
Write-PipelineTelemetryError -category 'NativeToolsBootstrap' "Execution Failed: no install script"
fi
else
Write-PipelineTelemetryError -category 'NativeToolsBootstrap' "Execution Failed: no install script"
exit 1
fi
fi fi
done done
fi fi
@ -134,8 +164,10 @@ if [[ -d $install_bin ]]; then
echo "Native tools are available from $install_bin" echo "Native tools are available from $install_bin"
echo "##vso[task.prependpath]$install_bin" echo "##vso[task.prependpath]$install_bin"
else else
echo "Native tools install directory does not exist, installation failed" >&2 if [[ $installed_any = true ]]; then
exit 1 Write-PipelineTelemetryError -category 'NativeToolsBootstrap' "Native tools install directory does not exist, installation failed"
exit 1
fi
fi fi
exit 0 exit 0

28
eng/common/internal-feed-operations.ps1

@ -6,9 +6,8 @@ param(
[switch] $IsFeedPrivate [switch] $IsFeedPrivate
) )
$ErrorActionPreference = "Stop" $ErrorActionPreference = 'Stop'
Set-StrictMode -Version 2.0 Set-StrictMode -Version 2.0
. $PSScriptRoot\tools.ps1 . $PSScriptRoot\tools.ps1
# Sets VSS_NUGET_EXTERNAL_FEED_ENDPOINTS based on the "darc-int-*" feeds defined in NuGet.config. This is needed # Sets VSS_NUGET_EXTERNAL_FEED_ENDPOINTS based on the "darc-int-*" feeds defined in NuGet.config. This is needed
@ -21,7 +20,7 @@ function SetupCredProvider {
) )
# Install the Cred Provider NuGet plugin # Install the Cred Provider NuGet plugin
Write-Host "Setting up Cred Provider NuGet plugin in the agent..." Write-Host 'Setting up Cred Provider NuGet plugin in the agent...'
Write-Host "Getting 'installcredprovider.ps1' from 'https://github.com/microsoft/artifacts-credprovider'..." Write-Host "Getting 'installcredprovider.ps1' from 'https://github.com/microsoft/artifacts-credprovider'..."
$url = 'https://raw.githubusercontent.com/microsoft/artifacts-credprovider/master/helpers/installcredprovider.ps1' $url = 'https://raw.githubusercontent.com/microsoft/artifacts-credprovider/master/helpers/installcredprovider.ps1'
@ -29,18 +28,18 @@ function SetupCredProvider {
Write-Host "Writing the contents of 'installcredprovider.ps1' locally..." Write-Host "Writing the contents of 'installcredprovider.ps1' locally..."
Invoke-WebRequest $url -OutFile installcredprovider.ps1 Invoke-WebRequest $url -OutFile installcredprovider.ps1
Write-Host "Installing plugin..." Write-Host 'Installing plugin...'
.\installcredprovider.ps1 -Force .\installcredprovider.ps1 -Force
Write-Host "Deleting local copy of 'installcredprovider.ps1'..." Write-Host "Deleting local copy of 'installcredprovider.ps1'..."
Remove-Item .\installcredprovider.ps1 Remove-Item .\installcredprovider.ps1
if (-Not("$env:USERPROFILE\.nuget\plugins\netcore")) { if (-Not("$env:USERPROFILE\.nuget\plugins\netcore")) {
Write-Host "CredProvider plugin was not installed correctly!" Write-PipelineTelemetryError -Category 'Arcade' -Message 'CredProvider plugin was not installed correctly!'
ExitWithExitCode 1 ExitWithExitCode 1
} }
else { else {
Write-Host "CredProvider plugin was installed correctly!" Write-Host 'CredProvider plugin was installed correctly!'
} }
# Then, we set the 'VSS_NUGET_EXTERNAL_FEED_ENDPOINTS' environment variable to restore from the stable # Then, we set the 'VSS_NUGET_EXTERNAL_FEED_ENDPOINTS' environment variable to restore from the stable
@ -49,7 +48,7 @@ function SetupCredProvider {
$nugetConfigPath = "$RepoRoot\NuGet.config" $nugetConfigPath = "$RepoRoot\NuGet.config"
if (-Not (Test-Path -Path $nugetConfigPath)) { if (-Not (Test-Path -Path $nugetConfigPath)) {
Write-Host "NuGet.config file not found in repo's root!" Write-PipelineTelemetryError -Category 'Build' -Message 'NuGet.config file not found in repo root!'
ExitWithExitCode 1 ExitWithExitCode 1
} }
@ -81,7 +80,7 @@ function SetupCredProvider {
} }
else else
{ {
Write-Host "No internal endpoints found in NuGet.config" Write-Host 'No internal endpoints found in NuGet.config'
} }
} }
@ -99,7 +98,7 @@ function InstallDotNetSdkAndRestoreArcade {
& $dotnet restore $restoreProjPath & $dotnet restore $restoreProjPath
Write-Host "Arcade SDK restored!" Write-Host 'Arcade SDK restored!'
if (Test-Path -Path $restoreProjPath) { if (Test-Path -Path $restoreProjPath) {
Remove-Item $restoreProjPath Remove-Item $restoreProjPath
@ -113,23 +112,22 @@ function InstallDotNetSdkAndRestoreArcade {
try { try {
Push-Location $PSScriptRoot Push-Location $PSScriptRoot
if ($Operation -like "setup") { if ($Operation -like 'setup') {
SetupCredProvider $AuthToken SetupCredProvider $AuthToken
} }
elseif ($Operation -like "install-restore") { elseif ($Operation -like 'install-restore') {
InstallDotNetSdkAndRestoreArcade InstallDotNetSdkAndRestoreArcade
} }
else { else {
Write-Host "Unknown operation '$Operation'!" Write-PipelineTelemetryError -Category 'Arcade' -Message "Unknown operation '$Operation'!"
ExitWithExitCode 1 ExitWithExitCode 1
} }
} }
catch { catch {
Write-Host $_
Write-Host $_.Exception
Write-Host $_.ScriptStackTrace Write-Host $_.ScriptStackTrace
Write-PipelineTelemetryError -Category 'Arcade' -Message $_
ExitWithExitCode 1 ExitWithExitCode 1
} }
finally { finally {
Pop-Location Pop-Location
} }

4
eng/common/internal-feed-operations.sh

@ -30,7 +30,7 @@ function SetupCredProvider {
rm installcredprovider.sh rm installcredprovider.sh
if [ ! -d "$HOME/.nuget/plugins" ]; then if [ ! -d "$HOME/.nuget/plugins" ]; then
echo "CredProvider plugin was not installed correctly!" Write-PipelineTelemetryError -category 'Build' 'CredProvider plugin was not installed correctly!'
ExitWithExitCode 1 ExitWithExitCode 1
else else
echo "CredProvider plugin was installed correctly!" echo "CredProvider plugin was installed correctly!"
@ -42,7 +42,7 @@ function SetupCredProvider {
local nugetConfigPath="$repo_root/NuGet.config" local nugetConfigPath="$repo_root/NuGet.config"
if [ ! "$nugetConfigPath" ]; then if [ ! "$nugetConfigPath" ]; then
echo "NuGet.config file not found in repo's root!" Write-PipelineTelemetryError -category 'Build' "NuGet.config file not found in repo's root!"
ExitWithExitCode 1 ExitWithExitCode 1
fi fi

5
eng/common/msbuild.ps1

@ -1,6 +1,6 @@
[CmdletBinding(PositionalBinding=$false)] [CmdletBinding(PositionalBinding=$false)]
Param( Param(
[string] $verbosity = "minimal", [string] $verbosity = 'minimal',
[bool] $warnAsError = $true, [bool] $warnAsError = $true,
[bool] $nodeReuse = $true, [bool] $nodeReuse = $true,
[switch] $ci, [switch] $ci,
@ -18,9 +18,8 @@ try {
MSBuild @extraArgs MSBuild @extraArgs
} }
catch { catch {
Write-Host $_
Write-Host $_.Exception
Write-Host $_.ScriptStackTrace Write-Host $_.ScriptStackTrace
Write-PipelineTelemetryError -Category 'Build' -Message $_
ExitWithExitCode 1 ExitWithExitCode 1
} }

10
eng/common/native/common-library.sh

@ -34,7 +34,7 @@ function ExpandZip {
echo "'Force flag enabled, but '$output_directory' exists. Removing directory" echo "'Force flag enabled, but '$output_directory' exists. Removing directory"
rm -rf $output_directory rm -rf $output_directory
if [[ $? != 0 ]]; then if [[ $? != 0 ]]; then
echo Unable to remove '$output_directory'>&2 Write-PipelineTelemetryError -category 'NativeToolsBootstrap' "Unable to remove '$output_directory'"
return 1 return 1
fi fi
fi fi
@ -45,7 +45,7 @@ function ExpandZip {
echo "Extracting archive" echo "Extracting archive"
tar -xf $zip_path -C $output_directory tar -xf $zip_path -C $output_directory
if [[ $? != 0 ]]; then if [[ $? != 0 ]]; then
echo "Unable to extract '$zip_path'" >&2 Write-PipelineTelemetryError -category 'NativeToolsBootstrap' "Unable to extract '$zip_path'"
return 1 return 1
fi fi
@ -117,7 +117,7 @@ function DownloadAndExtract {
# Download file # Download file
GetFile "$uri" "$temp_tool_path" $force $download_retries $retry_wait_time_seconds GetFile "$uri" "$temp_tool_path" $force $download_retries $retry_wait_time_seconds
if [[ $? != 0 ]]; then if [[ $? != 0 ]]; then
echo "Failed to download '$uri' to '$temp_tool_path'." >&2 Write-PipelineTelemetryError -category 'NativeToolsBootstrap' "Failed to download '$uri' to '$temp_tool_path'."
return 1 return 1
fi fi
@ -125,7 +125,7 @@ function DownloadAndExtract {
echo "extracting from $temp_tool_path to $installDir" echo "extracting from $temp_tool_path to $installDir"
ExpandZip "$temp_tool_path" "$installDir" $force $download_retries $retry_wait_time_seconds ExpandZip "$temp_tool_path" "$installDir" $force $download_retries $retry_wait_time_seconds
if [[ $? != 0 ]]; then if [[ $? != 0 ]]; then
echo "Failed to extract '$temp_tool_path' to '$installDir'." >&2 Write-PipelineTelemetryError -category 'NativeToolsBootstrap' "Failed to extract '$temp_tool_path' to '$installDir'."
return 1 return 1
fi fi
@ -148,7 +148,7 @@ function NewScriptShim {
fi fi
if [[ ! -f $tool_file_path ]]; then if [[ ! -f $tool_file_path ]]; then
echo "Specified tool file path:'$tool_file_path' does not exist" >&2 Write-PipelineTelemetryError -category 'NativeToolsBootstrap' "Specified tool file path:'$tool_file_path' does not exist"
return 1 return 1
fi fi

4
eng/common/native/install-cmake-test.sh

@ -101,7 +101,7 @@ fi
DownloadAndExtract $uri $tool_install_directory $force $download_retries $retry_wait_time_seconds DownloadAndExtract $uri $tool_install_directory $force $download_retries $retry_wait_time_seconds
if [[ $? != 0 ]]; then if [[ $? != 0 ]]; then
echo "Installation failed" >&2 Write-PipelineTelemetryError -category 'NativeToolsBootstrap' 'Installation failed'
exit 1 exit 1
fi fi
@ -110,7 +110,7 @@ fi
NewScriptShim $shim_path $tool_file_path true NewScriptShim $shim_path $tool_file_path true
if [[ $? != 0 ]]; then if [[ $? != 0 ]]; then
echo "Shim generation failed" >&2 Write-PipelineTelemetryError -category 'NativeToolsBootstrap' 'Shim generation failed'
exit 1 exit 1
fi fi

4
eng/common/native/install-cmake.sh

@ -101,7 +101,7 @@ fi
DownloadAndExtract $uri $tool_install_directory $force $download_retries $retry_wait_time_seconds DownloadAndExtract $uri $tool_install_directory $force $download_retries $retry_wait_time_seconds
if [[ $? != 0 ]]; then if [[ $? != 0 ]]; then
echo "Installation failed" >&2 Write-PipelineTelemetryError -category 'NativeToolsBootstrap' 'Installation failed'
exit 1 exit 1
fi fi
@ -110,7 +110,7 @@ fi
NewScriptShim $shim_path $tool_file_path true NewScriptShim $shim_path $tool_file_path true
if [[ $? != 0 ]]; then if [[ $? != 0 ]]; then
echo "Shim generation failed" >&2 Write-PipelineTelemetryError -category 'NativeToolsBootstrap' 'Shim generation failed'
exit 1 exit 1
fi fi

12
eng/common/native/install-tool.ps1

@ -46,6 +46,8 @@ Param (
[int] $RetryWaitTimeInSeconds = 30 [int] $RetryWaitTimeInSeconds = 30
) )
. $PSScriptRoot\..\pipeline-logging-functions.ps1
# Import common library modules # Import common library modules
Import-Module -Name (Join-Path $CommonLibraryDirectory "CommonLibrary.psm1") Import-Module -Name (Join-Path $CommonLibraryDirectory "CommonLibrary.psm1")
@ -93,7 +95,7 @@ try {
-Verbose:$Verbose -Verbose:$Verbose
if ($InstallStatus -Eq $False) { if ($InstallStatus -Eq $False) {
Write-Error "Installation failed" Write-PipelineTelemetryError "Installation failed" -Category "NativeToolsetBootstrapping"
exit 1 exit 1
} }
} }
@ -103,7 +105,7 @@ try {
Write-Error "There are multiple copies of $ToolName in $($ToolInstallDirectory): `n$(@($ToolFilePath | out-string))" Write-Error "There are multiple copies of $ToolName in $($ToolInstallDirectory): `n$(@($ToolFilePath | out-string))"
exit 1 exit 1
} elseif (@($ToolFilePath).Length -Lt 1) { } elseif (@($ToolFilePath).Length -Lt 1) {
Write-Error "$ToolName was not found in $ToolFilePath." Write-Host "$ToolName was not found in $ToolFilePath."
exit 1 exit 1
} }
@ -117,14 +119,14 @@ try {
-Verbose:$Verbose -Verbose:$Verbose
if ($GenerateShimStatus -Eq $False) { if ($GenerateShimStatus -Eq $False) {
Write-Error "Generate shim failed" Write-PipelineTelemetryError "Generate shim failed" -Category "NativeToolsetBootstrapping"
return 1 return 1
} }
exit 0 exit 0
} }
catch { catch {
Write-Host $_ Write-Host $_.ScriptStackTrace
Write-Host $_.Exception Write-PipelineTelemetryError -Category "NativeToolsetBootstrapping" -Message $_
exit 1 exit 1
} }

21
eng/common/performance/perfhelixpublish.proj

@ -6,7 +6,7 @@
<Python>py -3</Python> <Python>py -3</Python>
<CoreRun>%HELIX_CORRELATION_PAYLOAD%\Core_Root\CoreRun.exe</CoreRun> <CoreRun>%HELIX_CORRELATION_PAYLOAD%\Core_Root\CoreRun.exe</CoreRun>
<BaselineCoreRun>%HELIX_CORRELATION_PAYLOAD%\Baseline_Core_Root\CoreRun.exe</BaselineCoreRun> <BaselineCoreRun>%HELIX_CORRELATION_PAYLOAD%\Baseline_Core_Root\CoreRun.exe</BaselineCoreRun>
<HelixPreCommands>$(HelixPreCommands);call %HELIX_CORRELATION_PAYLOAD%\performance\tools\machine-setup.cmd</HelixPreCommands> <HelixPreCommands>$(HelixPreCommands);call %HELIX_CORRELATION_PAYLOAD%\performance\tools\machine-setup.cmd;set PYTHONPATH=%HELIX_WORKITEM_PAYLOAD%\scripts%3B%HELIX_WORKITEM_PAYLOAD%</HelixPreCommands>
<ArtifactsDirectory>%HELIX_CORRELATION_PAYLOAD%\artifacts\BenchmarkDotNet.Artifacts</ArtifactsDirectory> <ArtifactsDirectory>%HELIX_CORRELATION_PAYLOAD%\artifacts\BenchmarkDotNet.Artifacts</ArtifactsDirectory>
<BaselineArtifactsDirectory>%HELIX_CORRELATION_PAYLOAD%\artifacts\BenchmarkDotNet.Artifacts_Baseline</BaselineArtifactsDirectory> <BaselineArtifactsDirectory>%HELIX_CORRELATION_PAYLOAD%\artifacts\BenchmarkDotNet.Artifacts_Baseline</BaselineArtifactsDirectory>
<ResultsComparer>%HELIX_CORRELATION_PAYLOAD%\performance\src\tools\ResultsComparer\ResultsComparer.csproj</ResultsComparer> <ResultsComparer>%HELIX_CORRELATION_PAYLOAD%\performance\src\tools\ResultsComparer\ResultsComparer.csproj</ResultsComparer>
@ -99,4 +99,23 @@
<Timeout>4:00</Timeout> <Timeout>4:00</Timeout>
</HelixWorkItem> </HelixWorkItem>
</ItemGroup> </ItemGroup>
<ItemGroup Condition="'$(AGENT_OS)' == 'Windows_NT'">
<HelixWorkItem Include="Crossgen System.Private.Xml.dll">
<PayloadDirectory>$(WorkItemDirectory)\ScenarioCorrelation</PayloadDirectory>
<Command>$(Python) %HELIX_CORRELATION_PAYLOAD%\performance\src\scenarios\crossgen\test.py crossgen --test-name System.Private.Xml.dll --core-root %HELIX_CORRELATION_PAYLOAD%\Core_Root</Command>
</HelixWorkItem>
<HelixWorkItem Include="Crossgen System.Linq.Expressions.dll">
<PayloadDirectory>$(WorkItemDirectory)\ScenarioCorrelation</PayloadDirectory>
<Command>$(Python) %HELIX_CORRELATION_PAYLOAD%\performance\src\scenarios\crossgen\test.py crossgen --test-name System.Linq.Expressions.dll --core-root %HELIX_CORRELATION_PAYLOAD%\Core_Root</Command>
</HelixWorkItem>
<HelixWorkItem Include="Crossgen Microsoft.CodeAnalysis.VisualBasic.dll">
<PayloadDirectory>$(WorkItemDirectory)\ScenarioCorrelation</PayloadDirectory>
<Command>$(Python) %HELIX_CORRELATION_PAYLOAD%\performance\src\scenarios\crossgen\test.py crossgen --test-name Microsoft.CodeAnalysis.VisualBasic.dll --core-root %HELIX_CORRELATION_PAYLOAD%\Core_Root</Command>
</HelixWorkItem>
<HelixWorkItem Include="Crossgen Microsoft.CodeAnalysis.CSharp.dll">
<PayloadDirectory>$(WorkItemDirectory)\ScenarioCorrelation</PayloadDirectory>
<Command>$(Python) %HELIX_CORRELATION_PAYLOAD%\performance\src\scenarios\crossgen\test.py crossgen --test-name Microsoft.CodeAnalysis.CSharp.dll --core-root %HELIX_CORRELATION_PAYLOAD%\Core_Root</Command>
</HelixWorkItem>
</ItemGroup>
</Project> </Project>

7
eng/common/performance/performance-setup.ps1

@ -9,12 +9,12 @@ Param(
[string] $Branch=$env:BUILD_SOURCEBRANCH, [string] $Branch=$env:BUILD_SOURCEBRANCH,
[string] $CommitSha=$env:BUILD_SOURCEVERSION, [string] $CommitSha=$env:BUILD_SOURCEVERSION,
[string] $BuildNumber=$env:BUILD_BUILDNUMBER, [string] $BuildNumber=$env:BUILD_BUILDNUMBER,
[string] $RunCategories="coreclr corefx", [string] $RunCategories="Libraries Runtime",
[string] $Csproj="src\benchmarks\micro\MicroBenchmarks.csproj", [string] $Csproj="src\benchmarks\micro\MicroBenchmarks.csproj",
[string] $Kind="micro", [string] $Kind="micro",
[switch] $Internal, [switch] $Internal,
[switch] $Compare, [switch] $Compare,
[string] $Configurations="CompilationMode=$CompilationMode" [string] $Configurations="CompilationMode=$CompilationMode RunKind=$Kind"
) )
$RunFromPerformanceRepo = ($Repository -eq "dotnet/performance") -or ($Repository -eq "dotnet-performance") $RunFromPerformanceRepo = ($Repository -eq "dotnet/performance") -or ($Repository -eq "dotnet-performance")
@ -49,7 +49,8 @@ if ($Internal) {
$HelixSourcePrefix = "official" $HelixSourcePrefix = "official"
} }
$CommonSetupArguments="--frameworks $Framework --queue $Queue --build-number $BuildNumber --build-configs $Configurations" # FIX ME: This is a workaround until we get this from the actual pipeline
$CommonSetupArguments="--channel master --queue $Queue --build-number $BuildNumber --build-configs $Configurations --architecture $Architecture"
$SetupArguments = "--repository https://github.com/$Repository --branch $Branch --get-perf-hash --commit-sha $CommitSha $CommonSetupArguments" $SetupArguments = "--repository https://github.com/$Repository --branch $Branch --get-perf-hash --commit-sha $CommitSha $CommonSetupArguments"
if ($RunFromPerformanceRepo) { if ($RunFromPerformanceRepo) {

6
eng/common/performance/performance-setup.sh

@ -13,9 +13,9 @@ build_number=$BUILD_BUILDNUMBER
internal=false internal=false
compare=false compare=false
kind="micro" kind="micro"
run_categories="coreclr corefx" run_categories="Libraries Runtime"
csproj="src\benchmarks\micro\MicroBenchmarks.csproj" csproj="src\benchmarks\micro\MicroBenchmarks.csproj"
configurations= configurations="CompliationMode=$compilation_mode RunKind=$kind"
run_from_perf_repo=false run_from_perf_repo=false
use_core_run=true use_core_run=true
use_baseline_core_run=true use_baseline_core_run=true
@ -164,7 +164,7 @@ if [[ "$internal" == true ]]; then
fi fi
fi fi
common_setup_arguments="--frameworks $framework --queue $queue --build-number $build_number --build-configs $configurations" common_setup_arguments="--channel master --queue $queue --build-number $build_number --build-configs $configurations --architecture $architecture"
setup_arguments="--repository https://github.com/$repository --branch $branch --get-perf-hash --commit-sha $commit_sha $common_setup_arguments" setup_arguments="--repository https://github.com/$repository --branch $branch --get-perf-hash --commit-sha $commit_sha $common_setup_arguments"
if [[ "$run_from_perf_repo" = true ]]; then if [[ "$run_from_perf_repo" = true ]]; then

70
eng/common/pipeline-logging-functions.ps1

@ -12,6 +12,7 @@ $script:loggingCommandEscapeMappings = @( # TODO: WHAT ABOUT "="? WHAT ABOUT "%"
# TODO: BUG: Escape % ??? # TODO: BUG: Escape % ???
# TODO: Add test to verify don't need to escape "=". # TODO: Add test to verify don't need to escape "=".
# Specify "-Force" to force pipeline formatted output even if "$ci" is false or not set
function Write-PipelineTelemetryError { function Write-PipelineTelemetryError {
[CmdletBinding()] [CmdletBinding()]
param( param(
@ -25,49 +26,55 @@ function Write-PipelineTelemetryError {
[string]$SourcePath, [string]$SourcePath,
[string]$LineNumber, [string]$LineNumber,
[string]$ColumnNumber, [string]$ColumnNumber,
[switch]$AsOutput) [switch]$AsOutput,
[switch]$Force)
$PSBoundParameters.Remove("Category") | Out-Null
$Message = "(NETCORE_ENGINEERING_TELEMETRY=$Category) $Message" $PSBoundParameters.Remove('Category') | Out-Null
$PSBoundParameters.Remove("Message") | Out-Null
$PSBoundParameters.Add("Message", $Message)
if($Force -Or ((Test-Path variable:ci) -And $ci)) {
$Message = "(NETCORE_ENGINEERING_TELEMETRY=$Category) $Message"
}
$PSBoundParameters.Remove('Message') | Out-Null
$PSBoundParameters.Add('Message', $Message)
Write-PipelineTaskError @PSBoundParameters Write-PipelineTaskError @PSBoundParameters
} }
# Specify "-Force" to force pipeline formatted output even if "$ci" is false or not set
function Write-PipelineTaskError { function Write-PipelineTaskError {
[CmdletBinding()] [CmdletBinding()]
param( param(
[Parameter(Mandatory = $true)] [Parameter(Mandatory = $true)]
[string]$Message, [string]$Message,
[Parameter(Mandatory = $false)] [Parameter(Mandatory = $false)]
[string]$Type = 'error', [string]$Type = 'error',
[string]$ErrCode, [string]$ErrCode,
[string]$SourcePath, [string]$SourcePath,
[string]$LineNumber, [string]$LineNumber,
[string]$ColumnNumber, [string]$ColumnNumber,
[switch]$AsOutput) [switch]$AsOutput,
[switch]$Force
if(!$ci) { )
if(!$Force -And (-Not (Test-Path variable:ci) -Or !$ci)) {
if($Type -eq 'error') { if($Type -eq 'error') {
Write-Host $Message -ForegroundColor Red Write-Host $Message -ForegroundColor Red
return return
} }
elseif ($Type -eq 'warning') { elseif ($Type -eq 'warning') {
Write-Host $Message -ForegroundColor Yellow Write-Host $Message -ForegroundColor Yellow
return return
} }
} }
if(($Type -ne 'error') -and ($Type -ne 'warning')) { if(($Type -ne 'error') -and ($Type -ne 'warning')) {
Write-Host $Message Write-Host $Message
return return
} }
if(-not $PSBoundParameters.ContainsKey('Type')) { $PSBoundParameters.Remove('Force') | Out-Null
if(-not $PSBoundParameters.ContainsKey('Type')) {
$PSBoundParameters.Add('Type', 'error') $PSBoundParameters.Add('Type', 'error')
} }
Write-LogIssue @PSBoundParameters Write-LogIssue @PSBoundParameters
} }
function Write-PipelineSetVariable { function Write-PipelineSetVariable {
@ -80,7 +87,7 @@ function Write-PipelineTaskError {
[switch]$AsOutput, [switch]$AsOutput,
[bool]$IsMultiJobVariable=$true) [bool]$IsMultiJobVariable=$true)
if($ci) { if((Test-Path variable:ci) -And $ci) {
Write-LoggingCommand -Area 'task' -Event 'setvariable' -Data $Value -Properties @{ Write-LoggingCommand -Area 'task' -Event 'setvariable' -Data $Value -Properties @{
'variable' = $Name 'variable' = $Name
'isSecret' = $Secret 'isSecret' = $Secret
@ -95,7 +102,8 @@ function Write-PipelineTaskError {
[Parameter(Mandatory=$true)] [Parameter(Mandatory=$true)]
[string]$Path, [string]$Path,
[switch]$AsOutput) [switch]$AsOutput)
if($ci) {
if((Test-Path variable:ci) -And $ci) {
Write-LoggingCommand -Area 'task' -Event 'prependpath' -Data $Path -AsOutput:$AsOutput Write-LoggingCommand -Area 'task' -Event 'prependpath' -Data $Path -AsOutput:$AsOutput
} }
} }
@ -231,4 +239,4 @@ function Write-LogIssue {
} }
Write-Host $command -ForegroundColor $foregroundColor -BackgroundColor $backgroundColor Write-Host $command -ForegroundColor $foregroundColor -BackgroundColor $backgroundColor
} }

11
eng/common/pipeline-logging-functions.sh

@ -2,6 +2,7 @@
function Write-PipelineTelemetryError { function Write-PipelineTelemetryError {
local telemetry_category='' local telemetry_category=''
local force=false
local function_args=() local function_args=()
local message='' local message=''
while [[ $# -gt 0 ]]; do while [[ $# -gt 0 ]]; do
@ -11,6 +12,9 @@ function Write-PipelineTelemetryError {
telemetry_category=$2 telemetry_category=$2
shift shift
;; ;;
-force|-f)
force=true
;;
-*) -*)
function_args+=("$1 $2") function_args+=("$1 $2")
shift shift
@ -22,19 +26,22 @@ function Write-PipelineTelemetryError {
shift shift
done done
if [[ "$ci" != true ]]; then if [[ $force != true ]] && [[ "$ci" != true ]]; then
echo "$message" >&2 echo "$message" >&2
return return
fi fi
message="(NETCORE_ENGINEERING_TELEMETRY=$telemetry_category) $message" message="(NETCORE_ENGINEERING_TELEMETRY=$telemetry_category) $message"
function_args+=("$message") function_args+=("$message")
if [[ $force == true ]]; then
function_args+=("-force")
fi
Write-PipelineTaskError $function_args Write-PipelineTaskError $function_args
} }
function Write-PipelineTaskError { function Write-PipelineTaskError {
if [[ "$ci" != true ]]; then if [[ $force != true ]] && [[ "$ci" != true ]]; then
echo "$@" >&2 echo "$@" >&2
return return
fi fi

20
eng/common/post-build/promote-build.ps1 → eng/common/post-build/add-build-to-channel.ps1

@ -2,26 +2,26 @@ param(
[Parameter(Mandatory=$true)][int] $BuildId, [Parameter(Mandatory=$true)][int] $BuildId,
[Parameter(Mandatory=$true)][int] $ChannelId, [Parameter(Mandatory=$true)][int] $ChannelId,
[Parameter(Mandatory=$true)][string] $MaestroApiAccessToken, [Parameter(Mandatory=$true)][string] $MaestroApiAccessToken,
[Parameter(Mandatory=$false)][string] $MaestroApiEndPoint = "https://maestro-prod.westus2.cloudapp.azure.com", [Parameter(Mandatory=$false)][string] $MaestroApiEndPoint = 'https://maestro-prod.westus2.cloudapp.azure.com',
[Parameter(Mandatory=$false)][string] $MaestroApiVersion = "2019-01-16" [Parameter(Mandatory=$false)][string] $MaestroApiVersion = '2019-01-16'
) )
. $PSScriptRoot\post-build-utils.ps1
try { try {
. $PSScriptRoot\post-build-utils.ps1
# Check that the channel we are going to promote the build to exist # Check that the channel we are going to promote the build to exist
$channelInfo = Get-MaestroChannel -ChannelId $ChannelId $channelInfo = Get-MaestroChannel -ChannelId $ChannelId
if (!$channelInfo) { if (!$channelInfo) {
Write-Host "Channel with BAR ID $ChannelId was not found in BAR!" Write-PipelineTelemetryCategory -Category 'PromoteBuild' -Message "Channel with BAR ID $ChannelId was not found in BAR!"
ExitWithExitCode 1 ExitWithExitCode 1
} }
# Get info about which channels the build has already been promoted to # Get info about which channel(s) the build has already been promoted to
$buildInfo = Get-MaestroBuild -BuildId $BuildId $buildInfo = Get-MaestroBuild -BuildId $BuildId
if (!$buildInfo) { if (!$buildInfo) {
Write-Host "Build with BAR ID $BuildId was not found in BAR!" Write-PipelineTelemetryError -Category 'PromoteBuild' -Message "Build with BAR ID $BuildId was not found in BAR!"
ExitWithExitCode 1 ExitWithExitCode 1
} }
@ -39,10 +39,10 @@ try {
Assign-BuildToChannel -BuildId $BuildId -ChannelId $ChannelId Assign-BuildToChannel -BuildId $BuildId -ChannelId $ChannelId
Write-Host "done." Write-Host 'done.'
} }
catch { catch {
Write-Host "There was an error while trying to promote build '$BuildId' to channel '$ChannelId'"
Write-Host $_ Write-Host $_
Write-Host $_.ScriptStackTrace Write-PipelineTelemetryError -Category 'PromoteBuild' -Message "There was an error while trying to promote build '$BuildId' to channel '$ChannelId'"
ExitWithExitCode 1
} }

25
eng/common/post-build/check-channel-consistency.ps1

@ -0,0 +1,25 @@
param(
[Parameter(Mandatory=$true)][string] $PromoteToChannels, # List of channels that the build should be promoted to
[Parameter(Mandatory=$true)][array] $AvailableChannelIds # List of channel IDs available in the YAML implementation
)
try {
. $PSScriptRoot\post-build-utils.ps1
# Check that every channel that Maestro told to promote the build to
# is available in YAML
$PromoteToChannelsIds = $PromoteToChannels -split "\D" | Where-Object { $_ }
foreach ($id in $PromoteToChannelsIds) {
if (($id -ne 0) -and ($id -notin $AvailableChannelIds)) {
Write-PipelineTaskError -Type 'warning' -Message "Channel $id is not present in the post-build YAML configuration!"
}
}
Write-Host 'done.'
}
catch {
Write-Host $_
Write-PipelineTelemetryError -Category 'CheckChannelConsistency' -Message "There was an error while trying to check consistency of Maestro default channels for the build and post-build YAML configuration."
ExitWithExitCode 1
}

45
eng/common/post-build/darc-gather-drop.ps1

@ -1,45 +0,0 @@
param(
[Parameter(Mandatory=$true)][int] $BarBuildId, # ID of the build which assets should be downloaded
[Parameter(Mandatory=$true)][string] $DropLocation, # Where the assets should be downloaded to
[Parameter(Mandatory=$true)][string] $MaestroApiAccessToken, # Token used to access Maestro API
[Parameter(Mandatory=$false)][string] $MaestroApiEndPoint = "https://maestro-prod.westus2.cloudapp.azure.com", # Maestro API URL
[Parameter(Mandatory=$false)][string] $MaestroApiVersion = "2019-01-16" # Version of Maestro API to use
)
. $PSScriptRoot\post-build-utils.ps1
try {
Write-Host "Installing DARC ..."
. $PSScriptRoot\..\darc-init.ps1
$exitCode = $LASTEXITCODE
if ($exitCode -ne 0) {
Write-PipelineTaskError "Something failed while running 'darc-init.ps1'. Check for errors above. Exiting now..."
ExitWithExitCode $exitCode
}
# For now, only use a dry run.
# Ideally we would change darc to enable a quick request that
# would check whether the file exists that you can download it,
# and that it won't conflict with other files.
# https://github.com/dotnet/arcade/issues/3674
# Right now we can't remove continue-on-error because we ocassionally will have
# dependencies that have no associated builds (e.g. an old dependency).
# We need to add an option to baseline specific dependencies away, or add them manually
# to the BAR.
darc gather-drop --non-shipping `
--dry-run `
--continue-on-error `
--id $BarBuildId `
--output-dir $DropLocation `
--bar-uri $MaestroApiEndpoint `
--password $MaestroApiAccessToken `
--latest-location
}
catch {
Write-Host $_
Write-Host $_.Exception
Write-Host $_.ScriptStackTrace
ExitWithExitCode 1
}

11
eng/common/post-build/nuget-validation.ps1

@ -6,20 +6,19 @@ param(
[Parameter(Mandatory=$true)][string] $ToolDestinationPath # Where the validation tool should be downloaded to [Parameter(Mandatory=$true)][string] $ToolDestinationPath # Where the validation tool should be downloaded to
) )
. $PSScriptRoot\post-build-utils.ps1
try { try {
$url = "https://raw.githubusercontent.com/NuGet/NuGetGallery/jver-verify/src/VerifyMicrosoftPackage/verify.ps1" . $PSScriptRoot\post-build-utils.ps1
$url = 'https://raw.githubusercontent.com/NuGet/NuGetGallery/3e25ad135146676bcab0050a516939d9958bfa5d/src/VerifyMicrosoftPackage/verify.ps1'
New-Item -ItemType "directory" -Path ${ToolDestinationPath} -Force New-Item -ItemType 'directory' -Path ${ToolDestinationPath} -Force
Invoke-WebRequest $url -OutFile ${ToolDestinationPath}\verify.ps1 Invoke-WebRequest $url -OutFile ${ToolDestinationPath}\verify.ps1
& ${ToolDestinationPath}\verify.ps1 ${PackagesPath}\*.nupkg & ${ToolDestinationPath}\verify.ps1 ${PackagesPath}\*.nupkg
} }
catch { catch {
Write-PipelineTaskError "NuGet package validation failed. Please check error logs."
Write-Host $_
Write-Host $_.ScriptStackTrace Write-Host $_.ScriptStackTrace
Write-PipelineTelemetryError -Category 'NuGetValidation' -Message $_
ExitWithExitCode 1 ExitWithExitCode 1
} }

27
eng/common/post-build/post-build-utils.ps1

@ -1,16 +1,17 @@
# Most of the functions in this file require the variables `MaestroApiEndPoint`, # Most of the functions in this file require the variables `MaestroApiEndPoint`,
# `MaestroApiVersion` and `MaestroApiAccessToken` to be globally available. # `MaestroApiVersion` and `MaestroApiAccessToken` to be globally available.
$ErrorActionPreference = "Stop" $ErrorActionPreference = 'Stop'
Set-StrictMode -Version 2.0 Set-StrictMode -Version 2.0
# `tools.ps1` checks $ci to perform some actions. Since the post-build # `tools.ps1` checks $ci to perform some actions. Since the post-build
# scripts don't necessarily execute in the same agent that run the # scripts don't necessarily execute in the same agent that run the
# build.ps1/sh script this variable isn't automatically set. # build.ps1/sh script this variable isn't automatically set.
$ci = $true $ci = $true
$disableConfigureToolsetImport = $true
. $PSScriptRoot\..\tools.ps1 . $PSScriptRoot\..\tools.ps1
function Create-MaestroApiRequestHeaders([string]$ContentType = "application/json") { function Create-MaestroApiRequestHeaders([string]$ContentType = 'application/json') {
Validate-MaestroVars Validate-MaestroVars
$headers = New-Object 'System.Collections.Generic.Dictionary[[String],[String]]' $headers = New-Object 'System.Collections.Generic.Dictionary[[String],[String]]'
@ -50,20 +51,20 @@ function Get-MaestroSubscriptions([string]$SourceRepository, [int]$ChannelId) {
return $result return $result
} }
function Trigger-Subscription([string]$SubscriptionId) { function Assign-BuildToChannel([int]$BuildId, [int]$ChannelId) {
Validate-MaestroVars Validate-MaestroVars
$apiHeaders = Create-MaestroApiRequestHeaders -AuthToken $MaestroApiAccessToken $apiHeaders = Create-MaestroApiRequestHeaders -AuthToken $MaestroApiAccessToken
$apiEndpoint = "$MaestroApiEndPoint/api/subscriptions/$SubscriptionId/trigger?api-version=$MaestroApiVersion" $apiEndpoint = "$MaestroApiEndPoint/api/channels/${ChannelId}/builds/${BuildId}?api-version=$MaestroApiVersion"
Invoke-WebRequest -Uri $apiEndpoint -Headers $apiHeaders -Method Post | Out-Null Invoke-WebRequest -Method Post -Uri $apiEndpoint -Headers $apiHeaders | Out-Null
} }
function Assign-BuildToChannel([int]$BuildId, [int]$ChannelId) { function Trigger-Subscription([string]$SubscriptionId) {
Validate-MaestroVars Validate-MaestroVars
$apiHeaders = Create-MaestroApiRequestHeaders -AuthToken $MaestroApiAccessToken $apiHeaders = Create-MaestroApiRequestHeaders -AuthToken $MaestroApiAccessToken
$apiEndpoint = "$MaestroApiEndPoint/api/channels/${ChannelId}/builds/${BuildId}?api-version=$MaestroApiVersion" $apiEndpoint = "$MaestroApiEndPoint/api/subscriptions/$SubscriptionId/trigger?api-version=$MaestroApiVersion"
Invoke-WebRequest -Method Post -Uri $apiEndpoint -Headers $apiHeaders | Out-Null Invoke-WebRequest -Uri $apiEndpoint -Headers $apiHeaders -Method Post | Out-Null
} }
function Validate-MaestroVars { function Validate-MaestroVars {
@ -72,18 +73,18 @@ function Validate-MaestroVars {
Get-Variable MaestroApiVersion -Scope Global | Out-Null Get-Variable MaestroApiVersion -Scope Global | Out-Null
Get-Variable MaestroApiAccessToken -Scope Global | Out-Null Get-Variable MaestroApiAccessToken -Scope Global | Out-Null
if (!($MaestroApiEndPoint -Match "^http[s]?://maestro-(int|prod).westus2.cloudapp.azure.com$")) { if (!($MaestroApiEndPoint -Match '^http[s]?://maestro-(int|prod).westus2.cloudapp.azure.com$')) {
Write-PipelineTaskError "MaestroApiEndPoint is not a valid Maestro URL. '$MaestroApiEndPoint'" Write-PipelineTelemetryError -Category 'MaestroVars' -Message "MaestroApiEndPoint is not a valid Maestro URL. '$MaestroApiEndPoint'"
ExitWithExitCode 1 ExitWithExitCode 1
} }
if (!($MaestroApiVersion -Match "^[0-9]{4}-[0-9]{2}-[0-9]{2}$")) { if (!($MaestroApiVersion -Match '^[0-9]{4}-[0-9]{2}-[0-9]{2}$')) {
Write-PipelineTaskError "MaestroApiVersion does not match a version string in the format yyyy-MM-DD. '$MaestroApiVersion'" Write-PipelineTelemetryError -Category 'MaestroVars' -Message "MaestroApiVersion does not match a version string in the format yyyy-MM-DD. '$MaestroApiVersion'"
ExitWithExitCode 1 ExitWithExitCode 1
} }
} }
catch { catch {
Write-PipelineTaskError "Error: Variables `MaestroApiEndPoint`, `MaestroApiVersion` and `MaestroApiAccessToken` are required while using this script." Write-PipelineTelemetryError -Category 'MaestroVars' -Message 'Error: Variables `MaestroApiEndPoint`, `MaestroApiVersion` and `MaestroApiAccessToken` are required while using this script.'
Write-Host $_ Write-Host $_
ExitWithExitCode 1 ExitWithExitCode 1
} }

26
eng/common/post-build/setup-maestro-vars.ps1

@ -1,26 +0,0 @@
param(
[Parameter(Mandatory=$true)][string] $ReleaseConfigsPath # Full path to ReleaseConfigs.txt asset
)
. $PSScriptRoot\post-build-utils.ps1
try {
$Content = Get-Content $ReleaseConfigsPath
$BarId = $Content | Select -Index 0
$Channels = ""
$Content | Select -Index 1 | ForEach-Object { $Channels += "$_ ," }
$IsStableBuild = $Content | Select -Index 2
Write-PipelineSetVariable -Name 'BARBuildId' -Value $BarId
Write-PipelineSetVariable -Name 'InitialChannels' -Value "$Channels"
Write-PipelineSetVariable -Name 'IsStableBuild' -Value $IsStableBuild
}
catch {
Write-Host $_
Write-Host $_.Exception
Write-Host $_.ScriptStackTrace
ExitWithExitCode 1
}

42
eng/common/post-build/sourcelink-validation.ps1

@ -34,9 +34,9 @@ $ValidatePackage = {
# Extensions for which we'll look for SourceLink information # Extensions for which we'll look for SourceLink information
# For now we'll only care about Portable & Embedded PDBs # For now we'll only care about Portable & Embedded PDBs
$RelevantExtensions = @(".dll", ".exe", ".pdb") $RelevantExtensions = @('.dll', '.exe', '.pdb')
Write-Host -NoNewLine "Validating" ([System.IO.Path]::GetFileName($PackagePath)) "... " Write-Host -NoNewLine 'Validating ' ([System.IO.Path]::GetFileName($PackagePath)) '...'
$PackageId = [System.IO.Path]::GetFileNameWithoutExtension($PackagePath) $PackageId = [System.IO.Path]::GetFileNameWithoutExtension($PackagePath)
$ExtractPath = Join-Path -Path $using:ExtractPath -ChildPath $PackageId $ExtractPath = Join-Path -Path $using:ExtractPath -ChildPath $PackageId
@ -58,7 +58,7 @@ $ValidatePackage = {
$TargetFile = Join-Path -Path $ExtractPath -ChildPath $FakeName $TargetFile = Join-Path -Path $ExtractPath -ChildPath $FakeName
# We ignore resource DLLs # We ignore resource DLLs
if ($FileName.EndsWith(".resources.dll")) { if ($FileName.EndsWith('.resources.dll')) {
return return
} }
@ -96,7 +96,7 @@ $ValidatePackage = {
$Uri = $Link -as [System.URI] $Uri = $Link -as [System.URI]
# Only GitHub links are valid # Only GitHub links are valid
if ($Uri.AbsoluteURI -ne $null -and ($Uri.Host -match "github" -or $Uri.Host -match "githubusercontent")) { if ($Uri.AbsoluteURI -ne $null -and ($Uri.Host -match 'github' -or $Uri.Host -match 'githubusercontent')) {
$Status = (Invoke-WebRequest -Uri $Link -UseBasicParsing -Method HEAD -TimeoutSec 5).StatusCode $Status = (Invoke-WebRequest -Uri $Link -UseBasicParsing -Method HEAD -TimeoutSec 5).StatusCode
} }
else { else {
@ -143,19 +143,19 @@ $ValidatePackage = {
} }
if ($FailedFiles -eq 0) { if ($FailedFiles -eq 0) {
Write-Host "Passed." Write-Host 'Passed.'
return 0 return 0
} }
else { else {
Write-Host "$PackagePath has broken SourceLink links." Write-PipelineTelemetryError -Category 'SourceLink' -Message "$PackagePath has broken SourceLink links."
return 1 return 1
} }
} }
function ValidateSourceLinkLinks { function ValidateSourceLinkLinks {
if ($GHRepoName -ne "" -and !($GHRepoName -Match "^[^\s\/]+/[^\s\/]+$")) { if ($GHRepoName -ne '' -and !($GHRepoName -Match '^[^\s\/]+/[^\s\/]+$')) {
if (!($GHRepoName -Match "^[^\s-]+-[^\s]+$")) { if (!($GHRepoName -Match '^[^\s-]+-[^\s]+$')) {
Write-PipelineTaskError "GHRepoName should be in the format <org>/<repo> or <org>-<repo>. '$GHRepoName'" Write-PipelineTelemetryError -Category 'SourceLink' -Message "GHRepoName should be in the format <org>/<repo> or <org>-<repo>. '$GHRepoName'"
ExitWithExitCode 1 ExitWithExitCode 1
} }
else { else {
@ -163,14 +163,14 @@ function ValidateSourceLinkLinks {
} }
} }
if ($GHCommit -ne "" -and !($GHCommit -Match "^[0-9a-fA-F]{40}$")) { if ($GHCommit -ne '' -and !($GHCommit -Match '^[0-9a-fA-F]{40}$')) {
Write-PipelineTaskError "GHCommit should be a 40 chars hexadecimal string. '$GHCommit'" Write-PipelineTelemetryError -Category 'SourceLink' -Message "GHCommit should be a 40 chars hexadecimal string. '$GHCommit'"
ExitWithExitCode 1 ExitWithExitCode 1
} }
if ($GHRepoName -ne "" -and $GHCommit -ne "") { if ($GHRepoName -ne '' -and $GHCommit -ne '') {
$RepoTreeURL = -Join("http://api.github.com/repos/", $GHRepoName, "/git/trees/", $GHCommit, "?recursive=1") $RepoTreeURL = -Join('http://api.github.com/repos/', $GHRepoName, '/git/trees/', $GHCommit, '?recursive=1')
$CodeExtensions = @(".cs", ".vb", ".fs", ".fsi", ".fsx", ".fsscript") $CodeExtensions = @('.cs', '.vb', '.fs', '.fsi', '.fsx', '.fsscript')
try { try {
# Retrieve the list of files in the repo at that particular commit point and store them in the RepoFiles hash # Retrieve the list of files in the repo at that particular commit point and store them in the RepoFiles hash
@ -188,8 +188,8 @@ function ValidateSourceLinkLinks {
Write-Host "Problems downloading the list of files from the repo. Url used: $RepoTreeURL . Execution will proceed without caching." Write-Host "Problems downloading the list of files from the repo. Url used: $RepoTreeURL . Execution will proceed without caching."
} }
} }
elseif ($GHRepoName -ne "" -or $GHCommit -ne "") { elseif ($GHRepoName -ne '' -or $GHCommit -ne '') {
Write-Host "For using the http caching mechanism both GHRepoName and GHCommit should be informed." Write-Host 'For using the http caching mechanism both GHRepoName and GHCommit should be informed.'
} }
if (Test-Path $ExtractPath) { if (Test-Path $ExtractPath) {
@ -217,18 +217,18 @@ function ValidateSourceLinkLinks {
$ValidationFailures = 0 $ValidationFailures = 0
foreach ($Job in @(Get-Job)) { foreach ($Job in @(Get-Job)) {
$jobResult = Wait-Job -Id $Job.Id | Receive-Job $jobResult = Wait-Job -Id $Job.Id | Receive-Job
if ($jobResult -ne "0") { if ($jobResult -ne '0') {
$ValidationFailures++ $ValidationFailures++
} }
} }
if ($ValidationFailures -gt 0) { if ($ValidationFailures -gt 0) {
Write-PipelineTaskError " $ValidationFailures package(s) failed validation." Write-PipelineTelemetryError -Category 'SourceLink' -Message "$ValidationFailures package(s) failed validation."
ExitWithExitCode 1 ExitWithExitCode 1
} }
} }
function InstallSourcelinkCli { function InstallSourcelinkCli {
$sourcelinkCliPackageName = "sourcelink" $sourcelinkCliPackageName = 'sourcelink'
$dotnetRoot = InitializeDotNetCli -install:$true $dotnetRoot = InitializeDotNetCli -install:$true
$dotnet = "$dotnetRoot\dotnet.exe" $dotnet = "$dotnetRoot\dotnet.exe"
@ -239,7 +239,7 @@ function InstallSourcelinkCli {
} }
else { else {
Write-Host "Installing SourceLink CLI version $sourcelinkCliVersion..." Write-Host "Installing SourceLink CLI version $sourcelinkCliVersion..."
Write-Host "You may need to restart your command window if this is the first dotnet tool you have installed." Write-Host 'You may need to restart your command window if this is the first dotnet tool you have installed.'
& "$dotnet" tool install $sourcelinkCliPackageName --version $sourcelinkCliVersion --verbosity "minimal" --global & "$dotnet" tool install $sourcelinkCliPackageName --version $sourcelinkCliVersion --verbosity "minimal" --global
} }
} }
@ -250,8 +250,8 @@ try {
ValidateSourceLinkLinks ValidateSourceLinkLinks
} }
catch { catch {
Write-Host $_
Write-Host $_.Exception Write-Host $_.Exception
Write-Host $_.ScriptStackTrace Write-Host $_.ScriptStackTrace
Write-PipelineTelemetryError -Category 'SourceLink' -Message $_
ExitWithExitCode 1 ExitWithExitCode 1
} }

71
eng/common/post-build/symbols-validation.ps1

@ -4,10 +4,6 @@ param(
[Parameter(Mandatory=$true)][string] $DotnetSymbolVersion # Version of dotnet symbol to use [Parameter(Mandatory=$true)][string] $DotnetSymbolVersion # Version of dotnet symbol to use
) )
. $PSScriptRoot\post-build-utils.ps1
Add-Type -AssemblyName System.IO.Compression.FileSystem
function FirstMatchingSymbolDescriptionOrDefault { function FirstMatchingSymbolDescriptionOrDefault {
param( param(
[string] $FullPath, # Full path to the module that has to be checked [string] $FullPath, # Full path to the module that has to be checked
@ -23,19 +19,19 @@ function FirstMatchingSymbolDescriptionOrDefault {
# checking and which type of file was uploaded. # checking and which type of file was uploaded.
# The file itself is returned # The file itself is returned
$SymbolPath = $SymbolsPath + "\" + $FileName $SymbolPath = $SymbolsPath + '\' + $FileName
# PDB file for the module # PDB file for the module
$PdbPath = $SymbolPath.Replace($Extension, ".pdb") $PdbPath = $SymbolPath.Replace($Extension, '.pdb')
# PDB file for R2R module (created by crossgen) # PDB file for R2R module (created by crossgen)
$NGenPdb = $SymbolPath.Replace($Extension, ".ni.pdb") $NGenPdb = $SymbolPath.Replace($Extension, '.ni.pdb')
# DBG file for a .so library # DBG file for a .so library
$SODbg = $SymbolPath.Replace($Extension, ".so.dbg") $SODbg = $SymbolPath.Replace($Extension, '.so.dbg')
# DWARF file for a .dylib # DWARF file for a .dylib
$DylibDwarf = $SymbolPath.Replace($Extension, ".dylib.dwarf") $DylibDwarf = $SymbolPath.Replace($Extension, '.dylib.dwarf')
$dotnetSymbolExe = "$env:USERPROFILE\.dotnet\tools" $dotnetSymbolExe = "$env:USERPROFILE\.dotnet\tools"
$dotnetSymbolExe = Resolve-Path "$dotnetSymbolExe\dotnet-symbol.exe" $dotnetSymbolExe = Resolve-Path "$dotnetSymbolExe\dotnet-symbol.exe"
@ -43,19 +39,19 @@ function FirstMatchingSymbolDescriptionOrDefault {
& $dotnetSymbolExe --symbols --modules --windows-pdbs $TargetServerParam $FullPath -o $SymbolsPath | Out-Null & $dotnetSymbolExe --symbols --modules --windows-pdbs $TargetServerParam $FullPath -o $SymbolsPath | Out-Null
if (Test-Path $PdbPath) { if (Test-Path $PdbPath) {
return "PDB" return 'PDB'
} }
elseif (Test-Path $NGenPdb) { elseif (Test-Path $NGenPdb) {
return "NGen PDB" return 'NGen PDB'
} }
elseif (Test-Path $SODbg) { elseif (Test-Path $SODbg) {
return "DBG for SO" return 'DBG for SO'
} }
elseif (Test-Path $DylibDwarf) { elseif (Test-Path $DylibDwarf) {
return "Dwarf for Dylib" return 'Dwarf for Dylib'
} }
elseif (Test-Path $SymbolPath) { elseif (Test-Path $SymbolPath) {
return "Module" return 'Module'
} }
else { else {
return $null return $null
@ -74,7 +70,7 @@ function CountMissingSymbols {
} }
# Extensions for which we'll look for symbols # Extensions for which we'll look for symbols
$RelevantExtensions = @(".dll", ".exe", ".so", ".dylib") $RelevantExtensions = @('.dll', '.exe', '.so', '.dylib')
# How many files are missing symbol information # How many files are missing symbol information
$MissingSymbols = 0 $MissingSymbols = 0
@ -82,38 +78,38 @@ function CountMissingSymbols {
$PackageId = [System.IO.Path]::GetFileNameWithoutExtension($PackagePath) $PackageId = [System.IO.Path]::GetFileNameWithoutExtension($PackagePath)
$PackageGuid = New-Guid $PackageGuid = New-Guid
$ExtractPath = Join-Path -Path $ExtractPath -ChildPath $PackageGuid $ExtractPath = Join-Path -Path $ExtractPath -ChildPath $PackageGuid
$SymbolsPath = Join-Path -Path $ExtractPath -ChildPath "Symbols" $SymbolsPath = Join-Path -Path $ExtractPath -ChildPath 'Symbols'
[System.IO.Compression.ZipFile]::ExtractToDirectory($PackagePath, $ExtractPath) [System.IO.Compression.ZipFile]::ExtractToDirectory($PackagePath, $ExtractPath)
Get-ChildItem -Recurse $ExtractPath | Get-ChildItem -Recurse $ExtractPath |
Where-Object {$RelevantExtensions -contains $_.Extension} | Where-Object {$RelevantExtensions -contains $_.Extension} |
ForEach-Object { ForEach-Object {
if ($_.FullName -Match "\\ref\\") { if ($_.FullName -Match '\\ref\\') {
Write-Host "`t Ignoring reference assembly file" $_.FullName Write-Host "`t Ignoring reference assembly file " $_.FullName
return return
} }
$SymbolsOnMSDL = FirstMatchingSymbolDescriptionOrDefault $_.FullName "--microsoft-symbol-server" $SymbolsPath $SymbolsOnMSDL = FirstMatchingSymbolDescriptionOrDefault $_.FullName '--microsoft-symbol-server' $SymbolsPath
$SymbolsOnSymWeb = FirstMatchingSymbolDescriptionOrDefault $_.FullName "--internal-server" $SymbolsPath $SymbolsOnSymWeb = FirstMatchingSymbolDescriptionOrDefault $_.FullName '--internal-server' $SymbolsPath
Write-Host -NoNewLine "`t Checking file" $_.FullName "... " Write-Host -NoNewLine "`t Checking file " $_.FullName "... "
if ($SymbolsOnMSDL -ne $null -and $SymbolsOnSymWeb -ne $null) { if ($SymbolsOnMSDL -ne $null -and $SymbolsOnSymWeb -ne $null) {
Write-Host "Symbols found on MSDL (" $SymbolsOnMSDL ") and SymWeb (" $SymbolsOnSymWeb ")" Write-Host "Symbols found on MSDL ($SymbolsOnMSDL) and SymWeb ($SymbolsOnSymWeb)"
} }
else { else {
$MissingSymbols++ $MissingSymbols++
if ($SymbolsOnMSDL -eq $null -and $SymbolsOnSymWeb -eq $null) { if ($SymbolsOnMSDL -eq $null -and $SymbolsOnSymWeb -eq $null) {
Write-Host "No symbols found on MSDL or SymWeb!" Write-Host 'No symbols found on MSDL or SymWeb!'
} }
else { else {
if ($SymbolsOnMSDL -eq $null) { if ($SymbolsOnMSDL -eq $null) {
Write-Host "No symbols found on MSDL!" Write-Host 'No symbols found on MSDL!'
} }
else { else {
Write-Host "No symbols found on SymWeb!" Write-Host 'No symbols found on SymWeb!'
} }
} }
} }
@ -132,27 +128,27 @@ function CheckSymbolsAvailable {
Get-ChildItem "$InputPath\*.nupkg" | Get-ChildItem "$InputPath\*.nupkg" |
ForEach-Object { ForEach-Object {
$FileName = $_.Name $FileName = $_.Name
# These packages from Arcade-Services include some native libraries that # These packages from Arcade-Services include some native libraries that
# our current symbol uploader can't handle. Below is a workaround until # our current symbol uploader can't handle. Below is a workaround until
# we get issue: https://github.com/dotnet/arcade/issues/2457 sorted. # we get issue: https://github.com/dotnet/arcade/issues/2457 sorted.
if ($FileName -Match "Microsoft\.DotNet\.Darc\.") { if ($FileName -Match 'Microsoft\.DotNet\.Darc\.') {
Write-Host "Ignoring Arcade-services file: $FileName" Write-Host "Ignoring Arcade-services file: $FileName"
Write-Host Write-Host
return return
} }
elseif ($FileName -Match "Microsoft\.DotNet\.Maestro\.Tasks\.") { elseif ($FileName -Match 'Microsoft\.DotNet\.Maestro\.Tasks\.') {
Write-Host "Ignoring Arcade-services file: $FileName" Write-Host "Ignoring Arcade-services file: $FileName"
Write-Host Write-Host
return return
} }
Write-Host "Validating $FileName " Write-Host "Validating $FileName "
$Status = CountMissingSymbols "$InputPath\$FileName" $Status = CountMissingSymbols "$InputPath\$FileName"
if ($Status -ne 0) { if ($Status -ne 0) {
Write-PipelineTaskError "Missing symbols for $Status modules in the package $FileName" Write-PipelineTelemetryError -Category 'CheckSymbols' -Message "Missing symbols for $Status modules in the package $FileName"
ExitWithExitCode $exitCode ExitWithExitCode $exitCode
} }
Write-Host Write-Host
@ -160,7 +156,7 @@ function CheckSymbolsAvailable {
} }
function InstallDotnetSymbol { function InstallDotnetSymbol {
$dotnetSymbolPackageName = "dotnet-symbol" $dotnetSymbolPackageName = 'dotnet-symbol'
$dotnetRoot = InitializeDotNetCli -install:$true $dotnetRoot = InitializeDotNetCli -install:$true
$dotnet = "$dotnetRoot\dotnet.exe" $dotnet = "$dotnetRoot\dotnet.exe"
@ -171,19 +167,22 @@ function InstallDotnetSymbol {
} }
else { else {
Write-Host "Installing dotnet-symbol version $dotnetSymbolVersion..." Write-Host "Installing dotnet-symbol version $dotnetSymbolVersion..."
Write-Host "You may need to restart your command window if this is the first dotnet tool you have installed." Write-Host 'You may need to restart your command window if this is the first dotnet tool you have installed.'
& "$dotnet" tool install $dotnetSymbolPackageName --version $dotnetSymbolVersion --verbosity "minimal" --global & "$dotnet" tool install $dotnetSymbolPackageName --version $dotnetSymbolVersion --verbosity "minimal" --global
} }
} }
try { try {
. $PSScriptRoot\post-build-utils.ps1
Add-Type -AssemblyName System.IO.Compression.FileSystem
InstallDotnetSymbol InstallDotnetSymbol
CheckSymbolsAvailable CheckSymbolsAvailable
} }
catch { catch {
Write-Host $_
Write-Host $_.Exception
Write-Host $_.ScriptStackTrace Write-Host $_.ScriptStackTrace
Write-PipelineTelemetryError -Category 'CheckSymbols' -Message $_
ExitWithExitCode 1 ExitWithExitCode 1
} }

87
eng/common/post-build/trigger-subscriptions.ps1

@ -2,56 +2,63 @@ param(
[Parameter(Mandatory=$true)][string] $SourceRepo, [Parameter(Mandatory=$true)][string] $SourceRepo,
[Parameter(Mandatory=$true)][int] $ChannelId, [Parameter(Mandatory=$true)][int] $ChannelId,
[Parameter(Mandatory=$true)][string] $MaestroApiAccessToken, [Parameter(Mandatory=$true)][string] $MaestroApiAccessToken,
[Parameter(Mandatory=$false)][string] $MaestroApiEndPoint = "https://maestro-prod.westus2.cloudapp.azure.com", [Parameter(Mandatory=$false)][string] $MaestroApiEndPoint = 'https://maestro-prod.westus2.cloudapp.azure.com',
[Parameter(Mandatory=$false)][string] $MaestroApiVersion = "2019-01-16" [Parameter(Mandatory=$false)][string] $MaestroApiVersion = '2019-01-16'
) )
. $PSScriptRoot\post-build-utils.ps1 try {
. $PSScriptRoot\post-build-utils.ps1
# Get all the $SourceRepo subscriptions # Get all the $SourceRepo subscriptions
$normalizedSourceRepo = $SourceRepo.Replace('dnceng@', '') $normalizedSourceRepo = $SourceRepo.Replace('dnceng@', '')
$subscriptions = Get-MaestroSubscriptions -SourceRepository $normalizedSourceRepo -ChannelId $ChannelId $subscriptions = Get-MaestroSubscriptions -SourceRepository $normalizedSourceRepo -ChannelId $ChannelId
if (!$subscriptions) { if (!$subscriptions) {
Write-Host "No subscriptions found for source repo '$normalizedSourceRepo' in channel '$ChannelId'" Write-PipelineTelemetryError -Category 'TriggerSubscriptions' -Message "No subscriptions found for source repo '$normalizedSourceRepo' in channel '$ChannelId'"
ExitWithExitCode 0 ExitWithExitCode 0
} }
$subscriptionsToTrigger = New-Object System.Collections.Generic.List[string] $subscriptionsToTrigger = New-Object System.Collections.Generic.List[string]
$failedTriggeredSubscription = $false $failedTriggeredSubscription = $false
# Get all enabled subscriptions that need dependency flow on 'everyBuild' # Get all enabled subscriptions that need dependency flow on 'everyBuild'
foreach ($subscription in $subscriptions) { foreach ($subscription in $subscriptions) {
if ($subscription.enabled -and $subscription.policy.updateFrequency -like 'everyBuild' -and $subscription.channel.id -eq $ChannelId) { if ($subscription.enabled -and $subscription.policy.updateFrequency -like 'everyBuild' -and $subscription.channel.id -eq $ChannelId) {
Write-Host "Should trigger this subscription: $subscription.id" Write-Host "Should trigger this subscription: ${$subscription.id}"
[void]$subscriptionsToTrigger.Add($subscription.id) [void]$subscriptionsToTrigger.Add($subscription.id)
}
} }
}
foreach ($subscriptionToTrigger in $subscriptionsToTrigger) { foreach ($subscriptionToTrigger in $subscriptionsToTrigger) {
try { try {
Write-Host "Triggering subscription '$subscriptionToTrigger'." Write-Host "Triggering subscription '$subscriptionToTrigger'."
Trigger-Subscription -SubscriptionId $subscriptionToTrigger Trigger-Subscription -SubscriptionId $subscriptionToTrigger
Write-Host "done." Write-Host 'done.'
} }
catch catch
{ {
Write-Host "There was an error while triggering subscription '$subscriptionToTrigger'" Write-Host "There was an error while triggering subscription '$subscriptionToTrigger'"
Write-Host $_ Write-Host $_
Write-Host $_.ScriptStackTrace Write-Host $_.ScriptStackTrace
$failedTriggeredSubscription = $true $failedTriggeredSubscription = $true
}
} }
}
if ($subscriptionsToTrigger.Count -eq 0) { if ($subscriptionsToTrigger.Count -eq 0) {
Write-Host "No subscription matched source repo '$normalizedSourceRepo' and channel ID '$ChannelId'." Write-Host "No subscription matched source repo '$normalizedSourceRepo' and channel ID '$ChannelId'."
}
elseif ($failedTriggeredSubscription) {
Write-PipelineTelemetryError -Category 'TriggerSubscriptions' -Message 'At least one subscription failed to be triggered...'
ExitWithExitCode 1
}
else {
Write-Host 'All subscriptions were triggered successfully!'
}
} }
elseif ($failedTriggeredSubscription) { catch {
Write-Host "At least one subscription failed to be triggered..." Write-Host $_.ScriptStackTrace
Write-PipelineTelemetryError -Category 'TriggerSubscriptions' -Message $_
ExitWithExitCode 1 ExitWithExitCode 1
} }
else {
Write-Host "All subscriptions were triggered successfully!"
}

19
eng/common/sdk-task.ps1

@ -1,8 +1,8 @@
[CmdletBinding(PositionalBinding=$false)] [CmdletBinding(PositionalBinding=$false)]
Param( Param(
[string] $configuration = "Debug", [string] $configuration = 'Debug',
[string] $task, [string] $task,
[string] $verbosity = "minimal", [string] $verbosity = 'minimal',
[string] $msbuildEngine = $null, [string] $msbuildEngine = $null,
[switch] $restore, [switch] $restore,
[switch] $prepareMachine, [switch] $prepareMachine,
@ -32,7 +32,7 @@ function Print-Usage() {
} }
function Build([string]$target) { function Build([string]$target) {
$logSuffix = if ($target -eq "Execute") { "" } else { ".$target" } $logSuffix = if ($target -eq 'Execute') { '' } else { ".$target" }
$log = Join-Path $LogDir "$task$logSuffix.binlog" $log = Join-Path $LogDir "$task$logSuffix.binlog"
$outputPath = Join-Path $ToolsetDir "$task\\" $outputPath = Join-Path $ToolsetDir "$task\\"
@ -46,33 +46,32 @@ function Build([string]$target) {
} }
try { try {
if ($help -or (($null -ne $properties) -and ($properties.Contains("/help") -or $properties.Contains("/?")))) { if ($help -or (($null -ne $properties) -and ($properties.Contains('/help') -or $properties.Contains('/?')))) {
Print-Usage Print-Usage
exit 0 exit 0
} }
if ($task -eq "") { if ($task -eq "") {
Write-Host "Missing required parameter '-task <value>'" -ForegroundColor Red Write-PipelineTelemetryError -Category 'Build' -Message "Missing required parameter '-task <value>'" -ForegroundColor Red
Print-Usage Print-Usage
ExitWithExitCode 1 ExitWithExitCode 1
} }
$taskProject = GetSdkTaskProject $task $taskProject = GetSdkTaskProject $task
if (!(Test-Path $taskProject)) { if (!(Test-Path $taskProject)) {
Write-Host "Unknown task: $task" -ForegroundColor Red Write-PipelineTelemetryError -Category 'Build' -Message "Unknown task: $task" -ForegroundColor Red
ExitWithExitCode 1 ExitWithExitCode 1
} }
if ($restore) { if ($restore) {
Build "Restore" Build 'Restore'
} }
Build "Execute" Build 'Execute'
} }
catch { catch {
Write-Host $_
Write-Host $_.Exception
Write-Host $_.ScriptStackTrace Write-Host $_.ScriptStackTrace
Write-PipelineTelemetryError -Category 'Build' -Message $_
ExitWithExitCode 1 ExitWithExitCode 1
} }

180
eng/common/sdl/execute-all-sdl-tools.ps1

@ -1,100 +1,114 @@
Param( Param(
[string] $GuardianPackageName, # Required: the name of guardian CLI package (not needed if GuardianCliLocation is specified) [string] $GuardianPackageName, # Required: the name of guardian CLI package (not needed if GuardianCliLocation is specified)
[string] $NugetPackageDirectory, # Required: directory where NuGet packages are installed (not needed if GuardianCliLocation is specified) [string] $NugetPackageDirectory, # Required: directory where NuGet packages are installed (not needed if GuardianCliLocation is specified)
[string] $GuardianCliLocation, # Optional: Direct location of Guardian CLI executable if GuardianPackageName & NugetPackageDirectory are not specified [string] $GuardianCliLocation, # Optional: Direct location of Guardian CLI executable if GuardianPackageName & NugetPackageDirectory are not specified
[string] $Repository=$env:BUILD_REPOSITORY_NAME, # Required: the name of the repository (e.g. dotnet/arcade) [string] $Repository=$env:BUILD_REPOSITORY_NAME, # Required: the name of the repository (e.g. dotnet/arcade)
[string] $BranchName=$env:BUILD_SOURCEBRANCH, # Optional: name of branch or version of gdn settings; defaults to master [string] $BranchName=$env:BUILD_SOURCEBRANCH, # Optional: name of branch or version of gdn settings; defaults to master
[string] $SourceDirectory=$env:BUILD_SOURCESDIRECTORY, # Required: the directory where source files are located [string] $SourceDirectory=$env:BUILD_SOURCESDIRECTORY, # Required: the directory where source files are located
[string] $ArtifactsDirectory = (Join-Path $env:BUILD_SOURCESDIRECTORY ("artifacts")), # Required: the directory where build artifacts are located [string] $ArtifactsDirectory = (Join-Path $env:BUILD_ARTIFACTSTAGINGDIRECTORY ('artifacts')), # Required: the directory where build artifacts are located
[string] $AzureDevOpsAccessToken, # Required: access token for dnceng; should be provided via KeyVault [string] $AzureDevOpsAccessToken, # Required: access token for dnceng; should be provided via KeyVault
[string[]] $SourceToolsList, # Optional: list of SDL tools to run on source code [string[]] $SourceToolsList, # Optional: list of SDL tools to run on source code
[string[]] $ArtifactToolsList, # Optional: list of SDL tools to run on built artifacts [string[]] $ArtifactToolsList, # Optional: list of SDL tools to run on built artifacts
[bool] $TsaPublish=$False, # Optional: true will publish results to TSA; only set to true after onboarding to TSA; TSA is the automated framework used to upload test results as bugs. [bool] $TsaPublish=$False, # Optional: true will publish results to TSA; only set to true after onboarding to TSA; TSA is the automated framework used to upload test results as bugs.
[string] $TsaBranchName=$env:BUILD_SOURCEBRANCH, # Optional: required for TSA publish; defaults to $(Build.SourceBranchName); TSA is the automated framework used to upload test results as bugs. [string] $TsaBranchName=$env:BUILD_SOURCEBRANCH, # Optional: required for TSA publish; defaults to $(Build.SourceBranchName); TSA is the automated framework used to upload test results as bugs.
[string] $TsaRepositoryName=$env:BUILD_REPOSITORY_NAME, # Optional: TSA repository name; will be generated automatically if not submitted; TSA is the automated framework used to upload test results as bugs. [string] $TsaRepositoryName=$env:BUILD_REPOSITORY_NAME, # Optional: TSA repository name; will be generated automatically if not submitted; TSA is the automated framework used to upload test results as bugs.
[string] $BuildNumber=$env:BUILD_BUILDNUMBER, # Optional: required for TSA publish; defaults to $(Build.BuildNumber) [string] $BuildNumber=$env:BUILD_BUILDNUMBER, # Optional: required for TSA publish; defaults to $(Build.BuildNumber)
[bool] $UpdateBaseline=$False, # Optional: if true, will update the baseline in the repository; should only be run after fixing any issues which need to be fixed [bool] $UpdateBaseline=$False, # Optional: if true, will update the baseline in the repository; should only be run after fixing any issues which need to be fixed
[bool] $TsaOnboard=$False, # Optional: if true, will onboard the repository to TSA; should only be run once; TSA is the automated framework used to upload test results as bugs. [bool] $TsaOnboard=$False, # Optional: if true, will onboard the repository to TSA; should only be run once; TSA is the automated framework used to upload test results as bugs.
[string] $TsaInstanceUrl, # Optional: only needed if TsaOnboard or TsaPublish is true; the instance-url registered with TSA; TSA is the automated framework used to upload test results as bugs. [string] $TsaInstanceUrl, # Optional: only needed if TsaOnboard or TsaPublish is true; the instance-url registered with TSA; TSA is the automated framework used to upload test results as bugs.
[string] $TsaCodebaseName, # Optional: only needed if TsaOnboard or TsaPublish is true; the name of the codebase registered with TSA; TSA is the automated framework used to upload test results as bugs. [string] $TsaCodebaseName, # Optional: only needed if TsaOnboard or TsaPublish is true; the name of the codebase registered with TSA; TSA is the automated framework used to upload test results as bugs.
[string] $TsaProjectName, # Optional: only needed if TsaOnboard or TsaPublish is true; the name of the project registered with TSA; TSA is the automated framework used to upload test results as bugs. [string] $TsaProjectName, # Optional: only needed if TsaOnboard or TsaPublish is true; the name of the project registered with TSA; TSA is the automated framework used to upload test results as bugs.
[string] $TsaNotificationEmail, # Optional: only needed if TsaOnboard is true; the email(s) which will receive notifications of TSA bug filings (e.g. alias@microsoft.com); TSA is the automated framework used to upload test results as bugs. [string] $TsaNotificationEmail, # Optional: only needed if TsaOnboard is true; the email(s) which will receive notifications of TSA bug filings (e.g. alias@microsoft.com); TSA is the automated framework used to upload test results as bugs.
[string] $TsaCodebaseAdmin, # Optional: only needed if TsaOnboard is true; the aliases which are admins of the TSA codebase (e.g. DOMAIN\alias); TSA is the automated framework used to upload test results as bugs. [string] $TsaCodebaseAdmin, # Optional: only needed if TsaOnboard is true; the aliases which are admins of the TSA codebase (e.g. DOMAIN\alias); TSA is the automated framework used to upload test results as bugs.
[string] $TsaBugAreaPath, # Optional: only needed if TsaOnboard is true; the area path where TSA will file bugs in AzDO; TSA is the automated framework used to upload test results as bugs. [string] $TsaBugAreaPath, # Optional: only needed if TsaOnboard is true; the area path where TSA will file bugs in AzDO; TSA is the automated framework used to upload test results as bugs.
[string] $TsaIterationPath, # Optional: only needed if TsaOnboard is true; the iteration path where TSA will file bugs in AzDO; TSA is the automated framework used to upload test results as bugs. [string] $TsaIterationPath, # Optional: only needed if TsaOnboard is true; the iteration path where TSA will file bugs in AzDO; TSA is the automated framework used to upload test results as bugs.
[string] $GuardianLoggerLevel="Standard", # Optional: the logger level for the Guardian CLI; options are Trace, Verbose, Standard, Warning, and Error [string] $GuardianLoggerLevel='Standard', # Optional: the logger level for the Guardian CLI; options are Trace, Verbose, Standard, Warning, and Error
[string[]] $CrScanAdditionalRunConfigParams, # Optional: Additional Params to custom build a CredScan run config in the format @("xyz:abc","sdf:1") [string[]] $CrScanAdditionalRunConfigParams, # Optional: Additional Params to custom build a CredScan run config in the format @("xyz:abc","sdf:1")
[string[]] $PoliCheckAdditionalRunConfigParams # Optional: Additional Params to custom build a Policheck run config in the format @("xyz:abc","sdf:1") [string[]] $PoliCheckAdditionalRunConfigParams # Optional: Additional Params to custom build a Policheck run config in the format @("xyz:abc","sdf:1")
) )
$ErrorActionPreference = "Stop" try {
Set-StrictMode -Version 2.0 $ErrorActionPreference = 'Stop'
$LASTEXITCODE = 0 Set-StrictMode -Version 2.0
$disableConfigureToolsetImport = $true
$LASTEXITCODE = 0
#Replace repo names to the format of org/repo # `tools.ps1` checks $ci to perform some actions. Since the SDL
if (!($Repository.contains('/'))) { # scripts don't necessarily execute in the same agent that run the
$RepoName = $Repository -replace '(.*?)-(.*)', '$1/$2'; # build.ps1/sh script this variable isn't automatically set.
} $ci = $true
else{ . $PSScriptRoot\..\tools.ps1
$RepoName = $Repository;
}
if ($GuardianPackageName) { #Replace repo names to the format of org/repo
$guardianCliLocation = Join-Path $NugetPackageDirectory (Join-Path $GuardianPackageName (Join-Path "tools" "guardian.cmd")) if (!($Repository.contains('/'))) {
} else { $RepoName = $Repository -replace '(.*?)-(.*)', '$1/$2';
$guardianCliLocation = $GuardianCliLocation }
} else{
$RepoName = $Repository;
}
if ($GuardianPackageName) {
$guardianCliLocation = Join-Path $NugetPackageDirectory (Join-Path $GuardianPackageName (Join-Path 'tools' 'guardian.cmd'))
} else {
$guardianCliLocation = $GuardianCliLocation
}
$workingDirectory = (Split-Path $SourceDirectory -Parent) $workingDirectory = (Split-Path $SourceDirectory -Parent)
$ValidPath = Test-Path $guardianCliLocation $ValidPath = Test-Path $guardianCliLocation
if ($ValidPath -eq $False) if ($ValidPath -eq $False)
{ {
Write-Host "Invalid Guardian CLI Location." Write-PipelineTelemetryError -Force -Category 'Sdl' -Message 'Invalid Guardian CLI Location.'
exit 1 ExitWithExitCode 1
} }
& $(Join-Path $PSScriptRoot "init-sdl.ps1") -GuardianCliLocation $guardianCliLocation -Repository $RepoName -BranchName $BranchName -WorkingDirectory $workingDirectory -AzureDevOpsAccessToken $AzureDevOpsAccessToken -GuardianLoggerLevel $GuardianLoggerLevel & $(Join-Path $PSScriptRoot 'init-sdl.ps1') -GuardianCliLocation $guardianCliLocation -Repository $RepoName -BranchName $BranchName -WorkingDirectory $workingDirectory -AzureDevOpsAccessToken $AzureDevOpsAccessToken -GuardianLoggerLevel $GuardianLoggerLevel
$gdnFolder = Join-Path $workingDirectory ".gdn" $gdnFolder = Join-Path $workingDirectory '.gdn'
if ($TsaOnboard) { if ($TsaOnboard) {
if ($TsaCodebaseName -and $TsaNotificationEmail -and $TsaCodebaseAdmin -and $TsaBugAreaPath) { if ($TsaCodebaseName -and $TsaNotificationEmail -and $TsaCodebaseAdmin -and $TsaBugAreaPath) {
Write-Host "$guardianCliLocation tsa-onboard --codebase-name `"$TsaCodebaseName`" --notification-alias `"$TsaNotificationEmail`" --codebase-admin `"$TsaCodebaseAdmin`" --instance-url `"$TsaInstanceUrl`" --project-name `"$TsaProjectName`" --area-path `"$TsaBugAreaPath`" --iteration-path `"$TsaIterationPath`" --working-directory $workingDirectory --logger-level $GuardianLoggerLevel" Write-Host "$guardianCliLocation tsa-onboard --codebase-name `"$TsaCodebaseName`" --notification-alias `"$TsaNotificationEmail`" --codebase-admin `"$TsaCodebaseAdmin`" --instance-url `"$TsaInstanceUrl`" --project-name `"$TsaProjectName`" --area-path `"$TsaBugAreaPath`" --iteration-path `"$TsaIterationPath`" --working-directory $workingDirectory --logger-level $GuardianLoggerLevel"
& $guardianCliLocation tsa-onboard --codebase-name "$TsaCodebaseName" --notification-alias "$TsaNotificationEmail" --codebase-admin "$TsaCodebaseAdmin" --instance-url "$TsaInstanceUrl" --project-name "$TsaProjectName" --area-path "$TsaBugAreaPath" --iteration-path "$TsaIterationPath" --working-directory $workingDirectory --logger-level $GuardianLoggerLevel & $guardianCliLocation tsa-onboard --codebase-name "$TsaCodebaseName" --notification-alias "$TsaNotificationEmail" --codebase-admin "$TsaCodebaseAdmin" --instance-url "$TsaInstanceUrl" --project-name "$TsaProjectName" --area-path "$TsaBugAreaPath" --iteration-path "$TsaIterationPath" --working-directory $workingDirectory --logger-level $GuardianLoggerLevel
if ($LASTEXITCODE -ne 0) { if ($LASTEXITCODE -ne 0) {
Write-Host "Guardian tsa-onboard failed with exit code $LASTEXITCODE." Write-PipelineTelemetryError -Force -Category 'Sdl' -Message "Guardian tsa-onboard failed with exit code $LASTEXITCODE."
exit $LASTEXITCODE ExitWithExitCode $LASTEXITCODE
}
} else {
Write-PipelineTelemetryError -Force -Category 'Sdl' -Message 'Could not onboard to TSA -- not all required values ($TsaCodebaseName, $TsaNotificationEmail, $TsaCodebaseAdmin, $TsaBugAreaPath) were specified.'
ExitWithExitCode 1
} }
} else {
Write-Host "Could not onboard to TSA -- not all required values ($$TsaCodebaseName, $$TsaNotificationEmail, $$TsaCodebaseAdmin, $$TsaBugAreaPath) were specified."
exit 1
} }
}
if ($ArtifactToolsList -and $ArtifactToolsList.Count -gt 0) { if ($ArtifactToolsList -and $ArtifactToolsList.Count -gt 0) {
& $(Join-Path $PSScriptRoot "run-sdl.ps1") -GuardianCliLocation $guardianCliLocation -WorkingDirectory $workingDirectory -TargetDirectory $ArtifactsDirectory -GdnFolder $gdnFolder -ToolsList $ArtifactToolsList -AzureDevOpsAccessToken $AzureDevOpsAccessToken -UpdateBaseline $UpdateBaseline -GuardianLoggerLevel $GuardianLoggerLevel -CrScanAdditionalRunConfigParams $CrScanAdditionalRunConfigParams -PoliCheckAdditionalRunConfigParams $PoliCheckAdditionalRunConfigParams & $(Join-Path $PSScriptRoot 'run-sdl.ps1') -GuardianCliLocation $guardianCliLocation -WorkingDirectory $workingDirectory -TargetDirectory $ArtifactsDirectory -GdnFolder $gdnFolder -ToolsList $ArtifactToolsList -AzureDevOpsAccessToken $AzureDevOpsAccessToken -UpdateBaseline $UpdateBaseline -GuardianLoggerLevel $GuardianLoggerLevel -CrScanAdditionalRunConfigParams $CrScanAdditionalRunConfigParams -PoliCheckAdditionalRunConfigParams $PoliCheckAdditionalRunConfigParams
} }
if ($SourceToolsList -and $SourceToolsList.Count -gt 0) { if ($SourceToolsList -and $SourceToolsList.Count -gt 0) {
& $(Join-Path $PSScriptRoot "run-sdl.ps1") -GuardianCliLocation $guardianCliLocation -WorkingDirectory $workingDirectory -TargetDirectory $SourceDirectory -GdnFolder $gdnFolder -ToolsList $SourceToolsList -AzureDevOpsAccessToken $AzureDevOpsAccessToken -UpdateBaseline $UpdateBaseline -GuardianLoggerLevel $GuardianLoggerLevel -CrScanAdditionalRunConfigParams $CrScanAdditionalRunConfigParams -PoliCheckAdditionalRunConfigParams $PoliCheckAdditionalRunConfigParams & $(Join-Path $PSScriptRoot 'run-sdl.ps1') -GuardianCliLocation $guardianCliLocation -WorkingDirectory $workingDirectory -TargetDirectory $SourceDirectory -GdnFolder $gdnFolder -ToolsList $SourceToolsList -AzureDevOpsAccessToken $AzureDevOpsAccessToken -UpdateBaseline $UpdateBaseline -GuardianLoggerLevel $GuardianLoggerLevel -CrScanAdditionalRunConfigParams $CrScanAdditionalRunConfigParams -PoliCheckAdditionalRunConfigParams $PoliCheckAdditionalRunConfigParams
} }
if ($UpdateBaseline) { if ($UpdateBaseline) {
& (Join-Path $PSScriptRoot "push-gdn.ps1") -Repository $RepoName -BranchName $BranchName -GdnFolder $GdnFolder -AzureDevOpsAccessToken $AzureDevOpsAccessToken -PushReason "Update baseline" & (Join-Path $PSScriptRoot 'push-gdn.ps1') -Repository $RepoName -BranchName $BranchName -GdnFolder $GdnFolder -AzureDevOpsAccessToken $AzureDevOpsAccessToken -PushReason 'Update baseline'
} }
if ($TsaPublish) { if ($TsaPublish) {
if ($TsaBranchName -and $BuildNumber) { if ($TsaBranchName -and $BuildNumber) {
if (-not $TsaRepositoryName) { if (-not $TsaRepositoryName) {
$TsaRepositoryName = "$($Repository)-$($BranchName)" $TsaRepositoryName = "$($Repository)-$($BranchName)"
} }
Write-Host "$guardianCliLocation tsa-publish --all-tools --repository-name `"$TsaRepositoryName`" --branch-name `"$TsaBranchName`" --build-number `"$BuildNumber`" --codebase-name `"$TsaCodebaseName`" --notification-alias `"$TsaNotificationEmail`" --codebase-admin `"$TsaCodebaseAdmin`" --instance-url `"$TsaInstanceUrl`" --project-name `"$TsaProjectName`" --area-path `"$TsaBugAreaPath`" --iteration-path `"$TsaIterationPath`" --working-directory $workingDirectory --logger-level $GuardianLoggerLevel" Write-Host "$guardianCliLocation tsa-publish --all-tools --repository-name `"$TsaRepositoryName`" --branch-name `"$TsaBranchName`" --build-number `"$BuildNumber`" --codebase-name `"$TsaCodebaseName`" --notification-alias `"$TsaNotificationEmail`" --codebase-admin `"$TsaCodebaseAdmin`" --instance-url `"$TsaInstanceUrl`" --project-name `"$TsaProjectName`" --area-path `"$TsaBugAreaPath`" --iteration-path `"$TsaIterationPath`" --working-directory $workingDirectory --logger-level $GuardianLoggerLevel"
& $guardianCliLocation tsa-publish --all-tools --repository-name "$TsaRepositoryName" --branch-name "$TsaBranchName" --build-number "$BuildNumber" --onboard $True --codebase-name "$TsaCodebaseName" --notification-alias "$TsaNotificationEmail" --codebase-admin "$TsaCodebaseAdmin" --instance-url "$TsaInstanceUrl" --project-name "$TsaProjectName" --area-path "$TsaBugAreaPath" --iteration-path "$TsaIterationPath" --working-directory $workingDirectory --logger-level $GuardianLoggerLevel & $guardianCliLocation tsa-publish --all-tools --repository-name "$TsaRepositoryName" --branch-name "$TsaBranchName" --build-number "$BuildNumber" --onboard $True --codebase-name "$TsaCodebaseName" --notification-alias "$TsaNotificationEmail" --codebase-admin "$TsaCodebaseAdmin" --instance-url "$TsaInstanceUrl" --project-name "$TsaProjectName" --area-path "$TsaBugAreaPath" --iteration-path "$TsaIterationPath" --working-directory $workingDirectory --logger-level $GuardianLoggerLevel
if ($LASTEXITCODE -ne 0) { if ($LASTEXITCODE -ne 0) {
Write-Host "Guardian tsa-publish failed with exit code $LASTEXITCODE." Write-PipelineTelemetryError -Force -Category 'Sdl' -Message "Guardian tsa-publish failed with exit code $LASTEXITCODE."
exit $LASTEXITCODE ExitWithExitCode $LASTEXITCODE
}
} else {
Write-PipelineTelemetryError -Force -Category 'Sdl' -Message 'Could not publish to TSA -- not all required values ($TsaBranchName, $BuildNumber) were specified.'
ExitWithExitCode 1
} }
} else {
Write-Host "Could not publish to TSA -- not all required values ($$TsaBranchName, $$BuildNumber) were specified."
exit 1
} }
} }
catch {
Write-Host $_.ScriptStackTrace
Write-PipelineTelemetryError -Force -Category 'Sdl' -Message $_
exit 1
}

97
eng/common/sdl/extract-artifact-packages.ps1

@ -3,54 +3,12 @@ param(
[Parameter(Mandatory=$true)][string] $ExtractPath # Full path to directory where the packages will be extracted [Parameter(Mandatory=$true)][string] $ExtractPath # Full path to directory where the packages will be extracted
) )
$ErrorActionPreference = "Stop" $ErrorActionPreference = 'Stop'
Set-StrictMode -Version 2.0 Set-StrictMode -Version 2.0
# `tools.ps1` checks $ci to perform some actions. Since the post-build $disableConfigureToolsetImport = $true
# scripts don't necessarily execute in the same agent that run the
# build.ps1/sh script this variable isn't automatically set.
$ci = $true
. $PSScriptRoot\..\tools.ps1
$ExtractPackage = { function ExtractArtifacts {
param(
[string] $PackagePath # Full path to a NuGet package
)
if (!(Test-Path $PackagePath)) {
Write-PipelineTaskError "Input file does not exist: $PackagePath"
ExitWithExitCode 1
}
$RelevantExtensions = @(".dll", ".exe", ".pdb")
Write-Host -NoNewLine "Extracting" ([System.IO.Path]::GetFileName($PackagePath)) "... "
$PackageId = [System.IO.Path]::GetFileNameWithoutExtension($PackagePath)
$ExtractPath = Join-Path -Path $using:ExtractPath -ChildPath $PackageId
Add-Type -AssemblyName System.IO.Compression.FileSystem
[System.IO.Directory]::CreateDirectory($ExtractPath);
try {
$zip = [System.IO.Compression.ZipFile]::OpenRead($PackagePath)
$zip.Entries |
Where-Object {$RelevantExtensions -contains [System.IO.Path]::GetExtension($_.Name)} |
ForEach-Object {
$TargetFile = Join-Path -Path $ExtractPath -ChildPath $_.Name
[System.IO.Compression.ZipFileExtensions]::ExtractToFile($_, $TargetFile, $true)
}
}
catch {
}
finally {
$zip.Dispose()
}
}
function ExtractArtifacts {
if (!(Test-Path $InputPath)) { if (!(Test-Path $InputPath)) {
Write-Host "Input Path does not exist: $InputPath" Write-Host "Input Path does not exist: $InputPath"
ExitWithExitCode 0 ExitWithExitCode 0
@ -67,11 +25,56 @@ $ExtractPackage = {
} }
try { try {
# `tools.ps1` checks $ci to perform some actions. Since the SDL
# scripts don't necessarily execute in the same agent that run the
# build.ps1/sh script this variable isn't automatically set.
$ci = $true
. $PSScriptRoot\..\tools.ps1
$ExtractPackage = {
param(
[string] $PackagePath # Full path to a NuGet package
)
if (!(Test-Path $PackagePath)) {
Write-PipelineTelemetryError -Category 'Build' -Message "Input file does not exist: $PackagePath"
ExitWithExitCode 1
}
$RelevantExtensions = @('.dll', '.exe', '.pdb')
Write-Host -NoNewLine 'Extracting ' ([System.IO.Path]::GetFileName($PackagePath)) '...'
$PackageId = [System.IO.Path]::GetFileNameWithoutExtension($PackagePath)
$ExtractPath = Join-Path -Path $using:ExtractPath -ChildPath $PackageId
Add-Type -AssemblyName System.IO.Compression.FileSystem
[System.IO.Directory]::CreateDirectory($ExtractPath);
try {
$zip = [System.IO.Compression.ZipFile]::OpenRead($PackagePath)
$zip.Entries |
Where-Object {$RelevantExtensions -contains [System.IO.Path]::GetExtension($_.Name)} |
ForEach-Object {
$TargetFile = Join-Path -Path $ExtractPath -ChildPath $_.Name
[System.IO.Compression.ZipFileExtensions]::ExtractToFile($_, $TargetFile, $true)
}
}
catch {
Write-Host $_.ScriptStackTrace
Write-PipelineTelemetryError -Force -Category 'Sdl' -Message $_
ExitWithExitCode 1
}
finally {
$zip.Dispose()
}
}
Measure-Command { ExtractArtifacts } Measure-Command { ExtractArtifacts }
} }
catch { catch {
Write-Host $_
Write-Host $_.Exception
Write-Host $_.ScriptStackTrace Write-Host $_.ScriptStackTrace
Write-PipelineTelemetryError -Force -Category 'Sdl' -Message $_
ExitWithExitCode 1 ExitWithExitCode 1
} }

42
eng/common/sdl/init-sdl.ps1

@ -1,16 +1,23 @@
Param( Param(
[string] $GuardianCliLocation, [string] $GuardianCliLocation,
[string] $Repository, [string] $Repository,
[string] $BranchName="master", [string] $BranchName='master',
[string] $WorkingDirectory, [string] $WorkingDirectory,
[string] $AzureDevOpsAccessToken, [string] $AzureDevOpsAccessToken,
[string] $GuardianLoggerLevel="Standard" [string] $GuardianLoggerLevel='Standard'
) )
$ErrorActionPreference = "Stop" $ErrorActionPreference = 'Stop'
Set-StrictMode -Version 2.0 Set-StrictMode -Version 2.0
$disableConfigureToolsetImport = $true
$LASTEXITCODE = 0 $LASTEXITCODE = 0
# `tools.ps1` checks $ci to perform some actions. Since the SDL
# scripts don't necessarily execute in the same agent that run the
# build.ps1/sh script this variable isn't automatically set.
$ci = $true
. $PSScriptRoot\..\tools.ps1
# Don't display the console progress UI - it's a huge perf hit # Don't display the console progress UI - it's a huge perf hit
$ProgressPreference = 'SilentlyContinue' $ProgressPreference = 'SilentlyContinue'
@ -21,11 +28,10 @@ $uri = "https://dev.azure.com/dnceng/internal/_apis/git/repositories/sdl-tool-cf
$zipFile = "$WorkingDirectory/gdn.zip" $zipFile = "$WorkingDirectory/gdn.zip"
Add-Type -AssemblyName System.IO.Compression.FileSystem Add-Type -AssemblyName System.IO.Compression.FileSystem
$gdnFolder = (Join-Path $WorkingDirectory ".gdn") $gdnFolder = (Join-Path $WorkingDirectory '.gdn')
Try try {
{
# We try to download the zip; if the request fails (e.g. the file doesn't exist), we catch it and init guardian instead # We try to download the zip; if the request fails (e.g. the file doesn't exist), we catch it and init guardian instead
Write-Host "Downloading gdn folder from internal config repostiory..." Write-Host 'Downloading gdn folder from internal config repostiory...'
Invoke-WebRequest -Headers @{ "Accept"="application/zip"; "Authorization"="Basic $encodedPat" } -Uri $uri -OutFile $zipFile Invoke-WebRequest -Headers @{ "Accept"="application/zip"; "Authorization"="Basic $encodedPat" } -Uri $uri -OutFile $zipFile
if (Test-Path $gdnFolder) { if (Test-Path $gdnFolder) {
# Remove the gdn folder if it exists (it shouldn't unless there's too much caching; this is just in case) # Remove the gdn folder if it exists (it shouldn't unless there's too much caching; this is just in case)
@ -33,19 +39,29 @@ Try
} }
[System.IO.Compression.ZipFile]::ExtractToDirectory($zipFile, $WorkingDirectory) [System.IO.Compression.ZipFile]::ExtractToDirectory($zipFile, $WorkingDirectory)
Write-Host $gdnFolder Write-Host $gdnFolder
} Catch [System.Net.WebException] { ExitWithExitCode 0
} catch [System.Net.WebException] { } # Catch and ignore webexception
try {
# if the folder does not exist, we'll do a guardian init and push it to the remote repository # if the folder does not exist, we'll do a guardian init and push it to the remote repository
Write-Host "Initializing Guardian..." Write-Host 'Initializing Guardian...'
Write-Host "$GuardianCliLocation init --working-directory $WorkingDirectory --logger-level $GuardianLoggerLevel" Write-Host "$GuardianCliLocation init --working-directory $WorkingDirectory --logger-level $GuardianLoggerLevel"
& $GuardianCliLocation init --working-directory $WorkingDirectory --logger-level $GuardianLoggerLevel & $GuardianCliLocation init --working-directory $WorkingDirectory --logger-level $GuardianLoggerLevel
if ($LASTEXITCODE -ne 0) { if ($LASTEXITCODE -ne 0) {
Write-Error "Guardian init failed with exit code $LASTEXITCODE." Write-PipelineTelemetryError -Force -Category 'Build' -Message "Guardian init failed with exit code $LASTEXITCODE."
ExitWithExitCode $LASTEXITCODE
} }
# We create the mainbaseline so it can be edited later # We create the mainbaseline so it can be edited later
Write-Host "$GuardianCliLocation baseline --working-directory $WorkingDirectory --name mainbaseline" Write-Host "$GuardianCliLocation baseline --working-directory $WorkingDirectory --name mainbaseline"
& $GuardianCliLocation baseline --working-directory $WorkingDirectory --name mainbaseline & $GuardianCliLocation baseline --working-directory $WorkingDirectory --name mainbaseline
if ($LASTEXITCODE -ne 0) { if ($LASTEXITCODE -ne 0) {
Write-Error "Guardian baseline failed with exit code $LASTEXITCODE." Write-PipelineTelemetryError -Force -Category 'Build' -Message "Guardian baseline failed with exit code $LASTEXITCODE."
ExitWithExitCode $LASTEXITCODE
} }
& $(Join-Path $PSScriptRoot "push-gdn.ps1") -Repository $Repository -BranchName $BranchName -GdnFolder $gdnFolder -AzureDevOpsAccessToken $AzureDevOpsAccessToken -PushReason "Initialize gdn folder" & $(Join-Path $PSScriptRoot 'push-gdn.ps1') -Repository $Repository -BranchName $BranchName -GdnFolder $gdnFolder -AzureDevOpsAccessToken $AzureDevOpsAccessToken -PushReason 'Initialize gdn folder'
} ExitWithExitCode 0
}
catch {
Write-Host $_.ScriptStackTrace
Write-PipelineTelemetryError -Force -Category 'Sdl' -Message $_
ExitWithExitCode 1
}

94
eng/common/sdl/push-gdn.ps1

@ -1,51 +1,69 @@
Param( Param(
[string] $Repository, [string] $Repository,
[string] $BranchName="master", [string] $BranchName='master',
[string] $GdnFolder, [string] $GdnFolder,
[string] $AzureDevOpsAccessToken, [string] $AzureDevOpsAccessToken,
[string] $PushReason [string] $PushReason
) )
$ErrorActionPreference = "Stop" $ErrorActionPreference = 'Stop'
Set-StrictMode -Version 2.0 Set-StrictMode -Version 2.0
$disableConfigureToolsetImport = $true
$LASTEXITCODE = 0 $LASTEXITCODE = 0
# We create the temp directory where we'll store the sdl-config repository try {
$sdlDir = Join-Path $env:TEMP "sdl" # `tools.ps1` checks $ci to perform some actions. Since the SDL
if (Test-Path $sdlDir) { # scripts don't necessarily execute in the same agent that run the
Remove-Item -Force -Recurse $sdlDir # build.ps1/sh script this variable isn't automatically set.
} $ci = $true
. $PSScriptRoot\..\tools.ps1
Write-Host "git clone https://dnceng:`$AzureDevOpsAccessToken@dev.azure.com/dnceng/internal/_git/sdl-tool-cfg $sdlDir" # We create the temp directory where we'll store the sdl-config repository
git clone https://dnceng:$AzureDevOpsAccessToken@dev.azure.com/dnceng/internal/_git/sdl-tool-cfg $sdlDir $sdlDir = Join-Path $env:TEMP 'sdl'
if ($LASTEXITCODE -ne 0) { if (Test-Path $sdlDir) {
Write-Error "Git clone failed with exit code $LASTEXITCODE." Remove-Item -Force -Recurse $sdlDir
} }
# We copy the .gdn folder from our local run into the git repository so it can be committed
$sdlRepositoryFolder = Join-Path (Join-Path (Join-Path $sdlDir $Repository) $BranchName) ".gdn" Write-Host "git clone https://dnceng:`$AzureDevOpsAccessToken@dev.azure.com/dnceng/internal/_git/sdl-tool-cfg $sdlDir"
if (Get-Command Robocopy) { git clone https://dnceng:$AzureDevOpsAccessToken@dev.azure.com/dnceng/internal/_git/sdl-tool-cfg $sdlDir
Robocopy /S $GdnFolder $sdlRepositoryFolder if ($LASTEXITCODE -ne 0) {
} else { Write-PipelineTelemetryError -Force -Category 'Sdl' -Message "Git clone failed with exit code $LASTEXITCODE."
rsync -r $GdnFolder $sdlRepositoryFolder ExitWithExitCode $LASTEXITCODE
} }
# cd to the sdl-config directory so we can run git there # We copy the .gdn folder from our local run into the git repository so it can be committed
Push-Location $sdlDir $sdlRepositoryFolder = Join-Path (Join-Path (Join-Path $sdlDir $Repository) $BranchName) '.gdn'
# git add . --> git commit --> git push if (Get-Command Robocopy) {
Write-Host "git add ." Robocopy /S $GdnFolder $sdlRepositoryFolder
git add . } else {
if ($LASTEXITCODE -ne 0) { rsync -r $GdnFolder $sdlRepositoryFolder
Write-Error "Git add failed with exit code $LASTEXITCODE." }
} # cd to the sdl-config directory so we can run git there
Write-Host "git -c user.email=`"dn-bot@microsoft.com`" -c user.name=`"Dotnet Bot`" commit -m `"$PushReason for $Repository/$BranchName`"" Push-Location $sdlDir
git -c user.email="dn-bot@microsoft.com" -c user.name="Dotnet Bot" commit -m "$PushReason for $Repository/$BranchName" # git add . --> git commit --> git push
if ($LASTEXITCODE -ne 0) { Write-Host 'git add .'
Write-Error "Git commit failed with exit code $LASTEXITCODE." git add .
if ($LASTEXITCODE -ne 0) {
Write-PipelineTelemetryError -Force -Category 'Sdl' -Message "Git add failed with exit code $LASTEXITCODE."
ExitWithExitCode $LASTEXITCODE
}
Write-Host "git -c user.email=`"dn-bot@microsoft.com`" -c user.name=`"Dotnet Bot`" commit -m `"$PushReason for $Repository/$BranchName`""
git -c user.email="dn-bot@microsoft.com" -c user.name="Dotnet Bot" commit -m "$PushReason for $Repository/$BranchName"
if ($LASTEXITCODE -ne 0) {
Write-PipelineTelemetryError -Force -Category 'Sdl' -Message "Git commit failed with exit code $LASTEXITCODE."
ExitWithExitCode $LASTEXITCODE
}
Write-Host 'git push'
git push
if ($LASTEXITCODE -ne 0) {
Write-PipelineTelemetryError -Force -Category 'Sdl' -Message "Git push failed with exit code $LASTEXITCODE."
ExitWithExitCode $LASTEXITCODE
}
# Return to the original directory
Pop-Location
} }
Write-Host "git push" catch {
git push Write-Host $_.ScriptStackTrace
if ($LASTEXITCODE -ne 0) { Write-PipelineTelemetryError -Category 'Sdl' -Message $_
Write-Error "Git push failed with exit code $LASTEXITCODE." ExitWithExitCode 1
} }
# Return to the original directory
Pop-Location

88
eng/common/sdl/run-sdl.ps1

@ -5,55 +5,69 @@ Param(
[string] $GdnFolder, [string] $GdnFolder,
[string[]] $ToolsList, [string[]] $ToolsList,
[string] $UpdateBaseline, [string] $UpdateBaseline,
[string] $GuardianLoggerLevel="Standard", [string] $GuardianLoggerLevel='Standard',
[string[]] $CrScanAdditionalRunConfigParams, [string[]] $CrScanAdditionalRunConfigParams,
[string[]] $PoliCheckAdditionalRunConfigParams [string[]] $PoliCheckAdditionalRunConfigParams
) )
$ErrorActionPreference = "Stop" $ErrorActionPreference = 'Stop'
Set-StrictMode -Version 2.0 Set-StrictMode -Version 2.0
$disableConfigureToolsetImport = $true
$LASTEXITCODE = 0 $LASTEXITCODE = 0
# We store config files in the r directory of .gdn try {
Write-Host $ToolsList # `tools.ps1` checks $ci to perform some actions. Since the SDL
$gdnConfigPath = Join-Path $GdnFolder "r" # scripts don't necessarily execute in the same agent that run the
$ValidPath = Test-Path $GuardianCliLocation # build.ps1/sh script this variable isn't automatically set.
$ci = $true
. $PSScriptRoot\..\tools.ps1
if ($ValidPath -eq $False) # We store config files in the r directory of .gdn
{ Write-Host $ToolsList
Write-Host "Invalid Guardian CLI Location." $gdnConfigPath = Join-Path $GdnFolder 'r'
exit 1 $ValidPath = Test-Path $GuardianCliLocation
}
$configParam = @("--config") if ($ValidPath -eq $False)
{
foreach ($tool in $ToolsList) { Write-PipelineTelemetryError -Force -Category 'Sdl' -Message "Invalid Guardian CLI Location."
$gdnConfigFile = Join-Path $gdnConfigPath "$tool-configure.gdnconfig" ExitWithExitCode 1
Write-Host $tool
# We have to manually configure tools that run on source to look at the source directory only
if ($tool -eq "credscan") {
Write-Host "$GuardianCliLocation configure --working-directory $WorkingDirectory --tool $tool --output-path $gdnConfigFile --logger-level $GuardianLoggerLevel --noninteractive --force --args `" TargetDirectory < $TargetDirectory `" `" OutputType < pre `" $(If ($CrScanAdditionalRunConfigParams) {$CrScanAdditionalRunConfigParams})"
& $GuardianCliLocation configure --working-directory $WorkingDirectory --tool $tool --output-path $gdnConfigFile --logger-level $GuardianLoggerLevel --noninteractive --force --args " TargetDirectory < $TargetDirectory " "OutputType < pre" $(If ($CrScanAdditionalRunConfigParams) {$CrScanAdditionalRunConfigParams})
if ($LASTEXITCODE -ne 0) {
Write-Host "Guardian configure for $tool failed with exit code $LASTEXITCODE."
exit $LASTEXITCODE
}
} }
if ($tool -eq "policheck") {
Write-Host "$GuardianCliLocation configure --working-directory $WorkingDirectory --tool $tool --output-path $gdnConfigFile --logger-level $GuardianLoggerLevel --noninteractive --force --args `" Target < $TargetDirectory `" $(If ($PoliCheckAdditionalRunConfigParams) {$PoliCheckAdditionalRunConfigParams})" $configParam = @('--config')
& $GuardianCliLocation configure --working-directory $WorkingDirectory --tool $tool --output-path $gdnConfigFile --logger-level $GuardianLoggerLevel --noninteractive --force --args " Target < $TargetDirectory " $(If ($PoliCheckAdditionalRunConfigParams) {$PoliCheckAdditionalRunConfigParams})
if ($LASTEXITCODE -ne 0) { foreach ($tool in $ToolsList) {
Write-Host "Guardian configure for $tool failed with exit code $LASTEXITCODE." $gdnConfigFile = Join-Path $gdnConfigPath "$tool-configure.gdnconfig"
exit $LASTEXITCODE Write-Host $tool
# We have to manually configure tools that run on source to look at the source directory only
if ($tool -eq 'credscan') {
Write-Host "$GuardianCliLocation configure --working-directory $WorkingDirectory --tool $tool --output-path $gdnConfigFile --logger-level $GuardianLoggerLevel --noninteractive --force --args `" TargetDirectory < $TargetDirectory `" `" OutputType < pre `" $(If ($CrScanAdditionalRunConfigParams) {$CrScanAdditionalRunConfigParams})"
& $GuardianCliLocation configure --working-directory $WorkingDirectory --tool $tool --output-path $gdnConfigFile --logger-level $GuardianLoggerLevel --noninteractive --force --args " TargetDirectory < $TargetDirectory " "OutputType < pre" $(If ($CrScanAdditionalRunConfigParams) {$CrScanAdditionalRunConfigParams})
if ($LASTEXITCODE -ne 0) {
Write-PipelineTelemetryError -Force -Category 'Sdl' -Message "Guardian configure for $tool failed with exit code $LASTEXITCODE."
ExitWithExitCode $LASTEXITCODE
}
} }
if ($tool -eq 'policheck') {
Write-Host "$GuardianCliLocation configure --working-directory $WorkingDirectory --tool $tool --output-path $gdnConfigFile --logger-level $GuardianLoggerLevel --noninteractive --force --args `" Target < $TargetDirectory `" $(If ($PoliCheckAdditionalRunConfigParams) {$PoliCheckAdditionalRunConfigParams})"
& $GuardianCliLocation configure --working-directory $WorkingDirectory --tool $tool --output-path $gdnConfigFile --logger-level $GuardianLoggerLevel --noninteractive --force --args " Target < $TargetDirectory " $(If ($PoliCheckAdditionalRunConfigParams) {$PoliCheckAdditionalRunConfigParams})
if ($LASTEXITCODE -ne 0) {
Write-PipelineTelemetryError -Force -Category 'Sdl' -Message "Guardian configure for $tool failed with exit code $LASTEXITCODE."
ExitWithExitCode $LASTEXITCODE
}
}
$configParam+=$gdnConfigFile
} }
$configParam+=$gdnConfigFile Write-Host "$GuardianCliLocation run --working-directory $WorkingDirectory --baseline mainbaseline --update-baseline $UpdateBaseline --logger-level $GuardianLoggerLevel $configParam"
& $GuardianCliLocation run --working-directory $WorkingDirectory --tool $tool --baseline mainbaseline --update-baseline $UpdateBaseline --logger-level $GuardianLoggerLevel $configParam
if ($LASTEXITCODE -ne 0) {
Write-PipelineTelemetryError -Force -Category 'Sdl' -Message "Guardian run for $ToolsList using $configParam failed with exit code $LASTEXITCODE."
ExitWithExitCode $LASTEXITCODE
}
} }
catch {
Write-Host "$GuardianCliLocation run --working-directory $WorkingDirectory --baseline mainbaseline --update-baseline $UpdateBaseline --logger-level $GuardianLoggerLevel $configParam" Write-Host $_.ScriptStackTrace
& $GuardianCliLocation run --working-directory $WorkingDirectory --tool $tool --baseline mainbaseline --update-baseline $UpdateBaseline --logger-level $GuardianLoggerLevel $configParam Write-PipelineTelemetryError -Force -Category 'Sdl' -Message $_
if ($LASTEXITCODE -ne 0) { ExitWithExitCode 1
Write-Host "Guardian run for $ToolsList using $configParam failed with exit code $LASTEXITCODE."
exit $LASTEXITCODE
} }

52
eng/common/templates/job/execute-sdl.yml

@ -1,4 +1,5 @@
parameters: parameters:
enable: 'false' # Whether the SDL validation job should execute or not
overrideParameters: '' # Optional: to override values for parameters. overrideParameters: '' # Optional: to override values for parameters.
additionalParameters: '' # Optional: parameters that need user specific values eg: '-SourceToolsList @("abc","def") -ArtifactToolsList @("ghi","jkl")' additionalParameters: '' # Optional: parameters that need user specific values eg: '-SourceToolsList @("abc","def") -ArtifactToolsList @("ghi","jkl")'
# There is some sort of bug (has been reported) in Azure DevOps where if this parameter is named # There is some sort of bug (has been reported) in Azure DevOps where if this parameter is named
@ -6,33 +7,62 @@ parameters:
# This can also be remedied by the caller (post-build.yml) if it does not use a nested parameter # This can also be remedied by the caller (post-build.yml) if it does not use a nested parameter
sdlContinueOnError: false # optional: determines whether to continue the build if the step errors; sdlContinueOnError: false # optional: determines whether to continue the build if the step errors;
dependsOn: '' # Optional: dependencies of the job dependsOn: '' # Optional: dependencies of the job
artifactNames: '' # Optional: patterns supplied to DownloadBuildArtifacts
# Usage:
# artifactNames:
# - 'BlobArtifacts'
# - 'Artifacts_Windows_NT_Release'
jobs: jobs:
- job: Run_SDL - job: Run_SDL
dependsOn: ${{ parameters.dependsOn }} dependsOn: ${{ parameters.dependsOn }}
displayName: Run SDL tool displayName: Run SDL tool
condition: eq( ${{ parameters.enable }}, 'true')
variables: variables:
- group: DotNet-VSTS-Bot - group: DotNet-VSTS-Bot
- name: AzDOProjectName
value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOProjectName'] ]
- name: AzDOPipelineId
value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOPipelineId'] ]
- name: AzDOBuildId
value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOBuildId'] ]
pool: pool:
name: Hosted VS2017 name: Hosted VS2017
steps: steps:
- checkout: self - checkout: self
clean: true clean: true
- task: DownloadBuildArtifacts@0 - ${{ if ne(parameters.artifactNames, '') }}:
displayName: Download Build Artifacts - ${{ each artifactName in parameters.artifactNames }}:
inputs: - task: DownloadBuildArtifacts@0
buildType: current displayName: Download Build Artifacts
downloadType: specific files inputs:
matchingPattern: "**" buildType: specific
downloadPath: $(Build.SourcesDirectory)\artifacts buildVersionToDownload: specific
project: $(AzDOProjectName)
pipeline: $(AzDOPipelineId)
buildId: $(AzDOBuildId)
artifactName: ${{ artifactName }}
downloadPath: $(Build.ArtifactStagingDirectory)\artifacts
- ${{ if eq(parameters.artifactNames, '') }}:
- task: DownloadBuildArtifacts@0
displayName: Download Build Artifacts
inputs:
buildType: specific
buildVersionToDownload: specific
project: $(AzDOProjectName)
pipeline: $(AzDOPipelineId)
buildId: $(AzDOBuildId)
downloadType: specific files
itemPattern: "**"
downloadPath: $(Build.ArtifactStagingDirectory)\artifacts
- powershell: eng/common/sdl/extract-artifact-packages.ps1 - powershell: eng/common/sdl/extract-artifact-packages.ps1
-InputPath $(Build.SourcesDirectory)\artifacts\BlobArtifacts -InputPath $(Build.ArtifactStagingDirectory)\artifacts\BlobArtifacts
-ExtractPath $(Build.SourcesDirectory)\artifacts\BlobArtifacts -ExtractPath $(Build.ArtifactStagingDirectory)\artifacts\BlobArtifacts
displayName: Extract Blob Artifacts displayName: Extract Blob Artifacts
continueOnError: ${{ parameters.sdlContinueOnError }} continueOnError: ${{ parameters.sdlContinueOnError }}
- powershell: eng/common/sdl/extract-artifact-packages.ps1 - powershell: eng/common/sdl/extract-artifact-packages.ps1
-InputPath $(Build.SourcesDirectory)\artifacts\PackageArtifacts -InputPath $(Build.ArtifactStagingDirectory)\artifacts\PackageArtifacts
-ExtractPath $(Build.SourcesDirectory)\artifacts\PackageArtifacts -ExtractPath $(Build.ArtifactStagingDirectory)\artifacts\PackageArtifacts
displayName: Extract Package Artifacts displayName: Extract Package Artifacts
continueOnError: ${{ parameters.sdlContinueOnError }} continueOnError: ${{ parameters.sdlContinueOnError }}
- task: NuGetToolInstaller@1 - task: NuGetToolInstaller@1

142
eng/common/templates/job/job.yml

@ -1,67 +1,33 @@
# Internal resources (telemetry, microbuild) can only be accessed from non-public projects,
# and some (Microbuild) should only be applied to non-PR cases for internal builds.
parameters: parameters:
# Job schema parameters - https://docs.microsoft.com/en-us/azure/devops/pipelines/yaml-schema?view=vsts&tabs=schema#job # Job schema parameters - https://docs.microsoft.com/en-us/azure/devops/pipelines/yaml-schema?view=vsts&tabs=schema#job
cancelTimeoutInMinutes: '' cancelTimeoutInMinutes: ''
condition: '' condition: ''
continueOnError: false
container: '' container: ''
continueOnError: false
dependsOn: '' dependsOn: ''
displayName: '' displayName: ''
steps: []
pool: '' pool: ''
steps: []
strategy: '' strategy: ''
timeoutInMinutes: '' timeoutInMinutes: ''
variables: [] variables: []
workspace: '' workspace: ''
# Job base template specific parameters # Job base template specific parameters
# Optional: Enable installing Microbuild plugin # See schema documentation - https://github.com/dotnet/arcade/blob/master/Documentation/AzureDevOps/TemplateSchema.md
# if 'true', these "variables" must be specified in the variables object or as part of the queue matrix artifacts: ''
# _TeamName - the name of your team
# _SignType - 'test' or 'real'
enableMicrobuild: false enableMicrobuild: false
# Optional: Include PublishBuildArtifacts task
enablePublishBuildArtifacts: false enablePublishBuildArtifacts: false
# Optional: Enable publishing to the build asset registry
enablePublishBuildAssets: false enablePublishBuildAssets: false
# Optional: Prevent gather/push manifest from executing when using publishing pipelines
enablePublishUsingPipelines: false
# Optional: Include PublishTestResults task
enablePublishTestResults: false enablePublishTestResults: false
enablePublishUsingPipelines: false
# Optional: enable sending telemetry
enableTelemetry: false
# Optional: define the helix repo for telemetry (example: 'dotnet/arcade')
helixRepo: ''
# Optional: define the helix type for telemetry (example: 'build/product/')
helixType: ''
# Required: name of the job
name: '' name: ''
preSteps: []
# Optional: should run as a public build even in the internal project
# if 'true', the build won't run any of the internal only steps, even if it is running in non-public projects.
runAsPublic: false runAsPublic: false
# Internal resources (telemetry, microbuild) can only be accessed from non-public projects,
# and some (Microbuild) should only be applied to non-PR cases for internal builds.
jobs: jobs:
- job: ${{ parameters.name }} - job: ${{ parameters.name }}
@ -93,7 +59,7 @@ jobs:
timeoutInMinutes: ${{ parameters.timeoutInMinutes }} timeoutInMinutes: ${{ parameters.timeoutInMinutes }}
variables: variables:
- ${{ if eq(parameters.enableTelemetry, 'true') }}: - ${{ if ne(parameters.enableTelemetry, 'false') }}:
- name: DOTNET_CLI_TELEMETRY_PROFILE - name: DOTNET_CLI_TELEMETRY_PROFILE
value: '$(Build.Repository.Uri)' value: '$(Build.Repository.Uri)'
- ${{ each variable in parameters.variables }}: - ${{ each variable in parameters.variables }}:
@ -125,21 +91,12 @@ jobs:
workspace: ${{ parameters.workspace }} workspace: ${{ parameters.workspace }}
steps: steps:
- ${{ if eq(parameters.enableTelemetry, 'true') }}: - ${{ if ne(parameters.preSteps, '') }}:
# Telemetry tasks are built from https://github.com/dotnet/arcade-extensions - ${{ each preStep in parameters.preSteps }}:
- task: sendStartTelemetry@0 - ${{ preStep }}
displayName: 'Send Helix Start Telemetry'
inputs:
helixRepo: ${{ parameters.helixRepo }}
${{ if ne(parameters.helixType, '') }}:
helixType: ${{ parameters.helixType }}
buildConfig: $(_BuildConfig)
runAsPublic: ${{ parameters.runAsPublic }}
continueOnError: ${{ parameters.continueOnError }}
condition: always()
- ${{ if eq(parameters.enableMicrobuild, 'true') }}: - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}:
- ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: - ${{ if eq(parameters.enableMicrobuild, 'true') }}:
- task: MicroBuildSigningPlugin@2 - task: MicroBuildSigningPlugin@2
displayName: Install MicroBuild plugin displayName: Install MicroBuild plugin
inputs: inputs:
@ -151,9 +108,16 @@ jobs:
continueOnError: ${{ parameters.continueOnError }} continueOnError: ${{ parameters.continueOnError }}
condition: and(succeeded(), in(variables['_SignType'], 'real', 'test'), eq(variables['Agent.Os'], 'Windows_NT')) condition: and(succeeded(), in(variables['_SignType'], 'real', 'test'), eq(variables['Agent.Os'], 'Windows_NT'))
- ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}:
- task: NuGetAuthenticate@0 - task: NuGetAuthenticate@0
- ${{ if or(eq(parameters.artifacts.download, 'true'), ne(parameters.artifacts.download, '')) }}:
- task: DownloadPipelineArtifact@2
inputs:
buildType: current
artifactName: ${{ coalesce(parameters.artifacts.download.name, 'Artifacts_$(Agent.OS)_$(_BuildConfig)') }}
targetPath: ${{ coalesce(parameters.artifacts.download.path, 'artifacts') }}
itemPattern: ${{ coalesce(parameters.artifacts.download.pattern, '**') }}
- ${{ each step in parameters.steps }}: - ${{ each step in parameters.steps }}:
- ${{ step }} - ${{ step }}
@ -166,20 +130,60 @@ jobs:
env: env:
TeamName: $(_TeamName) TeamName: $(_TeamName)
- ${{ if eq(parameters.enableTelemetry, 'true') }}: - ${{ if ne(parameters.artifacts.publish, '') }}:
# Telemetry tasks are built from https://github.com/dotnet/arcade-extensions - ${{ if or(eq(parameters.artifacts.publish.artifacts, 'true'), ne(parameters.artifacts.publish.artifacts, '')) }}:
- task: sendEndTelemetry@0 - task: CopyFiles@2
displayName: 'Send Helix End Telemetry' displayName: Gather binaries for publish to artifacts
continueOnError: ${{ parameters.continueOnError }} inputs:
condition: always() SourceFolder: 'artifacts/bin'
Contents: '**'
- ${{ if eq(parameters.enablePublishBuildArtifacts, 'true') }}: TargetFolder: '$(Build.ArtifactStagingDirectory)/artifacts/bin'
- task: CopyFiles@2
displayName: Gather packages for publish to artifacts
inputs:
SourceFolder: 'artifacts/packages'
Contents: '**'
TargetFolder: '$(Build.ArtifactStagingDirectory)/artifacts/packages'
- task: PublishBuildArtifacts@1
displayName: Publish pipeline artifacts
inputs:
PathtoPublish: '$(Build.ArtifactStagingDirectory)/artifacts'
PublishLocation: Container
ArtifactName: ${{ coalesce(parameters.artifacts.publish.artifacts.name , 'Artifacts_$(Agent.Os)_$(_BuildConfig)') }}
continueOnError: true
condition: always()
- ${{ if or(eq(parameters.artifacts.publish.logs, 'true'), ne(parameters.artifacts.publish.logs, '')) }}:
- publish: artifacts/log
artifact: ${{ coalesce(parameters.artifacts.publish.logs.name, 'Logs_Build_$(Agent.Os)_$(_BuildConfig)') }}
displayName: Publish logs
continueOnError: true
condition: always()
- ${{ if or(eq(parameters.artifacts.publish.manifests, 'true'), ne(parameters.artifacts.publish.manifests, '')) }}:
- ${{ if and(ne(parameters.enablePublishUsingPipelines, 'true'), eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}:
- task: CopyFiles@2
displayName: Gather Asset Manifests
inputs:
SourceFolder: '$(Build.SourcesDirectory)/artifacts/log/$(_BuildConfig)/AssetManifest'
TargetFolder: '$(Build.ArtifactStagingDirectory)/AssetManifests'
continueOnError: ${{ parameters.continueOnError }}
condition: and(succeeded(), eq(variables['_DotNetPublishToBlobFeed'], 'true'))
- task: PublishBuildArtifacts@1
displayName: Push Asset Manifests
inputs:
PathtoPublish: '$(Build.ArtifactStagingDirectory)/AssetManifests'
PublishLocation: Container
ArtifactName: AssetManifests
continueOnError: ${{ parameters.continueOnError }}
condition: and(succeeded(), eq(variables['_DotNetPublishToBlobFeed'], 'true'))
- ${{ if ne(parameters.enablePublishBuildArtifacts, 'false') }}:
- task: PublishBuildArtifacts@1 - task: PublishBuildArtifacts@1
displayName: Publish Logs displayName: Publish Logs
inputs: inputs:
PathtoPublish: '$(Build.SourcesDirectory)/artifacts/log/$(_BuildConfig)' PathtoPublish: '$(Build.SourcesDirectory)/artifacts/log/$(_BuildConfig)'
PublishLocation: Container PublishLocation: Container
ArtifactName: $(Agent.Os)_$(Agent.JobName) ArtifactName: ${{ coalesce(parameters.enablePublishBuildArtifacts.artifactName, '$(Agent.Os)_$(Agent.JobName)' ) }}
continueOnError: true continueOnError: true
condition: always() condition: always()

6
eng/common/templates/job/publish-build-assets.yml

@ -37,6 +37,12 @@ jobs:
- name: _BuildConfig - name: _BuildConfig
value: ${{ parameters.configuration }} value: ${{ parameters.configuration }}
- group: Publish-Build-Assets - group: Publish-Build-Assets
# Skip component governance and codesign validation for SDL. These jobs
# create no content.
- name: skipComponentGovernanceDetection
value: true
- name: runCodesignValidationInjection
value: false
steps: steps:
- ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}:

74
eng/common/templates/jobs/jobs.yml

@ -1,19 +1,10 @@
parameters: parameters:
# Optional: 'true' if failures in job.yml job should not fail the job # See schema documentation in /Documentation/AzureDevOps/TemplateSchema.md
continueOnError: false continueOnError: false
# Optional: Enable installing Microbuild plugin
# if 'true', these "variables" must be specified in the variables object or as part of the queue matrix
# _TeamName - the name of your team
# _SignType - 'test' or 'real'
enableMicrobuild: false
# Optional: Include PublishBuildArtifacts task # Optional: Include PublishBuildArtifacts task
enablePublishBuildArtifacts: false enablePublishBuildArtifacts: false
# Optional: Enable publishing to the build asset registry
enablePublishBuildAssets: false
# Optional: Enable publishing using release pipelines # Optional: Enable publishing using release pipelines
enablePublishUsingPipelines: false enablePublishUsingPipelines: false
@ -23,19 +14,9 @@ parameters:
# Optional: Include toolset dependencies in the generated graph files # Optional: Include toolset dependencies in the generated graph files
includeToolset: false includeToolset: false
# Optional: Include PublishTestResults task
enablePublishTestResults: false
# Optional: enable sending telemetry
# if enabled then the 'helixRepo' parameter should also be specified
enableTelemetry: false
# Required: A collection of jobs to run - https://docs.microsoft.com/en-us/azure/devops/pipelines/yaml-schema?view=vsts&tabs=schema#job # Required: A collection of jobs to run - https://docs.microsoft.com/en-us/azure/devops/pipelines/yaml-schema?view=vsts&tabs=schema#job
jobs: [] jobs: []
# Optional: define the helix repo for telemetry (example: 'dotnet/arcade')
helixRepo: ''
# Optional: Override automatically derived dependsOn value for "publish build assets" job # Optional: Override automatically derived dependsOn value for "publish build assets" job
publishBuildAssetsDependsOn: '' publishBuildAssetsDependsOn: ''
@ -62,29 +43,30 @@ jobs:
name: ${{ job.job }} name: ${{ job.job }}
- ${{ if and(eq(parameters.enablePublishBuildAssets, true), eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}:
- template: ../job/publish-build-assets.yml - ${{ if or(eq(parameters.enablePublishBuildAssets, true), eq(parameters.artifacts.publish.manifests, 'true'), ne(parameters.artifacts.publish.manifests, '')) }}:
parameters: - template: ../job/publish-build-assets.yml
continueOnError: ${{ parameters.continueOnError }} parameters:
dependsOn: continueOnError: ${{ parameters.continueOnError }}
- ${{ if ne(parameters.publishBuildAssetsDependsOn, '') }}: dependsOn:
- ${{ each job in parameters.publishBuildAssetsDependsOn }}: - ${{ if ne(parameters.publishBuildAssetsDependsOn, '') }}:
- ${{ job.job }} - ${{ each job in parameters.publishBuildAssetsDependsOn }}:
- ${{ if eq(parameters.publishBuildAssetsDependsOn, '') }}: - ${{ job.job }}
- ${{ each job in parameters.jobs }}: - ${{ if eq(parameters.publishBuildAssetsDependsOn, '') }}:
- ${{ job.job }} - ${{ each job in parameters.jobs }}:
pool: - ${{ job.job }}
vmImage: vs2017-win2016 pool:
runAsPublic: ${{ parameters.runAsPublic }} vmImage: vs2017-win2016
publishUsingPipelines: ${{ parameters.enablePublishUsingPipelines }} runAsPublic: ${{ parameters.runAsPublic }}
enablePublishBuildArtifacts: ${{ parameters.enablePublishBuildArtifacts }} publishUsingPipelines: ${{ parameters.enablePublishUsingPipelines }}
enablePublishBuildArtifacts: ${{ parameters.enablePublishBuildArtifacts }}
- ${{ if and(eq(parameters.graphFileGeneration.enabled, true), eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}:
- template: ../job/generate-graph-files.yml - ${{ if eq(parameters.graphFileGeneration.enabled, true) }}:
parameters: - template: ../job/generate-graph-files.yml
continueOnError: ${{ parameters.continueOnError }} parameters:
includeToolset: ${{ parameters.graphFileGeneration.includeToolset }} continueOnError: ${{ parameters.continueOnError }}
dependsOn: includeToolset: ${{ parameters.graphFileGeneration.includeToolset }}
- Asset_Registry_Publish dependsOn:
pool: - Asset_Registry_Publish
vmImage: vs2017-win2016 pool:
vmImage: vs2017-win2016

118
eng/common/templates/post-build/channels/netcore-internal-30.yml → eng/common/templates/post-build/channels/generic-internal-channel.yml

@ -1,36 +1,66 @@
parameters: parameters:
symbolPublishingAdditionalParameters: ''
artifactsPublishingAdditionalParameters: '' artifactsPublishingAdditionalParameters: ''
dependsOn:
- Validate
publishInstallersAndChecksums: false
symbolPublishingAdditionalParameters: ''
stageName: ''
channelName: ''
channelId: ''
transportFeed: ''
shippingFeed: ''
symbolsFeed: ''
stages: stages:
- stage: NetCore_30_Internal_Servicing_Publishing - stage: ${{ parameters.stageName }}
dependsOn: validate dependsOn: ${{ parameters.dependsOn }}
variables: variables:
- template: ../common-variables.yml - template: ../common-variables.yml
displayName: .NET Core 3.0 Internal Servicing Publishing displayName: ${{ parameters.channelName }} Publishing
jobs: jobs:
- template: ../setup-maestro-vars.yml - template: ../setup-maestro-vars.yml
- job: - job: publish_symbols
displayName: Symbol Publishing displayName: Symbol Publishing
dependsOn: setupMaestroVars dependsOn: setupMaestroVars
condition: contains(dependencies.setupMaestroVars.outputs['setReleaseVars.InitialChannels'], format('[{0}]', variables.InternalServicing_30_Channel_Id)) condition: or(contains(dependencies.setupMaestroVars.outputs['setReleaseVars.InitialChannels'], format('[{0}]', ${{ parameters.channelId }} )), eq(dependencies.setupMaestroVars.outputs['setReleaseVars.PromoteToMaestroChannelId'], ${{ parameters.channelId }}))
variables: variables:
- group: DotNet-Symbol-Server-Pats - group: DotNet-Symbol-Server-Pats
- name: AzDOProjectName
value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOProjectName'] ]
- name: AzDOPipelineId
value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOPipelineId'] ]
- name: AzDOBuildId
value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOBuildId'] ]
pool: pool:
vmImage: 'windows-2019' vmImage: 'windows-2019'
steps: steps:
- task: DownloadBuildArtifacts@0 - task: DownloadBuildArtifacts@0
displayName: Download Blob Artifacts displayName: Download Build Assets
inputs:
artifactName: 'BlobArtifacts'
continueOnError: true continueOnError: true
inputs:
buildType: specific
buildVersionToDownload: specific
project: $(AzDOProjectName)
pipeline: $(AzDOPipelineId)
buildId: $(AzDOBuildId)
downloadType: 'specific'
itemPattern: |
PdbArtifacts/**
BlobArtifacts/**
downloadPath: '$(Build.ArtifactStagingDirectory)'
- task: DownloadBuildArtifacts@0 # This is necessary whenever we want to publish/restore to an AzDO private feed
displayName: Download PDB Artifacts # Since sdk-task.ps1 tries to restore packages we need to do this authentication here
# otherwise it'll complain about accessing a private feed.
- task: NuGetAuthenticate@0
displayName: 'Authenticate to AzDO Feeds'
- task: PowerShell@2
displayName: Enable cross-org publishing
inputs: inputs:
artifactName: 'PDBArtifacts' filePath: eng\common\enable-cross-org-publishing.ps1
continueOnError: true arguments: -token $(dn-bot-dnceng-artifact-feeds-rw)
- task: PowerShell@2 - task: PowerShell@2
displayName: Publish displayName: Publish
@ -43,39 +73,48 @@ stages:
/p:BlobBasePath='$(Build.ArtifactStagingDirectory)/BlobArtifacts/' /p:BlobBasePath='$(Build.ArtifactStagingDirectory)/BlobArtifacts/'
/p:SymbolPublishingExclusionsFile='$(Build.SourcesDirectory)/eng/SymbolPublishingExclusionsFile.txt' /p:SymbolPublishingExclusionsFile='$(Build.SourcesDirectory)/eng/SymbolPublishingExclusionsFile.txt'
/p:Configuration=Release /p:Configuration=Release
/p:PublishToMSDL=false
${{ parameters.symbolPublishingAdditionalParameters }} ${{ parameters.symbolPublishingAdditionalParameters }}
- template: ../../steps/publish-logs.yml
parameters:
StageLabel: '${{ parameters.stageName }}'
JobLabel: 'SymbolPublishing'
- job: publish_assets - job: publish_assets
displayName: Publish Assets displayName: Publish Assets
dependsOn: setupMaestroVars dependsOn: setupMaestroVars
timeoutInMinutes: 120
variables: variables:
- group: DotNet-Blob-Feed
- group: AzureDevOps-Artifact-Feeds-Pats
- name: BARBuildId - name: BARBuildId
value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.BARBuildId'] ] value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.BARBuildId'] ]
- name: IsStableBuild - name: IsStableBuild
value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.IsStableBuild'] ] value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.IsStableBuild'] ]
condition: contains(dependencies.setupMaestroVars.outputs['setReleaseVars.InitialChannels'], format('[{0}]', variables.InternalServicing_30_Channel_Id)) - name: AzDOProjectName
value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOProjectName'] ]
- name: AzDOPipelineId
value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOPipelineId'] ]
- name: AzDOBuildId
value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOBuildId'] ]
condition: or(contains(dependencies.setupMaestroVars.outputs['setReleaseVars.InitialChannels'], format('[{0}]', ${{ parameters.channelId }} )), eq(dependencies.setupMaestroVars.outputs['setReleaseVars.PromoteToMaestroChannelId'], ${{ parameters.channelId }}))
pool: pool:
vmImage: 'windows-2019' vmImage: 'windows-2019'
steps: steps:
- task: DownloadBuildArtifacts@0 - task: DownloadBuildArtifacts@0
displayName: Download Package Artifacts displayName: Download Build Assets
inputs: continueOnError: true
buildType: current
artifactName: PackageArtifacts
- task: DownloadBuildArtifacts@0
displayName: Download Blob Artifacts
inputs:
buildType: current
artifactName: BlobArtifacts
- task: DownloadBuildArtifacts@0
displayName: Download Asset Manifests
inputs: inputs:
buildType: current buildType: specific
artifactName: AssetManifests buildVersionToDownload: specific
project: $(AzDOProjectName)
pipeline: $(AzDOPipelineId)
buildId: $(AzDOBuildId)
downloadType: 'specific'
itemPattern: |
PackageArtifacts/**
BlobArtifacts/**
AssetManifests/**
downloadPath: '$(Build.ArtifactStagingDirectory)'
- task: NuGetToolInstaller@1 - task: NuGetToolInstaller@1
displayName: 'Install NuGet.exe' displayName: 'Install NuGet.exe'
@ -114,15 +153,20 @@ stages:
/p:ChecksumsAzureAccountKey=$(InternalChecksumsBlobFeedKey) /p:ChecksumsAzureAccountKey=$(InternalChecksumsBlobFeedKey)
/p:InstallersTargetStaticFeed=$(InternalInstallersBlobFeedUrl) /p:InstallersTargetStaticFeed=$(InternalInstallersBlobFeedUrl)
/p:InstallersAzureAccountKey=$(InternalInstallersBlobFeedKey) /p:InstallersAzureAccountKey=$(InternalInstallersBlobFeedKey)
/p:PublishToAzureDevOpsNuGetFeeds=true /p:AzureDevOpsStaticShippingFeed='${{ parameters.shippingFeed }}'
/p:AzureDevOpsStaticShippingFeed='https://pkgs.dev.azure.com/dnceng/_packaging/dotnet3-internal/nuget/v3/index.json'
/p:AzureDevOpsStaticShippingFeedKey='$(dn-bot-dnceng-artifact-feeds-rw)' /p:AzureDevOpsStaticShippingFeedKey='$(dn-bot-dnceng-artifact-feeds-rw)'
/p:AzureDevOpsStaticTransportFeed='https://pkgs.dev.azure.com/dnceng/_packaging/dotnet3-internal-transport/nuget/v3/index.json' /p:AzureDevOpsStaticTransportFeed='${{ parameters.transportFeed }}'
/p:AzureDevOpsStaticTransportFeedKey='$(dn-bot-dnceng-artifact-feeds-rw)' /p:AzureDevOpsStaticTransportFeedKey='$(dn-bot-dnceng-artifact-feeds-rw)'
/p:AzureDevOpsStaticSymbolsFeed='https://pkgs.dev.azure.com/dnceng/_packaging/dotnet3-internal-symbols/nuget/v3/index.json' /p:AzureDevOpsStaticSymbolsFeed='${{ parameters.symbolsFeed }}'
/p:AzureDevOpsStaticSymbolsFeedKey='$(dn-bot-dnceng-artifact-feeds-rw)' /p:AzureDevOpsStaticSymbolsFeedKey='$(dn-bot-dnceng-artifact-feeds-rw)'
/p:PublishToMSDL=false
${{ parameters.artifactsPublishingAdditionalParameters }} ${{ parameters.artifactsPublishingAdditionalParameters }}
- template: ../../steps/promote-build.yml - template: ../../steps/publish-logs.yml
parameters:
StageLabel: '${{ parameters.stageName }}'
JobLabel: 'AssetsPublishing'
- template: ../../steps/add-build-to-channel.yml
parameters: parameters:
ChannelId: ${{ variables.InternalServicing_30_Channel_Id }} ChannelId: ${{ parameters.channelId }}

178
eng/common/templates/post-build/channels/generic-public-channel.yml

@ -0,0 +1,178 @@
parameters:
artifactsPublishingAdditionalParameters: ''
dependsOn:
- Validate
publishInstallersAndChecksums: false
symbolPublishingAdditionalParameters: ''
stageName: ''
channelName: ''
channelId: ''
transportFeed: ''
shippingFeed: ''
symbolsFeed: ''
# If the channel name is empty, no links will be generated
akaMSChannelName: ''
stages:
- stage: ${{ parameters.stageName }}
dependsOn: ${{ parameters.dependsOn }}
variables:
- template: ../common-variables.yml
displayName: ${{ parameters.channelName }} Publishing
jobs:
- template: ../setup-maestro-vars.yml
- job: publish_symbols
displayName: Symbol Publishing
dependsOn: setupMaestroVars
condition: or(contains(dependencies.setupMaestroVars.outputs['setReleaseVars.InitialChannels'], format('[{0}]', ${{ parameters.channelId }} )), eq(dependencies.setupMaestroVars.outputs['setReleaseVars.PromoteToMaestroChannelId'], ${{ parameters.channelId }}))
variables:
- group: DotNet-Symbol-Server-Pats
- name: AzDOProjectName
value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOProjectName'] ]
- name: AzDOPipelineId
value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOPipelineId'] ]
- name: AzDOBuildId
value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOBuildId'] ]
pool:
vmImage: 'windows-2019'
steps:
- task: DownloadBuildArtifacts@0
displayName: Download Build Assets
continueOnError: true
inputs:
buildType: specific
buildVersionToDownload: specific
project: $(AzDOProjectName)
pipeline: $(AzDOPipelineId)
buildId: $(AzDOBuildId)
downloadType: 'specific'
itemPattern: |
PdbArtifacts/**
BlobArtifacts/**
downloadPath: '$(Build.ArtifactStagingDirectory)'
# This is necessary whenever we want to publish/restore to an AzDO private feed
# Since sdk-task.ps1 tries to restore packages we need to do this authentication here
# otherwise it'll complain about accessing a private feed.
- task: NuGetAuthenticate@0
displayName: 'Authenticate to AzDO Feeds'
- task: PowerShell@2
displayName: Enable cross-org publishing
inputs:
filePath: eng\common\enable-cross-org-publishing.ps1
arguments: -token $(dn-bot-dnceng-artifact-feeds-rw)
- task: PowerShell@2
displayName: Publish
inputs:
filePath: eng\common\sdk-task.ps1
arguments: -task PublishToSymbolServers -restore -msbuildEngine dotnet
/p:DotNetSymbolServerTokenMsdl=$(microsoft-symbol-server-pat)
/p:DotNetSymbolServerTokenSymWeb=$(symweb-symbol-server-pat)
/p:PDBArtifactsDirectory='$(Build.ArtifactStagingDirectory)/PDBArtifacts/'
/p:BlobBasePath='$(Build.ArtifactStagingDirectory)/BlobArtifacts/'
/p:SymbolPublishingExclusionsFile='$(Build.SourcesDirectory)/eng/SymbolPublishingExclusionsFile.txt'
/p:Configuration=Release
${{ parameters.symbolPublishingAdditionalParameters }}
- template: ../../steps/publish-logs.yml
parameters:
StageLabel: '${{ parameters.stageName }}'
JobLabel: 'SymbolPublishing'
- job: publish_assets
displayName: Publish Assets
dependsOn: setupMaestroVars
timeoutInMinutes: 120
variables:
- name: BARBuildId
value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.BARBuildId'] ]
- name: IsStableBuild
value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.IsStableBuild'] ]
- name: AzDOProjectName
value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOProjectName'] ]
- name: AzDOPipelineId
value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOPipelineId'] ]
- name: AzDOBuildId
value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOBuildId'] ]
- name: ArtifactsCategory
value: ${{ coalesce(variables._DotNetArtifactsCategory, '.NETCore') }}
condition: or(contains(dependencies.setupMaestroVars.outputs['setReleaseVars.InitialChannels'], format('[{0}]', ${{ parameters.channelId }} )), eq(dependencies.setupMaestroVars.outputs['setReleaseVars.PromoteToMaestroChannelId'], ${{ parameters.channelId }}))
pool:
vmImage: 'windows-2019'
steps:
- task: DownloadBuildArtifacts@0
displayName: Download Build Assets
continueOnError: true
inputs:
buildType: specific
buildVersionToDownload: specific
project: $(AzDOProjectName)
pipeline: $(AzDOPipelineId)
buildId: $(AzDOBuildId)
downloadType: 'specific'
itemPattern: |
PackageArtifacts/**
BlobArtifacts/**
AssetManifests/**
downloadPath: '$(Build.ArtifactStagingDirectory)'
- task: NuGetToolInstaller@1
displayName: 'Install NuGet.exe'
# This is necessary whenever we want to publish/restore to an AzDO private feed
- task: NuGetAuthenticate@0
displayName: 'Authenticate to AzDO Feeds'
- task: PowerShell@2
displayName: Enable cross-org publishing
inputs:
filePath: eng\common\enable-cross-org-publishing.ps1
arguments: -token $(dn-bot-dnceng-artifact-feeds-rw)
- task: PowerShell@2
displayName: Publish Assets
inputs:
filePath: eng\common\sdk-task.ps1
arguments: -task PublishArtifactsInManifest -restore -msbuildEngine dotnet
/p:ArtifactsCategory=$(ArtifactsCategory)
/p:IsStableBuild=$(IsStableBuild)
/p:IsInternalBuild=$(IsInternalBuild)
/p:RepositoryName=$(Build.Repository.Name)
/p:CommitSha=$(Build.SourceVersion)
/p:NugetPath=$(NuGetExeToolPath)
/p:AzdoTargetFeedPAT='$(dn-bot-dnceng-universal-packages-rw)'
/p:AzureStorageTargetFeedPAT='$(dotnetfeed-storage-access-key-1)'
/p:BARBuildId=$(BARBuildId)
/p:MaestroApiEndpoint='$(MaestroApiEndPoint)'
/p:BuildAssetRegistryToken='$(MaestroApiAccessToken)'
/p:ManifestsBasePath='$(Build.ArtifactStagingDirectory)/AssetManifests/'
/p:BlobBasePath='$(Build.ArtifactStagingDirectory)/BlobArtifacts/'
/p:PackageBasePath='$(Build.ArtifactStagingDirectory)/PackageArtifacts/'
/p:Configuration=Release
/p:PublishInstallersAndChecksums=${{ parameters.publishInstallersAndChecksums }}
/p:InstallersTargetStaticFeed=$(InstallersBlobFeedUrl)
/p:InstallersAzureAccountKey=$(dotnetcli-storage-key)
/p:ChecksumsTargetStaticFeed=$(ChecksumsBlobFeedUrl)
/p:ChecksumsAzureAccountKey=$(dotnetclichecksums-storage-key)
/p:AzureDevOpsStaticShippingFeed='${{ parameters.shippingFeed }}'
/p:AzureDevOpsStaticShippingFeedKey='$(dn-bot-dnceng-artifact-feeds-rw)'
/p:AzureDevOpsStaticTransportFeed='${{ parameters.transportFeed }}'
/p:AzureDevOpsStaticTransportFeedKey='$(dn-bot-dnceng-artifact-feeds-rw)'
/p:AzureDevOpsStaticSymbolsFeed='${{ parameters.symbolsFeed }}'
/p:AzureDevOpsStaticSymbolsFeedKey='$(dn-bot-dnceng-artifact-feeds-rw)'
/p:LatestLinkShortUrlPrefix=dotnet/'${{ parameters.akaMSChannelName }}'
/p:AkaMSClientId=$(akams-client-id)
/p:AkaMSClientSecret=$(akams-client-secret)
${{ parameters.artifactsPublishingAdditionalParameters }}
- template: ../../steps/publish-logs.yml
parameters:
StageLabel: '${{ parameters.stageName }}'
JobLabel: 'AssetsPublishing'
- template: ../../steps/add-build-to-channel.yml
parameters:
ChannelId: ${{ parameters.channelId }}

95
eng/common/templates/post-build/channels/netcore-3-tools-validation.yml

@ -1,95 +0,0 @@
parameters:
artifactsPublishingAdditionalParameters: ''
publishInstallersAndChecksums: false
stages:
- stage: NetCore_3_Tools_Validation_Publish
dependsOn: validate
variables:
- template: ../common-variables.yml
displayName: .NET 3 Tools - Validation Publishing
jobs:
- template: ../setup-maestro-vars.yml
- job: publish_assets
displayName: Publish Assets
dependsOn: setupMaestroVars
variables:
- group: DotNet-Blob-Feed
- group: AzureDevOps-Artifact-Feeds-Pats
- name: BARBuildId
value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.BARBuildId'] ]
- name: IsStableBuild
value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.IsStableBuild'] ]
condition: contains(dependencies.setupMaestroVars.outputs['setReleaseVars.InitialChannels'], format('[{0}]', variables.NETCore_3_Tools_Validation_Channel_Id))
pool:
vmImage: 'windows-2019'
steps:
- task: DownloadBuildArtifacts@0
displayName: Download Package Artifacts
inputs:
buildType: current
artifactName: PackageArtifacts
- task: DownloadBuildArtifacts@0
displayName: Download Blob Artifacts
inputs:
buildType: current
artifactName: BlobArtifacts
- task: DownloadBuildArtifacts@0
displayName: Download Asset Manifests
inputs:
buildType: current
artifactName: AssetManifests
- task: NuGetToolInstaller@1
displayName: 'Install NuGet.exe'
# This is necessary whenever we want to publish/restore to an AzDO private feed
- task: NuGetAuthenticate@0
displayName: 'Authenticate to AzDO Feeds'
- task: PowerShell@2
displayName: Enable cross-org publishing
inputs:
filePath: eng\common\enable-cross-org-publishing.ps1
arguments: -token $(dn-bot-dnceng-artifact-feeds-rw)
- task: PowerShell@2
displayName: Publish Assets
inputs:
filePath: eng\common\sdk-task.ps1
arguments: -task PublishArtifactsInManifest -restore -msbuildEngine dotnet
/p:ArtifactsCategory=$(_DotNetValidationArtifactsCategory)
/p:IsStableBuild=$(IsStableBuild)
/p:IsInternalBuild=$(IsInternalBuild)
/p:RepositoryName=$(Build.Repository.Name)
/p:CommitSha=$(Build.SourceVersion)
/p:NugetPath=$(NuGetExeToolPath)
/p:AzdoTargetFeedPAT='$(dn-bot-dnceng-universal-packages-rw)'
/p:AzureStorageTargetFeedPAT='$(dotnetfeed-storage-access-key-1)'
/p:BARBuildId=$(BARBuildId)
/p:MaestroApiEndpoint='$(MaestroApiEndPoint)'
/p:BuildAssetRegistryToken='$(MaestroApiAccessToken)'
/p:ManifestsBasePath='$(Build.ArtifactStagingDirectory)/AssetManifests/'
/p:BlobBasePath='$(Build.ArtifactStagingDirectory)/BlobArtifacts/'
/p:PackageBasePath='$(Build.ArtifactStagingDirectory)/PackageArtifacts/'
/p:Configuration=Release
/p:PublishInstallersAndChecksums=${{ parameters.publishInstallersAndChecksums }}
/p:InstallersTargetStaticFeed=$(InstallersBlobFeedUrl)
/p:InstallersAzureAccountKey=$(dotnetcli-storage-key)
/p:ChecksumsTargetStaticFeed=$(ChecksumsBlobFeedUrl)
/p:ChecksumsAzureAccountKey=$(dotnetclichecksums-storage-key)
/p:PublishToAzureDevOpsNuGetFeeds=true
/p:AzureDevOpsStaticShippingFeed='https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools/nuget/v3/index.json'
/p:AzureDevOpsStaticShippingFeedKey='$(dn-bot-dnceng-artifact-feeds-rw)'
/p:AzureDevOpsStaticTransportFeed='https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools/nuget/v3/index.json'
/p:AzureDevOpsStaticTransportFeedKey='$(dn-bot-dnceng-artifact-feeds-rw)'
/p:AzureDevOpsStaticSymbolsFeed='https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools-symbols/nuget/v3/index.json'
/p:AzureDevOpsStaticSymbolsFeedKey='$(dn-bot-dnceng-artifact-feeds-rw)'
${{ parameters.artifactsPublishingAdditionalParameters }}
- template: ../../steps/promote-build.yml
parameters:
ChannelId: ${{ variables.NETCore_3_Tools_Validation_Channel_Id }}

130
eng/common/templates/post-build/channels/netcore-3-tools.yml

@ -1,130 +0,0 @@
parameters:
symbolPublishingAdditionalParameters: ''
artifactsPublishingAdditionalParameters: ''
publishInstallersAndChecksums: false
stages:
- stage: NetCore_3_Tools_Publish
dependsOn: validate
variables:
- template: ../common-variables.yml
displayName: .NET 3 Tools Publishing
jobs:
- template: ../setup-maestro-vars.yml
- job:
displayName: Symbol Publishing
dependsOn: setupMaestroVars
condition: contains(dependencies.setupMaestroVars.outputs['setReleaseVars.InitialChannels'], format('[{0}]', variables.NetCore_3_Tools_Channel_Id))
variables:
- group: DotNet-Symbol-Server-Pats
pool:
vmImage: 'windows-2019'
steps:
- task: DownloadBuildArtifacts@0
displayName: Download Blob Artifacts
inputs:
artifactName: 'BlobArtifacts'
continueOnError: true
- task: DownloadBuildArtifacts@0
displayName: Download PDB Artifacts
inputs:
artifactName: 'PDBArtifacts'
continueOnError: true
- task: PowerShell@2
displayName: Publish
inputs:
filePath: eng\common\sdk-task.ps1
arguments: -task PublishToSymbolServers -restore -msbuildEngine dotnet
/p:DotNetSymbolServerTokenMsdl=$(microsoft-symbol-server-pat)
/p:DotNetSymbolServerTokenSymWeb=$(symweb-symbol-server-pat)
/p:PDBArtifactsDirectory='$(Build.ArtifactStagingDirectory)/PDBArtifacts/'
/p:BlobBasePath='$(Build.ArtifactStagingDirectory)/BlobArtifacts/'
/p:SymbolPublishingExclusionsFile='$(Build.SourcesDirectory)/eng/SymbolPublishingExclusionsFile.txt'
/p:Configuration=Release
${{ parameters.symbolPublishingAdditionalParameters }}
- job: publish_assets
displayName: Publish Assets
dependsOn: setupMaestroVars
variables:
- group: DotNet-Blob-Feed
- group: AzureDevOps-Artifact-Feeds-Pats
- name: BARBuildId
value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.BARBuildId'] ]
- name: IsStableBuild
value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.IsStableBuild'] ]
condition: contains(dependencies.setupMaestroVars.outputs['setReleaseVars.InitialChannels'], format('[{0}]', variables.NetCore_3_Tools_Channel_Id))
pool:
vmImage: 'windows-2019'
steps:
- task: DownloadBuildArtifacts@0
displayName: Download Package Artifacts
inputs:
buildType: current
artifactName: PackageArtifacts
- task: DownloadBuildArtifacts@0
displayName: Download Blob Artifacts
inputs:
buildType: current
artifactName: BlobArtifacts
- task: DownloadBuildArtifacts@0
displayName: Download Asset Manifests
inputs:
buildType: current
artifactName: AssetManifests
- task: NuGetToolInstaller@1
displayName: 'Install NuGet.exe'
# This is necessary whenever we want to publish/restore to an AzDO private feed
- task: NuGetAuthenticate@0
displayName: 'Authenticate to AzDO Feeds'
- task: PowerShell@2
displayName: Enable cross-org publishing
inputs:
filePath: eng\common\enable-cross-org-publishing.ps1
arguments: -token $(dn-bot-dnceng-artifact-feeds-rw)
- task: PowerShell@2
displayName: Publish Assets
inputs:
filePath: eng\common\sdk-task.ps1
arguments: -task PublishArtifactsInManifest -restore -msbuildEngine dotnet
/p:ArtifactsCategory=$(_DotNetArtifactsCategory)
/p:IsStableBuild=$(IsStableBuild)
/p:IsInternalBuild=$(IsInternalBuild)
/p:RepositoryName=$(Build.Repository.Name)
/p:CommitSha=$(Build.SourceVersion)
/p:NugetPath=$(NuGetExeToolPath)
/p:AzdoTargetFeedPAT='$(dn-bot-dnceng-universal-packages-rw)'
/p:AzureStorageTargetFeedPAT='$(dotnetfeed-storage-access-key-1)'
/p:BARBuildId=$(BARBuildId)
/p:MaestroApiEndpoint='$(MaestroApiEndPoint)'
/p:BuildAssetRegistryToken='$(MaestroApiAccessToken)'
/p:ManifestsBasePath='$(Build.ArtifactStagingDirectory)/AssetManifests/'
/p:BlobBasePath='$(Build.ArtifactStagingDirectory)/BlobArtifacts/'
/p:PackageBasePath='$(Build.ArtifactStagingDirectory)/PackageArtifacts/'
/p:Configuration=Release
/p:PublishInstallersAndChecksums=${{ parameters.publishInstallersAndChecksums }}
/p:InstallersTargetStaticFeed=$(InstallersBlobFeedUrl)
/p:InstallersAzureAccountKey=$(dotnetcli-storage-key)
/p:ChecksumsTargetStaticFeed=$(ChecksumsBlobFeedUrl)
/p:ChecksumsAzureAccountKey=$(dotnetclichecksums-storage-key)
/p:PublishToAzureDevOpsNuGetFeeds=true
/p:AzureDevOpsStaticShippingFeed='https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools/nuget/v3/index.json'
/p:AzureDevOpsStaticShippingFeedKey='$(dn-bot-dnceng-artifact-feeds-rw)'
/p:AzureDevOpsStaticTransportFeed='https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools/nuget/v3/index.json'
/p:AzureDevOpsStaticTransportFeedKey='$(dn-bot-dnceng-artifact-feeds-rw)'
/p:AzureDevOpsStaticSymbolsFeed='https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools-symbols/nuget/v3/index.json'
/p:AzureDevOpsStaticSymbolsFeedKey='$(dn-bot-dnceng-artifact-feeds-rw)'
${{ parameters.artifactsPublishingAdditionalParameters }}
- template: ../../steps/promote-build.yml
parameters:
ChannelId: ${{ variables.NetCore_3_Tools_Channel_Id }}

130
eng/common/templates/post-build/channels/netcore-dev-31.yml

@ -1,130 +0,0 @@
parameters:
symbolPublishingAdditionalParameters: ''
artifactsPublishingAdditionalParameters: ''
publishInstallersAndChecksums: false
stages:
- stage: NetCore_Dev31_Publish
dependsOn: validate
variables:
- template: ../common-variables.yml
displayName: .NET Core 3.1 Dev Publishing
jobs:
- template: ../setup-maestro-vars.yml
- job:
displayName: Symbol Publishing
dependsOn: setupMaestroVars
condition: contains(dependencies.setupMaestroVars.outputs['setReleaseVars.InitialChannels'], format('[{0}]', variables.PublicDevRelease_31_Channel_Id))
variables:
- group: DotNet-Symbol-Server-Pats
pool:
vmImage: 'windows-2019'
steps:
- task: DownloadBuildArtifacts@0
displayName: Download Blob Artifacts
inputs:
artifactName: 'BlobArtifacts'
continueOnError: true
- task: DownloadBuildArtifacts@0
displayName: Download PDB Artifacts
inputs:
artifactName: 'PDBArtifacts'
continueOnError: true
- task: PowerShell@2
displayName: Publish
inputs:
filePath: eng\common\sdk-task.ps1
arguments: -task PublishToSymbolServers -restore -msbuildEngine dotnet
/p:DotNetSymbolServerTokenMsdl=$(microsoft-symbol-server-pat)
/p:DotNetSymbolServerTokenSymWeb=$(symweb-symbol-server-pat)
/p:PDBArtifactsDirectory='$(Build.ArtifactStagingDirectory)/PDBArtifacts/'
/p:BlobBasePath='$(Build.ArtifactStagingDirectory)/BlobArtifacts/'
/p:SymbolPublishingExclusionsFile='$(Build.SourcesDirectory)/eng/SymbolPublishingExclusionsFile.txt'
/p:Configuration=Release
${{ parameters.symbolPublishingAdditionalParameters }}
- job: publish_assets
displayName: Publish Assets
dependsOn: setupMaestroVars
variables:
- group: DotNet-Blob-Feed
- group: AzureDevOps-Artifact-Feeds-Pats
- name: BARBuildId
value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.BARBuildId'] ]
- name: IsStableBuild
value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.IsStableBuild'] ]
condition: contains(dependencies.setupMaestroVars.outputs['setReleaseVars.InitialChannels'], format('[{0}]', variables.PublicDevRelease_31_Channel_Id))
pool:
vmImage: 'windows-2019'
steps:
- task: DownloadBuildArtifacts@0
displayName: Download Package Artifacts
inputs:
buildType: current
artifactName: PackageArtifacts
- task: DownloadBuildArtifacts@0
displayName: Download Blob Artifacts
inputs:
buildType: current
artifactName: BlobArtifacts
- task: DownloadBuildArtifacts@0
displayName: Download Asset Manifests
inputs:
buildType: current
artifactName: AssetManifests
- task: NuGetToolInstaller@1
displayName: 'Install NuGet.exe'
# This is necessary whenever we want to publish/restore to an AzDO private feed
- task: NuGetAuthenticate@0
displayName: 'Authenticate to AzDO Feeds'
- task: PowerShell@2
displayName: Enable cross-org publishing
inputs:
filePath: eng\common\enable-cross-org-publishing.ps1
arguments: -token $(dn-bot-dnceng-artifact-feeds-rw)
- task: PowerShell@2
displayName: Publish Assets
inputs:
filePath: eng\common\sdk-task.ps1
arguments: -task PublishArtifactsInManifest -restore -msbuildEngine dotnet
/p:ArtifactsCategory=$(_DotNetArtifactsCategory)
/p:IsStableBuild=$(IsStableBuild)
/p:IsInternalBuild=$(IsInternalBuild)
/p:RepositoryName=$(Build.Repository.Name)
/p:CommitSha=$(Build.SourceVersion)
/p:NugetPath=$(NuGetExeToolPath)
/p:AzdoTargetFeedPAT='$(dn-bot-dnceng-universal-packages-rw)'
/p:AzureStorageTargetFeedPAT='$(dotnetfeed-storage-access-key-1)'
/p:BARBuildId=$(BARBuildId)
/p:MaestroApiEndpoint='$(MaestroApiEndPoint)'
/p:BuildAssetRegistryToken='$(MaestroApiAccessToken)'
/p:ManifestsBasePath='$(Build.ArtifactStagingDirectory)/AssetManifests/'
/p:BlobBasePath='$(Build.ArtifactStagingDirectory)/BlobArtifacts/'
/p:PackageBasePath='$(Build.ArtifactStagingDirectory)/PackageArtifacts/'
/p:Configuration=Release
/p:PublishInstallersAndChecksums=${{ parameters.publishInstallersAndChecksums }}
/p:InstallersTargetStaticFeed=$(InstallersBlobFeedUrl)
/p:InstallersAzureAccountKey=$(dotnetcli-storage-key)
/p:ChecksumsTargetStaticFeed=$(ChecksumsBlobFeedUrl)
/p:ChecksumsAzureAccountKey=$(dotnetclichecksums-storage-key)
/p:PublishToAzureDevOpsNuGetFeeds=true
/p:AzureDevOpsStaticShippingFeed='https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet3.1/nuget/v3/index.json'
/p:AzureDevOpsStaticShippingFeedKey='$(dn-bot-dnceng-artifact-feeds-rw)'
/p:AzureDevOpsStaticTransportFeed='https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet3.1-transport/nuget/v3/index.json'
/p:AzureDevOpsStaticTransportFeedKey='$(dn-bot-dnceng-artifact-feeds-rw)'
/p:AzureDevOpsStaticSymbolsFeed='https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet3.1-symbols/nuget/v3/index.json'
/p:AzureDevOpsStaticSymbolsFeedKey='$(dn-bot-dnceng-artifact-feeds-rw)'
${{ parameters.artifactsPublishingAdditionalParameters }}
- template: ../../steps/promote-build.yml
parameters:
ChannelId: ${{ variables.PublicDevRelease_31_Channel_Id }}

130
eng/common/templates/post-build/channels/netcore-dev-5.yml

@ -1,130 +0,0 @@
parameters:
symbolPublishingAdditionalParameters: ''
artifactsPublishingAdditionalParameters: ''
publishInstallersAndChecksums: false
stages:
- stage: NetCore_Dev5_Publish
dependsOn: validate
variables:
- template: ../common-variables.yml
displayName: .NET Core 5 Dev Publishing
jobs:
- template: ../setup-maestro-vars.yml
- job:
displayName: Symbol Publishing
dependsOn: setupMaestroVars
condition: contains(dependencies.setupMaestroVars.outputs['setReleaseVars.InitialChannels'], format('[{0}]', variables.NetCore_5_Dev_Channel_Id))
variables:
- group: DotNet-Symbol-Server-Pats
pool:
vmImage: 'windows-2019'
steps:
- task: DownloadBuildArtifacts@0
displayName: Download Blob Artifacts
inputs:
artifactName: 'BlobArtifacts'
continueOnError: true
- task: DownloadBuildArtifacts@0
displayName: Download PDB Artifacts
inputs:
artifactName: 'PDBArtifacts'
continueOnError: true
- task: PowerShell@2
displayName: Publish
inputs:
filePath: eng\common\sdk-task.ps1
arguments: -task PublishToSymbolServers -restore -msbuildEngine dotnet
/p:DotNetSymbolServerTokenMsdl=$(microsoft-symbol-server-pat)
/p:DotNetSymbolServerTokenSymWeb=$(symweb-symbol-server-pat)
/p:PDBArtifactsDirectory='$(Build.ArtifactStagingDirectory)/PDBArtifacts/'
/p:BlobBasePath='$(Build.ArtifactStagingDirectory)/BlobArtifacts/'
/p:SymbolPublishingExclusionsFile='$(Build.SourcesDirectory)/eng/SymbolPublishingExclusionsFile.txt'
/p:Configuration=Release
${{ parameters.symbolPublishingAdditionalParameters }}
- job: publish_assets
displayName: Publish Assets
dependsOn: setupMaestroVars
variables:
- group: DotNet-Blob-Feed
- group: AzureDevOps-Artifact-Feeds-Pats
- name: BARBuildId
value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.BARBuildId'] ]
- name: IsStableBuild
value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.IsStableBuild'] ]
condition: contains(dependencies.setupMaestroVars.outputs['setReleaseVars.InitialChannels'], format('[{0}]', variables.NetCore_5_Dev_Channel_Id))
pool:
vmImage: 'windows-2019'
steps:
- task: DownloadBuildArtifacts@0
displayName: Download Package Artifacts
inputs:
buildType: current
artifactName: PackageArtifacts
- task: DownloadBuildArtifacts@0
displayName: Download Blob Artifacts
inputs:
buildType: current
artifactName: BlobArtifacts
- task: DownloadBuildArtifacts@0
displayName: Download Asset Manifests
inputs:
buildType: current
artifactName: AssetManifests
- task: NuGetToolInstaller@1
displayName: 'Install NuGet.exe'
# This is necessary whenever we want to publish/restore to an AzDO private feed
- task: NuGetAuthenticate@0
displayName: 'Authenticate to AzDO Feeds'
- task: PowerShell@2
displayName: Enable cross-org publishing
inputs:
filePath: eng\common\enable-cross-org-publishing.ps1
arguments: -token $(dn-bot-dnceng-artifact-feeds-rw)
- task: PowerShell@2
displayName: Publish Assets
inputs:
filePath: eng\common\sdk-task.ps1
arguments: -task PublishArtifactsInManifest -restore -msbuildEngine dotnet
/p:ArtifactsCategory=$(_DotNetArtifactsCategory)
/p:IsStableBuild=$(IsStableBuild)
/p:IsInternalBuild=$(IsInternalBuild)
/p:RepositoryName=$(Build.Repository.Name)
/p:CommitSha=$(Build.SourceVersion)
/p:NugetPath=$(NuGetExeToolPath)
/p:AzdoTargetFeedPAT='$(dn-bot-dnceng-universal-packages-rw)'
/p:AzureStorageTargetFeedPAT='$(dotnetfeed-storage-access-key-1)'
/p:BARBuildId=$(BARBuildId)
/p:MaestroApiEndpoint='$(MaestroApiEndPoint)'
/p:BuildAssetRegistryToken='$(MaestroApiAccessToken)'
/p:ManifestsBasePath='$(Build.ArtifactStagingDirectory)/AssetManifests/'
/p:BlobBasePath='$(Build.ArtifactStagingDirectory)/BlobArtifacts/'
/p:PackageBasePath='$(Build.ArtifactStagingDirectory)/PackageArtifacts/'
/p:Configuration=Release
/p:PublishInstallersAndChecksums=${{ parameters.publishInstallersAndChecksums }}
/p:InstallersTargetStaticFeed=$(InstallersBlobFeedUrl)
/p:InstallersAzureAccountKey=$(dotnetcli-storage-key)
/p:ChecksumsTargetStaticFeed=$(ChecksumsBlobFeedUrl)
/p:ChecksumsAzureAccountKey=$(dotnetclichecksums-storage-key)
/p:PublishToAzureDevOpsNuGetFeeds=true
/p:AzureDevOpsStaticShippingFeed='https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet5/nuget/v3/index.json'
/p:AzureDevOpsStaticShippingFeedKey='$(dn-bot-dnceng-artifact-feeds-rw)'
/p:AzureDevOpsStaticTransportFeed='https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet5-transport/nuget/v3/index.json'
/p:AzureDevOpsStaticTransportFeedKey='$(dn-bot-dnceng-artifact-feeds-rw)'
/p:AzureDevOpsStaticSymbolsFeed='https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet5-symbols/nuget/v3/index.json'
/p:AzureDevOpsStaticSymbolsFeedKey='$(dn-bot-dnceng-artifact-feeds-rw)'
${{ parameters.artifactsPublishingAdditionalParameters }}
- template: ../../steps/promote-build.yml
parameters:
ChannelId: ${{ variables.NetCore_5_Dev_Channel_Id }}

130
eng/common/templates/post-build/channels/netcore-release-30.yml

@ -1,130 +0,0 @@
parameters:
symbolPublishingAdditionalParameters: ''
artifactsPublishingAdditionalParameters: ''
publishInstallersAndChecksums: false
stages:
- stage: NetCore_Release30_Publish
dependsOn: validate
variables:
- template: ../common-variables.yml
displayName: .NET Core 3.0 Release Publishing
jobs:
- template: ../setup-maestro-vars.yml
- job:
displayName: Symbol Publishing
dependsOn: setupMaestroVars
condition: contains(dependencies.setupMaestroVars.outputs['setReleaseVars.InitialChannels'], format('[{0}]', variables.PublicRelease_30_Channel_Id))
variables:
- group: DotNet-Symbol-Server-Pats
pool:
vmImage: 'windows-2019'
steps:
- task: DownloadBuildArtifacts@0
displayName: Download Blob Artifacts
inputs:
artifactName: 'BlobArtifacts'
continueOnError: true
- task: DownloadBuildArtifacts@0
displayName: Download PDB Artifacts
inputs:
artifactName: 'PDBArtifacts'
continueOnError: true
- task: PowerShell@2
displayName: Publish
inputs:
filePath: eng\common\sdk-task.ps1
arguments: -task PublishToSymbolServers -restore -msbuildEngine dotnet
/p:DotNetSymbolServerTokenMsdl=$(microsoft-symbol-server-pat)
/p:DotNetSymbolServerTokenSymWeb=$(symweb-symbol-server-pat)
/p:PDBArtifactsDirectory='$(Build.ArtifactStagingDirectory)/PDBArtifacts/'
/p:BlobBasePath='$(Build.ArtifactStagingDirectory)/BlobArtifacts/'
/p:SymbolPublishingExclusionsFile='$(Build.SourcesDirectory)/eng/SymbolPublishingExclusionsFile.txt'
/p:Configuration=Release
${{ parameters.symbolPublishingAdditionalParameters }}
- job: publish_assets
displayName: Publish Assets
dependsOn: setupMaestroVars
variables:
- group: DotNet-Blob-Feed
- group: AzureDevOps-Artifact-Feeds-Pats
- name: BARBuildId
value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.BARBuildId'] ]
- name: IsStableBuild
value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.IsStableBuild'] ]
condition: contains(dependencies.setupMaestroVars.outputs['setReleaseVars.InitialChannels'], format('[{0}]', variables.PublicRelease_30_Channel_Id))
pool:
vmImage: 'windows-2019'
steps:
- task: DownloadBuildArtifacts@0
displayName: Download Package Artifacts
inputs:
buildType: current
artifactName: PackageArtifacts
- task: DownloadBuildArtifacts@0
displayName: Download Blob Artifacts
inputs:
buildType: current
artifactName: BlobArtifacts
- task: DownloadBuildArtifacts@0
displayName: Download Asset Manifests
inputs:
buildType: current
artifactName: AssetManifests
- task: NuGetToolInstaller@1
displayName: 'Install NuGet.exe'
# This is necessary whenever we want to publish/restore to an AzDO private feed
- task: NuGetAuthenticate@0
displayName: 'Authenticate to AzDO Feeds'
- task: PowerShell@2
displayName: Enable cross-org publishing
inputs:
filePath: eng\common\enable-cross-org-publishing.ps1
arguments: -token $(dn-bot-dnceng-artifact-feeds-rw)
- task: PowerShell@2
displayName: Publish Assets
inputs:
filePath: eng\common\sdk-task.ps1
arguments: -task PublishArtifactsInManifest -restore -msbuildEngine dotnet
/p:ArtifactsCategory=$(_DotNetArtifactsCategory)
/p:IsStableBuild=$(IsStableBuild)
/p:IsInternalBuild=$(IsInternalBuild)
/p:RepositoryName=$(Build.Repository.Name)
/p:CommitSha=$(Build.SourceVersion)
/p:NugetPath=$(NuGetExeToolPath)
/p:AzdoTargetFeedPAT='$(dn-bot-dnceng-universal-packages-rw)'
/p:AzureStorageTargetFeedPAT='$(dotnetfeed-storage-access-key-1)'
/p:BARBuildId=$(BARBuildId)
/p:MaestroApiEndpoint='$(MaestroApiEndPoint)'
/p:BuildAssetRegistryToken='$(MaestroApiAccessToken)'
/p:ManifestsBasePath='$(Build.ArtifactStagingDirectory)/AssetManifests/'
/p:BlobBasePath='$(Build.ArtifactStagingDirectory)/BlobArtifacts/'
/p:PackageBasePath='$(Build.ArtifactStagingDirectory)/PackageArtifacts/'
/p:Configuration=Release
/p:PublishInstallersAndChecksums=${{ parameters.publishInstallersAndChecksums }}
/p:InstallersTargetStaticFeed=$(InstallersBlobFeedUrl)
/p:InstallersAzureAccountKey=$(dotnetcli-storage-key)
/p:ChecksumsTargetStaticFeed=$(ChecksumsBlobFeedUrl)
/p:ChecksumsAzureAccountKey=$(dotnetclichecksums-storage-key)
/p:PublishToAzureDevOpsNuGetFeeds=true
/p:AzureDevOpsStaticShippingFeed='https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet3/nuget/v3/index.json'
/p:AzureDevOpsStaticShippingFeedKey='$(dn-bot-dnceng-artifact-feeds-rw)'
/p:AzureDevOpsStaticTransportFeed='https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet3-transport/nuget/v3/index.json'
/p:AzureDevOpsStaticTransportFeedKey='$(dn-bot-dnceng-artifact-feeds-rw)'
/p:AzureDevOpsStaticSymbolsFeed='https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet3-symbols/nuget/v3/index.json'
/p:AzureDevOpsStaticSymbolsFeedKey='$(dn-bot-dnceng-artifact-feeds-rw)'
${{ parameters.artifactsPublishingAdditionalParameters }}
- template: ../../steps/promote-build.yml
parameters:
ChannelId: ${{ variables.PublicRelease_30_Channel_Id }}

130
eng/common/templates/post-build/channels/netcore-release-31.yml

@ -1,130 +0,0 @@
parameters:
symbolPublishingAdditionalParameters: ''
artifactsPublishingAdditionalParameters: ''
publishInstallersAndChecksums: false
stages:
- stage: NetCore_Release31_Publish
dependsOn: validate
variables:
- template: ../common-variables.yml
displayName: .NET Core 3.1 Release Publishing
jobs:
- template: ../setup-maestro-vars.yml
- job:
displayName: Symbol Publishing
dependsOn: setupMaestroVars
condition: contains(dependencies.setupMaestroVars.outputs['setReleaseVars.InitialChannels'], format('[{0}]', variables.PublicRelease_31_Channel_Id))
variables:
- group: DotNet-Symbol-Server-Pats
pool:
vmImage: 'windows-2019'
steps:
- task: DownloadBuildArtifacts@0
displayName: Download Blob Artifacts
inputs:
artifactName: 'BlobArtifacts'
continueOnError: true
- task: DownloadBuildArtifacts@0
displayName: Download PDB Artifacts
inputs:
artifactName: 'PDBArtifacts'
continueOnError: true
- task: PowerShell@2
displayName: Publish
inputs:
filePath: eng\common\sdk-task.ps1
arguments: -task PublishToSymbolServers -restore -msbuildEngine dotnet
/p:DotNetSymbolServerTokenMsdl=$(microsoft-symbol-server-pat)
/p:DotNetSymbolServerTokenSymWeb=$(symweb-symbol-server-pat)
/p:PDBArtifactsDirectory='$(Build.ArtifactStagingDirectory)/PDBArtifacts/'
/p:BlobBasePath='$(Build.ArtifactStagingDirectory)/BlobArtifacts/'
/p:SymbolPublishingExclusionsFile='$(Build.SourcesDirectory)/eng/SymbolPublishingExclusionsFile.txt'
/p:Configuration=Release
${{ parameters.symbolPublishingAdditionalParameters }}
- job: publish_assets
displayName: Publish Assets
dependsOn: setupMaestroVars
variables:
- group: DotNet-Blob-Feed
- group: AzureDevOps-Artifact-Feeds-Pats
- name: BARBuildId
value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.BARBuildId'] ]
- name: IsStableBuild
value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.IsStableBuild'] ]
condition: contains(dependencies.setupMaestroVars.outputs['setReleaseVars.InitialChannels'], format('[{0}]', variables.PublicRelease_31_Channel_Id))
pool:
vmImage: 'windows-2019'
steps:
- task: DownloadBuildArtifacts@0
displayName: Download Package Artifacts
inputs:
buildType: current
artifactName: PackageArtifacts
- task: DownloadBuildArtifacts@0
displayName: Download Blob Artifacts
inputs:
buildType: current
artifactName: BlobArtifacts
- task: DownloadBuildArtifacts@0
displayName: Download Asset Manifests
inputs:
buildType: current
artifactName: AssetManifests
- task: NuGetToolInstaller@1
displayName: 'Install NuGet.exe'
# This is necessary whenever we want to publish/restore to an AzDO private feed
- task: NuGetAuthenticate@0
displayName: 'Authenticate to AzDO Feeds'
- task: PowerShell@2
displayName: Enable cross-org publishing
inputs:
filePath: eng\common\enable-cross-org-publishing.ps1
arguments: -token $(dn-bot-dnceng-artifact-feeds-rw)
- task: PowerShell@2
displayName: Publish Assets
inputs:
filePath: eng\common\sdk-task.ps1
arguments: -task PublishArtifactsInManifest -restore -msbuildEngine dotnet
/p:ArtifactsCategory=$(_DotNetArtifactsCategory)
/p:IsStableBuild=$(IsStableBuild)
/p:IsInternalBuild=$(IsInternalBuild)
/p:RepositoryName=$(Build.Repository.Name)
/p:CommitSha=$(Build.SourceVersion)
/p:NugetPath=$(NuGetExeToolPath)
/p:AzdoTargetFeedPAT='$(dn-bot-dnceng-universal-packages-rw)'
/p:AzureStorageTargetFeedPAT='$(dotnetfeed-storage-access-key-1)'
/p:BARBuildId=$(BARBuildId)
/p:MaestroApiEndpoint='$(MaestroApiEndPoint)'
/p:BuildAssetRegistryToken='$(MaestroApiAccessToken)'
/p:ManifestsBasePath='$(Build.ArtifactStagingDirectory)/AssetManifests/'
/p:BlobBasePath='$(Build.ArtifactStagingDirectory)/BlobArtifacts/'
/p:PackageBasePath='$(Build.ArtifactStagingDirectory)/PackageArtifacts/'
/p:Configuration=Release
/p:PublishInstallersAndChecksums=${{ parameters.publishInstallersAndChecksums }}
/p:InstallersTargetStaticFeed=$(InstallersBlobFeedUrl)
/p:InstallersAzureAccountKey=$(dotnetcli-storage-key)
/p:ChecksumsTargetStaticFeed=$(ChecksumsBlobFeedUrl)
/p:ChecksumsAzureAccountKey=$(dotnetclichecksums-storage-key)
/p:PublishToAzureDevOpsNuGetFeeds=true
/p:AzureDevOpsStaticShippingFeed='https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet3.1/nuget/v3/index.json'
/p:AzureDevOpsStaticShippingFeedKey='$(dn-bot-dnceng-artifact-feeds-rw)'
/p:AzureDevOpsStaticTransportFeed='https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet3.1-transport/nuget/v3/index.json'
/p:AzureDevOpsStaticTransportFeedKey='$(dn-bot-dnceng-artifact-feeds-rw)'
/p:AzureDevOpsStaticSymbolsFeed='https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet3.1-symbols/nuget/v3/index.json'
/p:AzureDevOpsStaticSymbolsFeedKey='$(dn-bot-dnceng-artifact-feeds-rw)'
${{ parameters.artifactsPublishingAdditionalParameters }}
- template: ../../steps/promote-build.yml
parameters:
ChannelId: ${{ variables.PublicRelease_31_Channel_Id }}

130
eng/common/templates/post-build/channels/netcore-tools-latest.yml

@ -1,130 +0,0 @@
parameters:
symbolPublishingAdditionalParameters: ''
artifactsPublishingAdditionalParameters: ''
publishInstallersAndChecksums: false
stages:
- stage: NetCore_Tools_Latest_Publish
dependsOn: validate
variables:
- template: ../common-variables.yml
displayName: .NET Tools - Latest Publishing
jobs:
- template: ../setup-maestro-vars.yml
- job:
displayName: Symbol Publishing
dependsOn: setupMaestroVars
condition: contains(dependencies.setupMaestroVars.outputs['setReleaseVars.InitialChannels'], format('[{0}]', variables.NetCore_Tools_Latest_Channel_Id))
variables:
- group: DotNet-Symbol-Server-Pats
pool:
vmImage: 'windows-2019'
steps:
- task: DownloadBuildArtifacts@0
displayName: Download Blob Artifacts
inputs:
artifactName: 'BlobArtifacts'
continueOnError: true
- task: DownloadBuildArtifacts@0
displayName: Download PDB Artifacts
inputs:
artifactName: 'PDBArtifacts'
continueOnError: true
- task: PowerShell@2
displayName: Publish
inputs:
filePath: eng\common\sdk-task.ps1
arguments: -task PublishToSymbolServers -restore -msbuildEngine dotnet
/p:DotNetSymbolServerTokenMsdl=$(microsoft-symbol-server-pat)
/p:DotNetSymbolServerTokenSymWeb=$(symweb-symbol-server-pat)
/p:PDBArtifactsDirectory='$(Build.ArtifactStagingDirectory)/PDBArtifacts/'
/p:BlobBasePath='$(Build.ArtifactStagingDirectory)/BlobArtifacts/'
/p:SymbolPublishingExclusionsFile='$(Build.SourcesDirectory)/eng/SymbolPublishingExclusionsFile.txt'
/p:Configuration=Release
${{ parameters.symbolPublishingAdditionalParameters }}
- job: publish_assets
displayName: Publish Assets
dependsOn: setupMaestroVars
variables:
- group: DotNet-Blob-Feed
- group: AzureDevOps-Artifact-Feeds-Pats
- name: BARBuildId
value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.BARBuildId'] ]
- name: IsStableBuild
value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.IsStableBuild'] ]
condition: contains(dependencies.setupMaestroVars.outputs['setReleaseVars.InitialChannels'], format('[{0}]', variables.NetCore_Tools_Latest_Channel_Id))
pool:
vmImage: 'windows-2019'
steps:
- task: DownloadBuildArtifacts@0
displayName: Download Package Artifacts
inputs:
buildType: current
artifactName: PackageArtifacts
- task: DownloadBuildArtifacts@0
displayName: Download Blob Artifacts
inputs:
buildType: current
artifactName: BlobArtifacts
- task: DownloadBuildArtifacts@0
displayName: Download Asset Manifests
inputs:
buildType: current
artifactName: AssetManifests
- task: NuGetToolInstaller@1
displayName: 'Install NuGet.exe'
# This is necessary whenever we want to publish/restore to an AzDO private feed
- task: NuGetAuthenticate@0
displayName: 'Authenticate to AzDO Feeds'
- task: PowerShell@2
displayName: Enable cross-org publishing
inputs:
filePath: eng\common\enable-cross-org-publishing.ps1
arguments: -token $(dn-bot-dnceng-artifact-feeds-rw)
- task: PowerShell@2
displayName: Publish Assets
inputs:
filePath: eng\common\sdk-task.ps1
arguments: -task PublishArtifactsInManifest -restore -msbuildEngine dotnet
/p:ArtifactsCategory=$(_DotNetArtifactsCategory)
/p:IsStableBuild=$(IsStableBuild)
/p:IsInternalBuild=$(IsInternalBuild)
/p:RepositoryName=$(Build.Repository.Name)
/p:CommitSha=$(Build.SourceVersion)
/p:NugetPath=$(NuGetExeToolPath)
/p:AzdoTargetFeedPAT='$(dn-bot-dnceng-universal-packages-rw)'
/p:AzureStorageTargetFeedPAT='$(dotnetfeed-storage-access-key-1)'
/p:BARBuildId=$(BARBuildId)
/p:MaestroApiEndpoint='$(MaestroApiEndPoint)'
/p:BuildAssetRegistryToken='$(MaestroApiAccessToken)'
/p:ManifestsBasePath='$(Build.ArtifactStagingDirectory)/AssetManifests/'
/p:BlobBasePath='$(Build.ArtifactStagingDirectory)/BlobArtifacts/'
/p:PackageBasePath='$(Build.ArtifactStagingDirectory)/PackageArtifacts/'
/p:Configuration=Release
/p:PublishInstallersAndChecksums=${{ parameters.publishInstallersAndChecksums }}
/p:InstallersTargetStaticFeed=$(InstallersBlobFeedUrl)
/p:InstallersAzureAccountKey=$(dotnetcli-storage-key)
/p:ChecksumsTargetStaticFeed=$(ChecksumsBlobFeedUrl)
/p:ChecksumsAzureAccountKey=$(dotnetclichecksums-storage-key)
/p:PublishToAzureDevOpsNuGetFeeds=true
/p:AzureDevOpsStaticShippingFeed='https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools/nuget/v3/index.json'
/p:AzureDevOpsStaticShippingFeedKey='$(dn-bot-dnceng-artifact-feeds-rw)'
/p:AzureDevOpsStaticTransportFeed='https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools/nuget/v3/index.json'
/p:AzureDevOpsStaticTransportFeedKey='$(dn-bot-dnceng-artifact-feeds-rw)'
/p:AzureDevOpsStaticSymbolsFeed='https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools-symbols/nuget/v3/index.json'
/p:AzureDevOpsStaticSymbolsFeedKey='$(dn-bot-dnceng-artifact-feeds-rw)'
${{ parameters.artifactsPublishingAdditionalParameters }}
- template: ../../steps/promote-build.yml
parameters:
ChannelId: ${{ variables.NetCore_Tools_Latest_Channel_Id }}

95
eng/common/templates/post-build/channels/netcore-tools-validation.yml

@ -1,95 +0,0 @@
parameters:
artifactsPublishingAdditionalParameters: ''
publishInstallersAndChecksums: false
stages:
- stage: PVR_Publish
dependsOn: validate
variables:
- template: ../common-variables.yml
displayName: .NET Tools - Validation Publishing
jobs:
- template: ../setup-maestro-vars.yml
- job: publish_assets
displayName: Publish Assets
dependsOn: setupMaestroVars
variables:
- group: DotNet-Blob-Feed
- group: AzureDevOps-Artifact-Feeds-Pats
- name: BARBuildId
value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.BARBuildId'] ]
- name: IsStableBuild
value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.IsStableBuild'] ]
condition: contains(dependencies.setupMaestroVars.outputs['setReleaseVars.InitialChannels'], format('[{0}]', variables.NetCore_Tools_Validation_Channel_Id))
pool:
vmImage: 'windows-2019'
steps:
- task: DownloadBuildArtifacts@0
displayName: Download Package Artifacts
inputs:
buildType: current
artifactName: PackageArtifacts
- task: DownloadBuildArtifacts@0
displayName: Download Blob Artifacts
inputs:
buildType: current
artifactName: BlobArtifacts
- task: DownloadBuildArtifacts@0
displayName: Download Asset Manifests
inputs:
buildType: current
artifactName: AssetManifests
- task: NuGetToolInstaller@1
displayName: 'Install NuGet.exe'
# This is necessary whenever we want to publish/restore to an AzDO private feed
- task: NuGetAuthenticate@0
displayName: 'Authenticate to AzDO Feeds'
- task: PowerShell@2
displayName: Enable cross-org publishing
inputs:
filePath: eng\common\enable-cross-org-publishing.ps1
arguments: -token $(dn-bot-dnceng-artifact-feeds-rw)
- task: PowerShell@2
displayName: Publish Assets
inputs:
filePath: eng\common\sdk-task.ps1
arguments: -task PublishArtifactsInManifest -restore -msbuildEngine dotnet
/p:ArtifactsCategory=$(_DotNetValidationArtifactsCategory)
/p:IsStableBuild=$(IsStableBuild)
/p:IsInternalBuild=$(IsInternalBuild)
/p:RepositoryName=$(Build.Repository.Name)
/p:CommitSha=$(Build.SourceVersion)
/p:NugetPath=$(NuGetExeToolPath)
/p:AzdoTargetFeedPAT='$(dn-bot-dnceng-universal-packages-rw)'
/p:AzureStorageTargetFeedPAT='$(dotnetfeed-storage-access-key-1)'
/p:BARBuildId=$(BARBuildId)
/p:MaestroApiEndpoint='$(MaestroApiEndPoint)'
/p:BuildAssetRegistryToken='$(MaestroApiAccessToken)'
/p:ManifestsBasePath='$(Build.ArtifactStagingDirectory)/AssetManifests/'
/p:BlobBasePath='$(Build.ArtifactStagingDirectory)/BlobArtifacts/'
/p:PackageBasePath='$(Build.ArtifactStagingDirectory)/PackageArtifacts/'
/p:Configuration=Release
/p:PublishInstallersAndChecksums=${{ parameters.publishInstallersAndChecksums }}
/p:InstallersTargetStaticFeed=$(InstallersBlobFeedUrl)
/p:InstallersAzureAccountKey=$(dotnetcli-storage-key)
/p:ChecksumsTargetStaticFeed=$(ChecksumsBlobFeedUrl)
/p:ChecksumsAzureAccountKey=$(dotnetclichecksums-storage-key)
/p:PublishToAzureDevOpsNuGetFeeds=true
/p:AzureDevOpsStaticShippingFeed='https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools/nuget/v3/index.json'
/p:AzureDevOpsStaticShippingFeedKey='$(dn-bot-dnceng-artifact-feeds-rw)'
/p:AzureDevOpsStaticTransportFeed='https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools/nuget/v3/index.json'
/p:AzureDevOpsStaticTransportFeedKey='$(dn-bot-dnceng-artifact-feeds-rw)'
/p:AzureDevOpsStaticSymbolsFeed='https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools-symbols/nuget/v3/index.json'
/p:AzureDevOpsStaticSymbolsFeedKey='$(dn-bot-dnceng-artifact-feeds-rw)'
${{ parameters.artifactsPublishingAdditionalParameters }}
- template: ../../steps/promote-build.yml
parameters:
ChannelId: ${{ variables.NetCore_Tools_Validation_Channel_Id }}

43
eng/common/templates/post-build/common-variables.yml

@ -1,29 +1,31 @@
variables: variables:
- group: Publish-Build-Assets - group: AzureDevOps-Artifact-Feeds-Pats
- group: DotNet-Blob-Feed
- group: DotNet-DotNetCli-Storage - group: DotNet-DotNetCli-Storage
- group: DotNet-MSRC-Storage - group: DotNet-MSRC-Storage
- group: Publish-Build-Assets
# .NET Core 3.1 Dev # .NET Core 3.1 Dev
- name: PublicDevRelease_31_Channel_Id - name: PublicDevRelease_31_Channel_Id
value: 128 value: 128
# .NET Core 5 Dev # .NET 5 Dev
- name: NetCore_5_Dev_Channel_Id - name: Net_5_Dev_Channel_Id
value: 131 value: 131
# .NET Tools - Validation # .NET Eng - Validation
- name: NetCore_Tools_Validation_Channel_Id - name: Net_Eng_Validation_Channel_Id
value: 9 value: 9
# .NET Tools - Latest # .NET Eng - Latest
- name: NetCore_Tools_Latest_Channel_Id - name: Net_Eng_Latest_Channel_Id
value: 2 value: 2
# .NET 3 Tools - Validation # .NET 3 Eng - Validation
- name: NETCore_3_Tools_Validation_Channel_Id - name: NET_3_Eng_Validation_Channel_Id
value: 390 value: 390
# .NET 3 Tools - Latest # .NET 3 Eng
- name: NetCore_3_Tools_Channel_Id - name: NetCore_3_Tools_Channel_Id
value: 344 value: 344
@ -39,6 +41,18 @@ variables:
- name: PublicRelease_31_Channel_Id - name: PublicRelease_31_Channel_Id
value: 129 value: 129
# General Testing
- name: GeneralTesting_Channel_Id
value: 529
# .NET Core 3.1 Blazor Features
- name: NetCore_31_Blazor_Features_Channel_Id
value: 531
# .NET Core Experimental
- name: NetCore_Experimental_Channel_Id
value: 562
# Whether the build is internal or not # Whether the build is internal or not
- name: IsInternalBuild - name: IsInternalBuild
value: ${{ and(ne(variables['System.TeamProject'], 'public'), contains(variables['Build.SourceBranch'], 'internal')) }} value: ${{ and(ne(variables['System.TeamProject'], 'public'), contains(variables['Build.SourceBranch'], 'internal')) }}
@ -76,3 +90,10 @@ variables:
value: https://dotnetclimsrc.blob.core.windows.net/dotnet/index.json value: https://dotnetclimsrc.blob.core.windows.net/dotnet/index.json
- name: InternalInstallersBlobFeedKey - name: InternalInstallersBlobFeedKey
value: $(dotnetclimsrc-access-key) value: $(dotnetclimsrc-access-key)
# Skip component governance and codesign validation for SDL. These jobs
# create no content.
- name: skipComponentGovernanceDetection
value: true
- name: runCodesignValidationInjection
value: false

23
eng/common/templates/post-build/darc-gather-drop.yml

@ -1,23 +0,0 @@
parameters:
ChannelId: 0
jobs:
- job: gatherDrop
displayName: Gather Drop
dependsOn: setupMaestroVars
condition: contains(dependencies.setupMaestroVars.outputs['setReleaseVars.InitialChannels'], format('[{0}]', ${{ parameters.ChannelId }}))
variables:
- name: BARBuildId
value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.BARBuildId'] ]
pool:
vmImage: 'windows-2019'
steps:
- task: PowerShell@2
displayName: Darc gather-drop
inputs:
filePath: $(Build.SourcesDirectory)/eng/common/post-build/darc-gather-drop.ps1
arguments: -BarBuildId $(BARBuildId)
-DropLocation $(Agent.BuildDirectory)/Temp/Drop/
-MaestroApiAccessToken $(MaestroApiAccessToken)
-MaestroApiEndPoint $(MaestroApiEndPoint)
-MaestroApiVersion $(MaestroApiVersion)

407
eng/common/templates/post-build/post-build.yml

@ -8,6 +8,7 @@ parameters:
enable: false enable: false
continueOnError: false continueOnError: false
params: '' params: ''
artifactNames: ''
# These parameters let the user customize the call to sdk-task.ps1 for publishing # These parameters let the user customize the call to sdk-task.ps1 for publishing
# symbols & general artifacts as well as for signing validation # symbols & general artifacts as well as for signing validation
@ -16,132 +17,366 @@ parameters:
signingValidationAdditionalParameters: '' signingValidationAdditionalParameters: ''
# Which stages should finish execution before post-build stages start # Which stages should finish execution before post-build stages start
dependsOn: [build] validateDependsOn:
- build
publishDependsOn:
- Validate
# Channel ID's instantiated in this file.
# When adding a new channel implementation the call to `check-channel-consistency.ps1`
# needs to be updated with the new channel ID
NetEngLatestChannelId: 2
NetEngValidationChannelId: 9
NetDev5ChannelId: 131
GeneralTestingChannelId: 529
NETCoreToolingDevChannelId: 548
NETCoreToolingReleaseChannelId: 549
NETInternalToolingChannelId: 551
NETCoreExperimentalChannelId: 562
NetEngServicesIntChannelId: 678
NetEngServicesProdChannelId: 679
Net5Preview2ChannelId: 738
Net5Preview3ChannelId: 739
NetCoreSDK313xxChannelId: 759
NetCoreSDK313xxInternalChannelId: 760
stages: stages:
- stage: validate - stage: Validate
dependsOn: ${{ parameters.dependsOn }} dependsOn: ${{ parameters.validateDependsOn }}
displayName: Validate displayName: Validate
variables:
- template: common-variables.yml
jobs: jobs:
- ${{ if eq(parameters.enableNugetValidation, 'true') }}: - template: setup-maestro-vars.yml
- job:
displayName: NuGet Validation - job:
pool: displayName: Post-build Checks
vmImage: 'windows-2019' dependsOn: setupMaestroVars
steps: variables:
- task: DownloadBuildArtifacts@0 - name: InitialChannels
displayName: Download Package Artifacts value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.InitialChannels'] ]
inputs: - name: PromoteToMaestroChannelId
buildType: current value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.PromoteToMaestroChannelId'] ]
artifactName: PackageArtifacts pool:
vmImage: 'windows-2019'
- task: PowerShell@2 steps:
displayName: Validate - task: PowerShell@2
inputs: displayName: Maestro Channels Consistency
filePath: $(Build.SourcesDirectory)/eng/common/post-build/nuget-validation.ps1 inputs:
arguments: -PackagesPath $(Build.ArtifactStagingDirectory)/PackageArtifacts/ filePath: $(Build.SourcesDirectory)/eng/common/post-build/check-channel-consistency.ps1
-ToolDestinationPath $(Agent.BuildDirectory)/Extract/ arguments: -PromoteToChannels "$(InitialChannels)[$(PromoteToMaestroChannelId)]"
-AvailableChannelIds ${{parameters.NetEngLatestChannelId}},${{parameters.NetEngValidationChannelId}},${{parameters.NetDev5ChannelId}},${{parameters.GeneralTestingChannelId}},${{parameters.NETCoreToolingDevChannelId}},${{parameters.NETCoreToolingReleaseChannelId}},${{parameters.NETInternalToolingChannelId}},${{parameters.NETCoreExperimentalChannelId}},${{parameters.NetEngServicesIntChannelId}},${{parameters.NetEngServicesProdChannelId}},${{parameters.Net5Preview2ChannelId}},${{parameters.Net5Preview3ChannelId}},${{parameters.NetCoreSDK313xxChannelId}},${{parameters.NetCoreSDK313xxInternalChannelId}}
- ${{ if eq(parameters.enableSigningValidation, 'true') }}:
- job: - job:
displayName: Signing Validation displayName: NuGet Validation
pool: dependsOn: setupMaestroVars
vmImage: 'windows-2019' condition: eq( ${{ parameters.enableNugetValidation }}, 'true')
steps: pool:
- task: DownloadBuildArtifacts@0 vmImage: 'windows-2019'
displayName: Download Package Artifacts variables:
inputs: - name: AzDOProjectName
buildType: current value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOProjectName'] ]
artifactName: PackageArtifacts - name: AzDOPipelineId
value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOPipelineId'] ]
- task: PowerShell@2 - name: AzDOBuildId
displayName: Validate value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOBuildId'] ]
inputs: steps:
filePath: eng\common\sdk-task.ps1 - task: DownloadBuildArtifacts@0
arguments: -task SigningValidation -restore -msbuildEngine dotnet displayName: Download Package Artifacts
/p:PackageBasePath='$(Build.ArtifactStagingDirectory)/PackageArtifacts' inputs:
/p:SignCheckExclusionsFile='$(Build.SourcesDirectory)/eng/SignCheckExclusionsFile.txt' buildType: specific
/p:Configuration=Release buildVersionToDownload: specific
${{ parameters.signingValidationAdditionalParameters }} project: $(AzDOProjectName)
pipeline: $(AzDOPipelineId)
- ${{ if eq(parameters.enableSourceLinkValidation, 'true') }}: buildId: $(AzDOBuildId)
- job: artifactName: PackageArtifacts
displayName: SourceLink Validation
variables: - task: PowerShell@2
- template: common-variables.yml displayName: Validate
pool: inputs:
vmImage: 'windows-2019' filePath: $(Build.SourcesDirectory)/eng/common/post-build/nuget-validation.ps1
steps: arguments: -PackagesPath $(Build.ArtifactStagingDirectory)/PackageArtifacts/
- task: DownloadBuildArtifacts@0 -ToolDestinationPath $(Agent.BuildDirectory)/Extract/
displayName: Download Blob Artifacts
inputs: - job:
buildType: current displayName: Signing Validation
artifactName: BlobArtifacts dependsOn: setupMaestroVars
condition: eq( ${{ parameters.enableSigningValidation }}, 'true')
- task: PowerShell@2 variables:
displayName: Validate - template: common-variables.yml
inputs: - name: AzDOProjectName
filePath: $(Build.SourcesDirectory)/eng/common/post-build/sourcelink-validation.ps1 value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOProjectName'] ]
arguments: -InputPath $(Build.ArtifactStagingDirectory)/BlobArtifacts/ - name: AzDOPipelineId
-ExtractPath $(Agent.BuildDirectory)/Extract/ value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOPipelineId'] ]
-GHRepoName $(Build.Repository.Name) - name: AzDOBuildId
-GHCommit $(Build.SourceVersion) value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOBuildId'] ]
-SourcelinkCliVersion $(SourceLinkCLIVersion) pool:
continueOnError: true vmImage: 'windows-2019'
steps:
- ${{ if eq(parameters.SDLValidationParameters.enable, 'true') }}: - task: DownloadBuildArtifacts@0
- template: /eng/common/templates/job/execute-sdl.yml displayName: Download Package Artifacts
parameters: inputs:
additionalParameters: ${{ parameters.SDLValidationParameters.params }} buildType: specific
continueOnError: ${{ parameters.SDLValidationParameters.continueOnError }} buildVersionToDownload: specific
project: $(AzDOProjectName)
- template: \eng\common\templates\post-build\channels\netcore-dev-5.yml pipeline: $(AzDOPipelineId)
buildId: $(AzDOBuildId)
artifactName: PackageArtifacts
# This is necessary whenever we want to publish/restore to an AzDO private feed
# Since sdk-task.ps1 tries to restore packages we need to do this authentication here
# otherwise it'll complain about accessing a private feed.
- task: NuGetAuthenticate@0
displayName: 'Authenticate to AzDO Feeds'
- task: PowerShell@2
displayName: Enable cross-org publishing
inputs:
filePath: eng\common\enable-cross-org-publishing.ps1
arguments: -token $(dn-bot-dnceng-artifact-feeds-rw)
- task: PowerShell@2
displayName: Validate
inputs:
filePath: eng\common\sdk-task.ps1
arguments: -task SigningValidation -restore -msbuildEngine dotnet
/p:PackageBasePath='$(Build.ArtifactStagingDirectory)/PackageArtifacts'
/p:SignCheckExclusionsFile='$(Build.SourcesDirectory)/eng/SignCheckExclusionsFile.txt'
${{ parameters.signingValidationAdditionalParameters }}
- template: ../steps/publish-logs.yml
parameters:
StageLabel: 'Validation'
JobLabel: 'Signing'
- job:
displayName: SourceLink Validation
dependsOn: setupMaestroVars
condition: eq( ${{ parameters.enableSourceLinkValidation }}, 'true')
variables:
- template: common-variables.yml
- name: AzDOProjectName
value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOProjectName'] ]
- name: AzDOPipelineId
value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOPipelineId'] ]
- name: AzDOBuildId
value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOBuildId'] ]
pool:
vmImage: 'windows-2019'
steps:
- task: DownloadBuildArtifacts@0
displayName: Download Blob Artifacts
inputs:
buildType: specific
buildVersionToDownload: specific
project: $(AzDOProjectName)
pipeline: $(AzDOPipelineId)
buildId: $(AzDOBuildId)
artifactName: BlobArtifacts
- task: PowerShell@2
displayName: Validate
inputs:
filePath: $(Build.SourcesDirectory)/eng/common/post-build/sourcelink-validation.ps1
arguments: -InputPath $(Build.ArtifactStagingDirectory)/BlobArtifacts/
-ExtractPath $(Agent.BuildDirectory)/Extract/
-GHRepoName $(Build.Repository.Name)
-GHCommit $(Build.SourceVersion)
-SourcelinkCliVersion $(SourceLinkCLIVersion)
continueOnError: true
- template: /eng/common/templates/job/execute-sdl.yml
parameters:
enable: ${{ parameters.SDLValidationParameters.enable }}
dependsOn: setupMaestroVars
additionalParameters: ${{ parameters.SDLValidationParameters.params }}
continueOnError: ${{ parameters.SDLValidationParameters.continueOnError }}
artifactNames: ${{ parameters.SDLValidationParameters.artifactNames }}
- template: \eng\common\templates\post-build\channels\generic-public-channel.yml
parameters: parameters:
symbolPublishingAdditionalParameters: ${{ parameters.symbolPublishingAdditionalParameters }}
artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }} artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }}
dependsOn: ${{ parameters.publishDependsOn }}
publishInstallersAndChecksums: ${{ parameters.publishInstallersAndChecksums }} publishInstallersAndChecksums: ${{ parameters.publishInstallersAndChecksums }}
symbolPublishingAdditionalParameters: ${{ parameters.symbolPublishingAdditionalParameters }}
stageName: 'NetCore_Dev5_Publish'
channelName: '.NET 5 Dev'
akaMSChannelName: 'net5/dev'
channelId: ${{ parameters.NetDev5ChannelId }}
transportFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet5-transport/nuget/v3/index.json'
shippingFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet5/nuget/v3/index.json'
symbolsFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet5-symbols/nuget/v3/index.json'
- template: \eng\common\templates\post-build\channels\netcore-dev-31.yml - template: \eng\common\templates\post-build\channels\generic-public-channel.yml
parameters: parameters:
symbolPublishingAdditionalParameters: ${{ parameters.symbolPublishingAdditionalParameters }}
artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }} artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }}
dependsOn: ${{ parameters.publishDependsOn }}
publishInstallersAndChecksums: ${{ parameters.publishInstallersAndChecksums }} publishInstallersAndChecksums: ${{ parameters.publishInstallersAndChecksums }}
symbolPublishingAdditionalParameters: ${{ parameters.symbolPublishingAdditionalParameters }}
stageName: 'Net5_Preview2_Publish'
channelName: '.NET 5 Preview 2'
akaMSChannelName: 'net5/preview2'
channelId: ${{ parameters.Net5Preview2ChannelId }}
transportFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet5-transport/nuget/v3/index.json'
shippingFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet5/nuget/v3/index.json'
symbolsFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet5-symbols/nuget/v3/index.json'
- template: \eng\common\templates\post-build\channels\netcore-tools-latest.yml - template: \eng\common\templates\post-build\channels\generic-public-channel.yml
parameters: parameters:
artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }}
dependsOn: ${{ parameters.publishDependsOn }}
publishInstallersAndChecksums: ${{ parameters.publishInstallersAndChecksums }}
symbolPublishingAdditionalParameters: ${{ parameters.symbolPublishingAdditionalParameters }} symbolPublishingAdditionalParameters: ${{ parameters.symbolPublishingAdditionalParameters }}
stageName: 'Net5_Preview3_Publish'
channelName: '.NET 5 Preview 3'
akaMSChannelName: 'net5/preview3'
channelId: ${{ parameters.Net5Preview3ChannelId }}
transportFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet5-transport/nuget/v3/index.json'
shippingFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet5/nuget/v3/index.json'
symbolsFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet5-symbols/nuget/v3/index.json'
- template: \eng\common\templates\post-build\channels\generic-public-channel.yml
parameters:
artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }} artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }}
dependsOn: ${{ parameters.publishDependsOn }}
publishInstallersAndChecksums: ${{ parameters.publishInstallersAndChecksums }} publishInstallersAndChecksums: ${{ parameters.publishInstallersAndChecksums }}
symbolPublishingAdditionalParameters: ${{ parameters.symbolPublishingAdditionalParameters }}
stageName: 'Net_Eng_Latest_Publish'
channelName: '.NET Eng - Latest'
akaMSChannelName: 'eng/daily'
channelId: ${{ parameters.NetEngLatestChannelId }}
transportFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-eng/nuget/v3/index.json'
shippingFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-eng/nuget/v3/index.json'
symbolsFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-eng-symbols/nuget/v3/index.json'
- template: \eng\common\templates\post-build\channels\netcore-tools-validation.yml - template: \eng\common\templates\post-build\channels\generic-public-channel.yml
parameters: parameters:
artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }} artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }}
dependsOn: ${{ parameters.publishDependsOn }}
publishInstallersAndChecksums: ${{ parameters.publishInstallersAndChecksums }} publishInstallersAndChecksums: ${{ parameters.publishInstallersAndChecksums }}
symbolPublishingAdditionalParameters: ${{ parameters.symbolPublishingAdditionalParameters }}
stageName: 'Net_Eng_Validation_Publish'
channelName: '.NET Eng - Validation'
akaMSChannelName: 'eng/validation'
channelId: ${{ parameters.NetEngValidationChannelId }}
transportFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-eng/nuget/v3/index.json'
shippingFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-eng/nuget/v3/index.json'
symbolsFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-eng-symbols/nuget/v3/index.json'
- template: \eng\common\templates\post-build\channels\netcore-3-tools-validation.yml - template: \eng\common\templates\post-build\channels\generic-public-channel.yml
parameters: parameters:
artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }} artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }}
dependsOn: ${{ parameters.publishDependsOn }}
publishInstallersAndChecksums: ${{ parameters.publishInstallersAndChecksums }} publishInstallersAndChecksums: ${{ parameters.publishInstallersAndChecksums }}
symbolPublishingAdditionalParameters: ${{ parameters.symbolPublishingAdditionalParameters }}
stageName: 'General_Testing_Publish'
channelName: 'General Testing'
akaMSChannelName: 'generaltesting'
channelId: ${{ parameters.GeneralTestingChannelId }}
transportFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/general-testing/nuget/v3/index.json'
shippingFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/general-testing/nuget/v3/index.json'
symbolsFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/general-testing-symbols/nuget/v3/index.json'
- template: \eng\common\templates\post-build\channels\netcore-3-tools.yml - template: \eng\common\templates\post-build\channels\generic-public-channel.yml
parameters: parameters:
artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }}
dependsOn: ${{ parameters.publishDependsOn }}
publishInstallersAndChecksums: ${{ parameters.publishInstallersAndChecksums }}
symbolPublishingAdditionalParameters: ${{ parameters.symbolPublishingAdditionalParameters }} symbolPublishingAdditionalParameters: ${{ parameters.symbolPublishingAdditionalParameters }}
stageName: 'NETCore_Tooling_Dev_Publishing'
channelName: '.NET Core Tooling Dev'
channelId: ${{ parameters.NETCoreToolingDevChannelId }}
transportFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools/nuget/v3/index.json'
shippingFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools/nuget/v3/index.json'
symbolsFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools-symbols/nuget/v3/index.json'
- template: \eng\common\templates\post-build\channels\generic-public-channel.yml
parameters:
artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }} artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }}
dependsOn: ${{ parameters.publishDependsOn }}
publishInstallersAndChecksums: ${{ parameters.publishInstallersAndChecksums }} publishInstallersAndChecksums: ${{ parameters.publishInstallersAndChecksums }}
symbolPublishingAdditionalParameters: ${{ parameters.symbolPublishingAdditionalParameters }}
stageName: 'NETCore_Tooling_Release_Publishing'
channelName: '.NET Core Tooling Release'
channelId: ${{ parameters.NETCoreToolingReleaseChannelId }}
transportFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools/nuget/v3/index.json'
shippingFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools/nuget/v3/index.json'
symbolsFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools-symbols/nuget/v3/index.json'
- template: \eng\common\templates\post-build\channels\netcore-release-30.yml - template: \eng\common\templates\post-build\channels\generic-internal-channel.yml
parameters: parameters:
artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }}
dependsOn: ${{ parameters.publishDependsOn }}
publishInstallersAndChecksums: ${{ parameters.publishInstallersAndChecksums }}
symbolPublishingAdditionalParameters: ${{ parameters.symbolPublishingAdditionalParameters }} symbolPublishingAdditionalParameters: ${{ parameters.symbolPublishingAdditionalParameters }}
stageName: 'NET_Internal_Tooling_Publishing'
channelName: '.NET Internal Tooling'
channelId: ${{ parameters.NETInternalToolingChannelId }}
transportFeed: 'https://pkgs.dev.azure.com/dnceng/internal/_packaging/dotnet-tools-internal/nuget/v3/index.json'
shippingFeed: 'https://pkgs.dev.azure.com/dnceng/internal/_packaging/dotnet-tools-internal/nuget/v3/index.json'
symbolsFeed: 'https://pkgs.dev.azure.com/dnceng/internal/_packaging/dotnet-tools-internal-symbols/nuget/v3/index.json'
- template: \eng\common\templates\post-build\channels\generic-public-channel.yml
parameters:
artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }} artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }}
dependsOn: ${{ parameters.publishDependsOn }}
publishInstallersAndChecksums: ${{ parameters.publishInstallersAndChecksums }} publishInstallersAndChecksums: ${{ parameters.publishInstallersAndChecksums }}
symbolPublishingAdditionalParameters: ${{ parameters.symbolPublishingAdditionalParameters }}
stageName: 'NETCore_Experimental_Publishing'
channelName: '.NET Core Experimental'
channelId: ${{ parameters.NETCoreExperimentalChannelId }}
transportFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-experimental/nuget/v3/index.json'
shippingFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-experimental/nuget/v3/index.json'
symbolsFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-experimental-symbols/nuget/v3/index.json'
- template: \eng\common\templates\post-build\channels\netcore-release-31.yml - template: \eng\common\templates\post-build\channels\generic-public-channel.yml
parameters: parameters:
artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }}
dependsOn: ${{ parameters.publishDependsOn }}
publishInstallersAndChecksums: ${{ parameters.publishInstallersAndChecksums }}
symbolPublishingAdditionalParameters: ${{ parameters.symbolPublishingAdditionalParameters }} symbolPublishingAdditionalParameters: ${{ parameters.symbolPublishingAdditionalParameters }}
stageName: 'Net_Eng_Services_Int_Publish'
channelName: '.NET Eng Services - Int'
channelId: ${{ parameters.NetEngServicesIntChannelId }}
transportFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-eng/nuget/v3/index.json'
shippingFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-eng/nuget/v3/index.json'
symbolsFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-eng-symbols/nuget/v3/index.json'
- template: \eng\common\templates\post-build\channels\generic-public-channel.yml
parameters:
artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }} artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }}
dependsOn: ${{ parameters.publishDependsOn }}
publishInstallersAndChecksums: ${{ parameters.publishInstallersAndChecksums }} publishInstallersAndChecksums: ${{ parameters.publishInstallersAndChecksums }}
symbolPublishingAdditionalParameters: ${{ parameters.symbolPublishingAdditionalParameters }}
stageName: 'Net_Eng_Services_Prod_Publish'
channelName: '.NET Eng Services - Prod'
channelId: ${{ parameters.NetEngServicesProdChannelId }}
transportFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-eng/nuget/v3/index.json'
shippingFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-eng/nuget/v3/index.json'
symbolsFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-eng-symbols/nuget/v3/index.json'
- template: \eng\common\templates\post-build\channels\netcore-internal-30.yml - template: \eng\common\templates\post-build\channels\generic-public-channel.yml
parameters: parameters:
artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }}
dependsOn: ${{ parameters.publishDependsOn }}
publishInstallersAndChecksums: ${{ parameters.publishInstallersAndChecksums }}
symbolPublishingAdditionalParameters: ${{ parameters.symbolPublishingAdditionalParameters }} symbolPublishingAdditionalParameters: ${{ parameters.symbolPublishingAdditionalParameters }}
stageName: 'NETCore_SDK_313xx_Publishing'
channelName: '.NET Core SDK 3.1.3xx'
channelId: ${{ parameters.NetCoreSDK313xxChannelId }}
transportFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet3.1-transport/nuget/v3/index.json'
shippingFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet3.1/nuget/v3/index.json'
symbolsFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet3.1-symbols/nuget/v3/index.json'
- template: \eng\common\templates\post-build\channels\generic-internal-channel.yml
parameters:
artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }} artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }}
dependsOn: ${{ parameters.publishDependsOn }}
publishInstallersAndChecksums: ${{ parameters.publishInstallersAndChecksums }}
symbolPublishingAdditionalParameters: ${{ parameters.symbolPublishingAdditionalParameters }}
stageName: 'NETCore_SDK_313xx_Internal_Publishing'
channelName: '.NET Core SDK 3.1.3xx Internal'
channelId: ${{ parameters.NetCoreSDK313xxInternalChannelId }}
transportFeed: 'https://pkgs.dev.azure.com/dnceng/_packaging/dotnet3.1-internal-transport/nuget/v3/index.json'
shippingFeed: 'https://pkgs.dev.azure.com/dnceng/_packaging/dotnet3.1-internal/nuget/v3/index.json'
symbolsFeed: 'https://pkgs.dev.azure.com/dnceng/_packaging/dotnet3.1-internal-symbols/nuget/v3/index.json'

25
eng/common/templates/post-build/promote-build.yml

@ -1,25 +0,0 @@
parameters:
ChannelId: 0
jobs:
- job:
displayName: Promote Build
dependsOn: setupMaestroVars
condition: contains(dependencies.setupMaestroVars.outputs['setReleaseVars.InitialChannels'], format('[{0}]', ${{ parameters.ChannelId }}))
variables:
- name: BARBuildId
value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.BARBuildId'] ]
- name: ChannelId
value: ${{ parameters.ChannelId }}
pool:
vmImage: 'windows-2019'
steps:
- task: PowerShell@2
displayName: Add Build to Channel
inputs:
filePath: $(Build.SourcesDirectory)/eng/common/post-build/promote-build.ps1
arguments: -BuildId $(BARBuildId)
-ChannelId $(ChannelId)
-MaestroApiAccessToken $(MaestroApiAccessToken)
-MaestroApiEndPoint $(MaestroApiEndPoint)
-MaestroApiVersion $(MaestroApiVersion)

67
eng/common/templates/post-build/setup-maestro-vars.yml

@ -1,11 +1,20 @@
jobs: jobs:
- job: setupMaestroVars - job: setupMaestroVars
displayName: Setup Maestro Vars displayName: Setup Maestro Vars
variables:
- template: common-variables.yml
- name: BuildId
value: $[ coalesce(variables.BARBuildId, 0) ]
- name: PromoteToChannelId
value: $[ coalesce(variables.PromoteToMaestroChannelId, 0) ]
pool: pool:
vmImage: 'windows-2019' vmImage: 'windows-2019'
steps: steps:
- checkout: none
- task: DownloadBuildArtifacts@0 - task: DownloadBuildArtifacts@0
displayName: Download Release Configs displayName: Download Release Configs
condition: eq(variables.PromoteToChannelId, 0)
inputs: inputs:
buildType: current buildType: current
artifactName: ReleaseConfigs artifactName: ReleaseConfigs
@ -14,5 +23,59 @@ jobs:
name: setReleaseVars name: setReleaseVars
displayName: Set Release Configs Vars displayName: Set Release Configs Vars
inputs: inputs:
filePath: $(Build.SourcesDirectory)/eng/common/post-build/setup-maestro-vars.ps1 targetType: inline
arguments: -ReleaseConfigsPath '$(Build.StagingDirectory)/ReleaseConfigs/ReleaseConfigs.txt' script: |
try {
if ($Env:PromoteToChannelId -eq 0) {
$Content = Get-Content $(Build.StagingDirectory)/ReleaseConfigs/ReleaseConfigs.txt
$BarId = $Content | Select -Index 0
$Channels = ""
$Content | Select -Index 1 | ForEach-Object { $Channels += "$_ ," }
$IsStableBuild = $Content | Select -Index 2
$AzureDevOpsProject = $Env:System_TeamProject
$AzureDevOpsBuildDefinitionId = $Env:System_DefinitionId
$AzureDevOpsBuildId = $Env:Build_BuildId
$PromoteToMaestroChannelId = 0
}
else {
$buildApiEndpoint = "${Env:MaestroApiEndPoint}/api/builds/${Env:BARBuildId}?api-version=${Env:MaestroApiVersion}"
$apiHeaders = New-Object 'System.Collections.Generic.Dictionary[[String],[String]]'
$apiHeaders.Add('Accept', 'application/json')
$apiHeaders.Add('Authorization',"Bearer ${Env:MAESTRO_API_TOKEN}")
$buildInfo = try { Invoke-WebRequest -Method Get -Uri $buildApiEndpoint -Headers $apiHeaders | ConvertFrom-Json } catch { Write-Host "Error: $_" }
$BarId = $Env:BARBuildId
$Channels = 'None'
#TODO: Fix this once this issue is done: https://github.com/dotnet/arcade/issues/3834
$IsStableBuild = 'False'
$AzureDevOpsProject = $buildInfo.azureDevOpsProject
$AzureDevOpsBuildDefinitionId = $buildInfo.azureDevOpsBuildDefinitionId
$AzureDevOpsBuildId = $buildInfo.azureDevOpsBuildId
$PromoteToMaestroChannelId = $Env:PromoteToMaestroChannelId
}
Write-Host "##vso[task.setvariable variable=BARBuildId;isOutput=true]$BarId"
Write-Host "##vso[task.setvariable variable=InitialChannels;isOutput=true]$Channels"
Write-Host "##vso[task.setvariable variable=IsStableBuild;isOutput=true]$IsStableBuild"
Write-Host "##vso[task.setvariable variable=AzDOProjectName;isOutput=true]$AzureDevOpsProject"
Write-Host "##vso[task.setvariable variable=AzDOPipelineId;isOutput=true]$AzureDevOpsBuildDefinitionId"
Write-Host "##vso[task.setvariable variable=AzDOBuildId;isOutput=true]$AzureDevOpsBuildId"
Write-Host "##vso[task.setvariable variable=PromoteToMaestroChannelId;isOutput=true]$PromoteToMaestroChannelId"
}
catch {
Write-Host $_
Write-Host $_.Exception
Write-Host $_.ScriptStackTrace
exit 1
}
env:
MAESTRO_API_TOKEN: $(MaestroApiAccessToken)

4
eng/common/templates/steps/promote-build.yml → eng/common/templates/steps/add-build-to-channel.yml

@ -5,9 +5,9 @@ steps:
- task: PowerShell@2 - task: PowerShell@2
displayName: Add Build to Channel displayName: Add Build to Channel
inputs: inputs:
filePath: $(Build.SourcesDirectory)/eng/common/post-build/promote-build.ps1 filePath: $(Build.SourcesDirectory)/eng/common/post-build/add-build-to-channel.ps1
arguments: -BuildId $(BARBuildId) arguments: -BuildId $(BARBuildId)
-ChannelId ${{ parameters.ChannelId }} -ChannelId ${{ parameters.ChannelId }}
-MaestroApiAccessToken $(MaestroApiAccessToken) -MaestroApiAccessToken $(MaestroApiAccessToken)
-MaestroApiEndPoint $(MaestroApiEndPoint) -MaestroApiEndPoint $(MaestroApiEndPoint)
-MaestroApiVersion $(MaestroApiVersion) -MaestroApiVersion $(MaestroApiVersion)

23
eng/common/templates/steps/publish-logs.yml

@ -0,0 +1,23 @@
parameters:
StageLabel: ''
JobLabel: ''
steps:
- task: Powershell@2
displayName: Prepare Binlogs to Upload
inputs:
targetType: inline
script: |
New-Item -ItemType Directory $(Build.SourcesDirectory)/PostBuildLogs/${{parameters.StageLabel}}/${{parameters.JobLabel}}/
Move-Item -Path $(Build.SourcesDirectory)/artifacts/log/Debug/* $(Build.SourcesDirectory)/PostBuildLogs/${{parameters.StageLabel}}/${{parameters.JobLabel}}/
continueOnError: true
condition: always()
- task: PublishBuildArtifacts@1
displayName: Publish Logs
inputs:
PathtoPublish: '$(Build.SourcesDirectory)/PostBuildLogs'
PublishLocation: Container
ArtifactName: PostBuildLogs
continueOnError: true
condition: always()

3
eng/common/templates/steps/send-to-helix.yml

@ -23,6 +23,7 @@ parameters:
EnableXUnitReporter: false # optional -- true enables XUnit result reporting to Mission Control EnableXUnitReporter: false # optional -- true enables XUnit result reporting to Mission Control
WaitForWorkItemCompletion: true # optional -- true will make the task wait until work items have been completed and fail the build if work items fail. False is "fire and forget." WaitForWorkItemCompletion: true # optional -- true will make the task wait until work items have been completed and fail the build if work items fail. False is "fire and forget."
IsExternal: false # [DEPRECATED] -- doesn't do anything, jobs are external if HelixAccessToken is empty and Creator is set IsExternal: false # [DEPRECATED] -- doesn't do anything, jobs are external if HelixAccessToken is empty and Creator is set
HelixBaseUri: 'https://helix.dot.net/' # optional -- sets the Helix API base URI (allows targeting int)
Creator: '' # optional -- if the build is external, use this to specify who is sending the job Creator: '' # optional -- if the build is external, use this to specify who is sending the job
DisplayNamePrefix: 'Run Tests' # optional -- rename the beginning of the displayName of the steps in AzDO DisplayNamePrefix: 'Run Tests' # optional -- rename the beginning of the displayName of the steps in AzDO
condition: succeeded() # optional -- condition for step to execute; defaults to succeeded() condition: succeeded() # optional -- condition for step to execute; defaults to succeeded()
@ -55,6 +56,7 @@ steps:
DotNetCliVersion: ${{ parameters.DotNetCliVersion }} DotNetCliVersion: ${{ parameters.DotNetCliVersion }}
EnableXUnitReporter: ${{ parameters.EnableXUnitReporter }} EnableXUnitReporter: ${{ parameters.EnableXUnitReporter }}
WaitForWorkItemCompletion: ${{ parameters.WaitForWorkItemCompletion }} WaitForWorkItemCompletion: ${{ parameters.WaitForWorkItemCompletion }}
HelixBaseUri: ${{ parameters.HelixBaseUri }}
Creator: ${{ parameters.Creator }} Creator: ${{ parameters.Creator }}
SYSTEM_ACCESSTOKEN: $(System.AccessToken) SYSTEM_ACCESSTOKEN: $(System.AccessToken)
condition: and(${{ parameters.condition }}, eq(variables['Agent.Os'], 'Windows_NT')) condition: and(${{ parameters.condition }}, eq(variables['Agent.Os'], 'Windows_NT'))
@ -85,6 +87,7 @@ steps:
DotNetCliVersion: ${{ parameters.DotNetCliVersion }} DotNetCliVersion: ${{ parameters.DotNetCliVersion }}
EnableXUnitReporter: ${{ parameters.EnableXUnitReporter }} EnableXUnitReporter: ${{ parameters.EnableXUnitReporter }}
WaitForWorkItemCompletion: ${{ parameters.WaitForWorkItemCompletion }} WaitForWorkItemCompletion: ${{ parameters.WaitForWorkItemCompletion }}
HelixBaseUri: ${{ parameters.HelixBaseUri }}
Creator: ${{ parameters.Creator }} Creator: ${{ parameters.Creator }}
SYSTEM_ACCESSTOKEN: $(System.AccessToken) SYSTEM_ACCESSTOKEN: $(System.AccessToken)
condition: and(${{ parameters.condition }}, ne(variables['Agent.Os'], 'Windows_NT')) condition: and(${{ parameters.condition }}, ne(variables['Agent.Os'], 'Windows_NT'))

261
eng/common/tools.ps1

@ -5,7 +5,7 @@
[bool]$ci = if (Test-Path variable:ci) { $ci } else { $false } [bool]$ci = if (Test-Path variable:ci) { $ci } else { $false }
# Build configuration. Common values include 'Debug' and 'Release', but the repository may use other names. # Build configuration. Common values include 'Debug' and 'Release', but the repository may use other names.
[string]$configuration = if (Test-Path variable:configuration) { $configuration } else { "Debug" } [string]$configuration = if (Test-Path variable:configuration) { $configuration } else { 'Debug' }
# Set to true to output binary log from msbuild. Note that emitting binary log slows down the build. # Set to true to output binary log from msbuild. Note that emitting binary log slows down the build.
# Binary log must be enabled on CI. # Binary log must be enabled on CI.
@ -24,7 +24,7 @@
[bool]$restore = if (Test-Path variable:restore) { $restore } else { $true } [bool]$restore = if (Test-Path variable:restore) { $restore } else { $true }
# Adjusts msbuild verbosity level. # Adjusts msbuild verbosity level.
[string]$verbosity = if (Test-Path variable:verbosity) { $verbosity } else { "minimal" } [string]$verbosity = if (Test-Path variable:verbosity) { $verbosity } else { 'minimal' }
# Set to true to reuse msbuild nodes. Recommended to not reuse on CI. # Set to true to reuse msbuild nodes. Recommended to not reuse on CI.
[bool]$nodeReuse = if (Test-Path variable:nodeReuse) { $nodeReuse } else { !$ci } [bool]$nodeReuse = if (Test-Path variable:nodeReuse) { $nodeReuse } else { !$ci }
@ -41,21 +41,23 @@
# Enable repos to use a particular version of the on-line dotnet-install scripts. # Enable repos to use a particular version of the on-line dotnet-install scripts.
# default URL: https://dot.net/v1/dotnet-install.ps1 # default URL: https://dot.net/v1/dotnet-install.ps1
[string]$dotnetInstallScriptVersion = if (Test-Path variable:dotnetInstallScriptVersion) { $dotnetInstallScriptVersion } else { "v1" } [string]$dotnetInstallScriptVersion = if (Test-Path variable:dotnetInstallScriptVersion) { $dotnetInstallScriptVersion } else { 'v1' }
# True to use global NuGet cache instead of restoring packages to repository-local directory. # True to use global NuGet cache instead of restoring packages to repository-local directory.
[bool]$useGlobalNuGetCache = if (Test-Path variable:useGlobalNuGetCache) { $useGlobalNuGetCache } else { !$ci } [bool]$useGlobalNuGetCache = if (Test-Path variable:useGlobalNuGetCache) { $useGlobalNuGetCache } else { !$ci }
# An array of names of processes to stop on script exit if prepareMachine is true. # An array of names of processes to stop on script exit if prepareMachine is true.
$processesToStopOnExit = if (Test-Path variable:processesToStopOnExit) { $processesToStopOnExit } else { @("msbuild", "dotnet", "vbcscompiler") } $processesToStopOnExit = if (Test-Path variable:processesToStopOnExit) { $processesToStopOnExit } else { @('msbuild', 'dotnet', 'vbcscompiler') }
$disableConfigureToolsetImport = if (Test-Path variable:disableConfigureToolsetImport) { $disableConfigureToolsetImport } else { $null }
set-strictmode -version 2.0 set-strictmode -version 2.0
$ErrorActionPreference = "Stop" $ErrorActionPreference = 'Stop'
[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12 [Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12
function Create-Directory([string[]] $path) { function Create-Directory([string[]] $path) {
if (!(Test-Path $path)) { if (!(Test-Path $path)) {
New-Item -path $path -force -itemType "Directory" | Out-Null New-Item -path $path -force -itemType 'Directory' | Out-Null
} }
} }
@ -96,7 +98,10 @@ function Exec-Process([string]$command, [string]$commandArgs) {
} }
} }
function InitializeDotNetCli([bool]$install) { # createSdkLocationFile parameter enables a file being generated under the toolset directory
# which writes the sdk's location into. This is only necessary for cmd --> powershell invocations
# as dot sourcing isn't possible.
function InitializeDotNetCli([bool]$install, [bool]$createSdkLocationFile) {
if (Test-Path variable:global:_DotNetInstallDir) { if (Test-Path variable:global:_DotNetInstallDir) {
return $global:_DotNetInstallDir return $global:_DotNetInstallDir
} }
@ -119,7 +124,7 @@ function InitializeDotNetCli([bool]$install) {
# Find the first path on %PATH% that contains the dotnet.exe # Find the first path on %PATH% that contains the dotnet.exe
if ($useInstalledDotNetCli -and (-not $globalJsonHasRuntimes) -and ($env:DOTNET_INSTALL_DIR -eq $null)) { if ($useInstalledDotNetCli -and (-not $globalJsonHasRuntimes) -and ($env:DOTNET_INSTALL_DIR -eq $null)) {
$dotnetCmd = Get-Command "dotnet.exe" -ErrorAction SilentlyContinue $dotnetCmd = Get-Command 'dotnet.exe' -ErrorAction SilentlyContinue
if ($dotnetCmd -ne $null) { if ($dotnetCmd -ne $null) {
$env:DOTNET_INSTALL_DIR = Split-Path $dotnetCmd.Path -Parent $env:DOTNET_INSTALL_DIR = Split-Path $dotnetCmd.Path -Parent
} }
@ -132,13 +137,13 @@ function InitializeDotNetCli([bool]$install) {
if ((-not $globalJsonHasRuntimes) -and ($env:DOTNET_INSTALL_DIR -ne $null) -and (Test-Path(Join-Path $env:DOTNET_INSTALL_DIR "sdk\$dotnetSdkVersion"))) { if ((-not $globalJsonHasRuntimes) -and ($env:DOTNET_INSTALL_DIR -ne $null) -and (Test-Path(Join-Path $env:DOTNET_INSTALL_DIR "sdk\$dotnetSdkVersion"))) {
$dotnetRoot = $env:DOTNET_INSTALL_DIR $dotnetRoot = $env:DOTNET_INSTALL_DIR
} else { } else {
$dotnetRoot = Join-Path $RepoRoot ".dotnet" $dotnetRoot = Join-Path $RepoRoot '.dotnet'
if (-not (Test-Path(Join-Path $dotnetRoot "sdk\$dotnetSdkVersion"))) { if (-not (Test-Path(Join-Path $dotnetRoot "sdk\$dotnetSdkVersion"))) {
if ($install) { if ($install) {
InstallDotNetSdk $dotnetRoot $dotnetSdkVersion InstallDotNetSdk $dotnetRoot $dotnetSdkVersion
} else { } else {
Write-PipelineTelemetryError -Category "InitializeToolset" -Message "Unable to find dotnet with SDK version '$dotnetSdkVersion'" Write-PipelineTelemetryError -Category 'InitializeToolset' -Message "Unable to find dotnet with SDK version '$dotnetSdkVersion'"
ExitWithExitCode 1 ExitWithExitCode 1
} }
} }
@ -146,6 +151,24 @@ function InitializeDotNetCli([bool]$install) {
$env:DOTNET_INSTALL_DIR = $dotnetRoot $env:DOTNET_INSTALL_DIR = $dotnetRoot
} }
# Creates a temporary file under the toolset dir.
# The following code block is protecting against concurrent access so that this function can
# be called in parallel.
if ($createSdkLocationFile) {
do {
$sdkCacheFileTemp = Join-Path $ToolsetDir $([System.IO.Path]::GetRandomFileName())
}
until (!(Test-Path $sdkCacheFileTemp))
Set-Content -Path $sdkCacheFileTemp -Value $dotnetRoot
try {
Rename-Item -Force -Path $sdkCacheFileTemp 'sdk.txt'
} catch {
# Somebody beat us
Remove-Item -Path $sdkCacheFileTemp
}
}
# Add dotnet to PATH. This prevents any bare invocation of dotnet in custom # Add dotnet to PATH. This prevents any bare invocation of dotnet in custom
# build steps from using anything other than what we've downloaded. # build steps from using anything other than what we've downloaded.
# It also ensures that VS msbuild will use the downloaded sdk targets. # It also ensures that VS msbuild will use the downloaded sdk targets.
@ -154,15 +177,6 @@ function InitializeDotNetCli([bool]$install) {
# Make Sure that our bootstrapped dotnet cli is available in future steps of the Azure Pipelines build # Make Sure that our bootstrapped dotnet cli is available in future steps of the Azure Pipelines build
Write-PipelinePrependPath -Path $dotnetRoot Write-PipelinePrependPath -Path $dotnetRoot
# Work around issues with Azure Artifacts credential provider
# https://github.com/dotnet/arcade/issues/3932
if ($ci) {
$env:NUGET_PLUGIN_HANDSHAKE_TIMEOUT_IN_SECONDS = 20
$env:NUGET_PLUGIN_REQUEST_TIMEOUT_IN_SECONDS = 20
Write-PipelineSetVariable -Name 'NUGET_PLUGIN_HANDSHAKE_TIMEOUT_IN_SECONDS' -Value '20'
Write-PipelineSetVariable -Name 'NUGET_PLUGIN_REQUEST_TIMEOUT_IN_SECONDS' -Value '20'
}
Write-PipelineSetVariable -Name 'DOTNET_MULTILEVEL_LOOKUP' -Value '0' Write-PipelineSetVariable -Name 'DOTNET_MULTILEVEL_LOOKUP' -Value '0'
Write-PipelineSetVariable -Name 'DOTNET_SKIP_FIRST_TIME_EXPERIENCE' -Value '1' Write-PipelineSetVariable -Name 'DOTNET_SKIP_FIRST_TIME_EXPERIENCE' -Value '1'
@ -170,21 +184,54 @@ function InitializeDotNetCli([bool]$install) {
} }
function GetDotNetInstallScript([string] $dotnetRoot) { function GetDotNetInstallScript([string] $dotnetRoot) {
$installScript = Join-Path $dotnetRoot "dotnet-install.ps1" $installScript = Join-Path $dotnetRoot 'dotnet-install.ps1'
if (!(Test-Path $installScript)) { if (!(Test-Path $installScript)) {
Create-Directory $dotnetRoot Create-Directory $dotnetRoot
$ProgressPreference = 'SilentlyContinue' # Don't display the console progress UI - it's a huge perf hit $ProgressPreference = 'SilentlyContinue' # Don't display the console progress UI - it's a huge perf hit
Invoke-WebRequest "https://dot.net/$dotnetInstallScriptVersion/dotnet-install.ps1" -OutFile $installScript
$maxRetries = 5
$retries = 1
$uri = "https://dot.net/$dotnetInstallScriptVersion/dotnet-install.ps1"
while($true) {
try {
Write-Host "GET $uri"
Invoke-WebRequest $uri -OutFile $installScript
break
}
catch {
Write-Host "Failed to download '$uri'"
Write-Error $_.Exception.Message -ErrorAction Continue
}
if (++$retries -le $maxRetries) {
$delayInSeconds = [math]::Pow(2, $retries) - 1 # Exponential backoff
Write-Host "Retrying. Waiting for $delayInSeconds seconds before next attempt ($retries of $maxRetries)."
Start-Sleep -Seconds $delayInSeconds
}
else {
throw "Unable to download file in $maxRetries attempts."
}
}
} }
return $installScript return $installScript
} }
function InstallDotNetSdk([string] $dotnetRoot, [string] $version, [string] $architecture = "") { function InstallDotNetSdk([string] $dotnetRoot, [string] $version, [string] $architecture = '') {
InstallDotNet $dotnetRoot $version $architecture InstallDotNet $dotnetRoot $version $architecture
} }
function InstallDotNet([string] $dotnetRoot, [string] $version, [string] $architecture = "", [string] $runtime = "", [bool] $skipNonVersionedFiles = $false) { function InstallDotNet([string] $dotnetRoot,
[string] $version,
[string] $architecture = '',
[string] $runtime = '',
[bool] $skipNonVersionedFiles = $false,
[string] $runtimeSourceFeed = '',
[string] $runtimeSourceFeedKey = '') {
$installScript = GetDotNetInstallScript $dotnetRoot $installScript = GetDotNetInstallScript $dotnetRoot
$installParameters = @{ $installParameters = @{
Version = $version Version = $version
@ -195,10 +242,32 @@ function InstallDotNet([string] $dotnetRoot, [string] $version, [string] $archit
if ($runtime) { $installParameters.Runtime = $runtime } if ($runtime) { $installParameters.Runtime = $runtime }
if ($skipNonVersionedFiles) { $installParameters.SkipNonVersionedFiles = $skipNonVersionedFiles } if ($skipNonVersionedFiles) { $installParameters.SkipNonVersionedFiles = $skipNonVersionedFiles }
& $installScript @installParameters try {
if ($lastExitCode -ne 0) { & $installScript @installParameters
Write-PipelineTelemetryError -Category "InitializeToolset" -Message "Failed to install dotnet cli (exit code '$lastExitCode')." }
ExitWithExitCode $lastExitCode catch {
Write-PipelineTelemetryError -Category 'InitializeToolset' -Message "Failed to install dotnet runtime '$runtime' from public location."
# Only the runtime can be installed from a custom [private] location.
if ($runtime -and ($runtimeSourceFeed -or $runtimeSourceFeedKey)) {
if ($runtimeSourceFeed) { $installParameters.AzureFeed = $runtimeSourceFeed }
if ($runtimeSourceFeedKey) {
$decodedBytes = [System.Convert]::FromBase64String($runtimeSourceFeedKey)
$decodedString = [System.Text.Encoding]::UTF8.GetString($decodedBytes)
$installParameters.FeedCredential = $decodedString
}
try {
& $installScript @installParameters
}
catch {
Write-PipelineTelemetryError -Category 'InitializeToolset' -Message "Failed to install dotnet runtime '$runtime' from custom location '$runtimeSourceFeed'."
ExitWithExitCode 1
}
} else {
ExitWithExitCode 1
}
} }
} }
@ -219,16 +288,16 @@ function InitializeVisualStudioMSBuild([bool]$install, [object]$vsRequirements =
} }
if (!$vsRequirements) { $vsRequirements = $GlobalJson.tools.vs } if (!$vsRequirements) { $vsRequirements = $GlobalJson.tools.vs }
$vsMinVersionStr = if ($vsRequirements.version) { $vsRequirements.version } else { "15.9" } $vsMinVersionStr = if ($vsRequirements.version) { $vsRequirements.version } else { '15.9' }
$vsMinVersion = [Version]::new($vsMinVersionStr) $vsMinVersion = [Version]::new($vsMinVersionStr)
# Try msbuild command available in the environment. # Try msbuild command available in the environment.
if ($env:VSINSTALLDIR -ne $null) { if ($env:VSINSTALLDIR -ne $null) {
$msbuildCmd = Get-Command "msbuild.exe" -ErrorAction SilentlyContinue $msbuildCmd = Get-Command 'msbuild.exe' -ErrorAction SilentlyContinue
if ($msbuildCmd -ne $null) { if ($msbuildCmd -ne $null) {
# Workaround for https://github.com/dotnet/roslyn/issues/35793 # Workaround for https://github.com/dotnet/roslyn/issues/35793
# Due to this issue $msbuildCmd.Version returns 0.0.0.0 for msbuild.exe 16.2+ # Due to this issue $msbuildCmd.Version returns 0.0.0.0 for msbuild.exe 16.2+
$msbuildVersion = [Version]::new((Get-Item $msbuildCmd.Path).VersionInfo.ProductVersion.Split(@('-', '+'))[0]) $msbuildVersion = [Version]::new((Get-Item $msbuildCmd.Path).VersionInfo.ProductVersion.Split([char[]]@('-', '+'))[0])
if ($msbuildVersion -ge $vsMinVersion) { if ($msbuildVersion -ge $vsMinVersion) {
return $global:_MSBuildExe = $msbuildCmd.Path return $global:_MSBuildExe = $msbuildCmd.Path
@ -248,7 +317,7 @@ function InitializeVisualStudioMSBuild([bool]$install, [object]$vsRequirements =
InitializeVisualStudioEnvironmentVariables $vsInstallDir $vsMajorVersion InitializeVisualStudioEnvironmentVariables $vsInstallDir $vsMajorVersion
} else { } else {
if (Get-Member -InputObject $GlobalJson.tools -Name "xcopy-msbuild") { if (Get-Member -InputObject $GlobalJson.tools -Name 'xcopy-msbuild') {
$xcopyMSBuildVersion = $GlobalJson.tools.'xcopy-msbuild' $xcopyMSBuildVersion = $GlobalJson.tools.'xcopy-msbuild'
$vsMajorVersion = $xcopyMSBuildVersion.Split('.')[0] $vsMajorVersion = $xcopyMSBuildVersion.Split('.')[0]
} else { } else {
@ -256,9 +325,12 @@ function InitializeVisualStudioMSBuild([bool]$install, [object]$vsRequirements =
$xcopyMSBuildVersion = "$vsMajorVersion.$($vsMinVersion.Minor).0-alpha" $xcopyMSBuildVersion = "$vsMajorVersion.$($vsMinVersion.Minor).0-alpha"
} }
$vsInstallDir = InitializeXCopyMSBuild $xcopyMSBuildVersion $install $vsInstallDir = $null
if ($xcopyMSBuildVersion.Trim() -ine "none") {
$vsInstallDir = InitializeXCopyMSBuild $xcopyMSBuildVersion $install
}
if ($vsInstallDir -eq $null) { if ($vsInstallDir -eq $null) {
throw "Unable to find Visual Studio that has required version and components installed" throw 'Unable to find Visual Studio that has required version and components installed'
} }
} }
@ -282,7 +354,7 @@ function InstallXCopyMSBuild([string]$packageVersion) {
} }
function InitializeXCopyMSBuild([string]$packageVersion, [bool]$install) { function InitializeXCopyMSBuild([string]$packageVersion, [bool]$install) {
$packageName = "RoslynTools.MSBuild" $packageName = 'RoslynTools.MSBuild'
$packageDir = Join-Path $ToolsDir "msbuild\$packageVersion" $packageDir = Join-Path $ToolsDir "msbuild\$packageVersion"
$packagePath = Join-Path $packageDir "$packageName.$packageVersion.nupkg" $packagePath = Join-Path $packageDir "$packageName.$packageVersion.nupkg"
@ -298,7 +370,7 @@ function InitializeXCopyMSBuild([string]$packageVersion, [bool]$install) {
Unzip $packagePath $packageDir Unzip $packagePath $packageDir
} }
return Join-Path $packageDir "tools" return Join-Path $packageDir 'tools'
} }
# #
@ -315,32 +387,37 @@ function InitializeXCopyMSBuild([string]$packageVersion, [bool]$install) {
# or $null if no instance meeting the requirements is found on the machine. # or $null if no instance meeting the requirements is found on the machine.
# #
function LocateVisualStudio([object]$vsRequirements = $null){ function LocateVisualStudio([object]$vsRequirements = $null){
if (Get-Member -InputObject $GlobalJson.tools -Name "vswhere") { if (Get-Member -InputObject $GlobalJson.tools -Name 'vswhere') {
$vswhereVersion = $GlobalJson.tools.vswhere $vswhereVersion = $GlobalJson.tools.vswhere
} else { } else {
$vswhereVersion = "2.5.2" $vswhereVersion = '2.5.2'
} }
$vsWhereDir = Join-Path $ToolsDir "vswhere\$vswhereVersion" $vsWhereDir = Join-Path $ToolsDir "vswhere\$vswhereVersion"
$vsWhereExe = Join-Path $vsWhereDir "vswhere.exe" $vsWhereExe = Join-Path $vsWhereDir 'vswhere.exe'
if (!(Test-Path $vsWhereExe)) { if (!(Test-Path $vsWhereExe)) {
Create-Directory $vsWhereDir Create-Directory $vsWhereDir
Write-Host "Downloading vswhere" Write-Host 'Downloading vswhere'
Invoke-WebRequest "https://github.com/Microsoft/vswhere/releases/download/$vswhereVersion/vswhere.exe" -OutFile $vswhereExe try {
Invoke-WebRequest "https://netcorenativeassets.blob.core.windows.net/resource-packages/external/windows/vswhere/$vswhereVersion/vswhere.exe" -OutFile $vswhereExe
}
catch {
Write-PipelineTelemetryError -Category 'InitializeToolset' -Message $_
}
} }
if (!$vsRequirements) { $vsRequirements = $GlobalJson.tools.vs } if (!$vsRequirements) { $vsRequirements = $GlobalJson.tools.vs }
$args = @("-latest", "-prerelease", "-format", "json", "-requires", "Microsoft.Component.MSBuild", "-products", "*") $args = @('-latest', '-prerelease', '-format', 'json', '-requires', 'Microsoft.Component.MSBuild', '-products', '*')
if (Get-Member -InputObject $vsRequirements -Name "version") { if (Get-Member -InputObject $vsRequirements -Name 'version') {
$args += "-version" $args += '-version'
$args += $vsRequirements.version $args += $vsRequirements.version
} }
if (Get-Member -InputObject $vsRequirements -Name "components") { if (Get-Member -InputObject $vsRequirements -Name 'components') {
foreach ($component in $vsRequirements.components) { foreach ($component in $vsRequirements.components) {
$args += "-requires" $args += '-requires'
$args += $component $args += $component
} }
} }
@ -366,27 +443,27 @@ function InitializeBuildTool() {
# Initialize dotnet cli if listed in 'tools' # Initialize dotnet cli if listed in 'tools'
$dotnetRoot = $null $dotnetRoot = $null
if (Get-Member -InputObject $GlobalJson.tools -Name "dotnet") { if (Get-Member -InputObject $GlobalJson.tools -Name 'dotnet') {
$dotnetRoot = InitializeDotNetCli -install:$restore $dotnetRoot = InitializeDotNetCli -install:$restore
} }
if ($msbuildEngine -eq "dotnet") { if ($msbuildEngine -eq 'dotnet') {
if (!$dotnetRoot) { if (!$dotnetRoot) {
Write-PipelineTelemetryError -Category "InitializeToolset" -Message "/global.json must specify 'tools.dotnet'." Write-PipelineTelemetryError -Category 'InitializeToolset' -Message "/global.json must specify 'tools.dotnet'."
ExitWithExitCode 1 ExitWithExitCode 1
} }
$buildTool = @{ Path = Join-Path $dotnetRoot "dotnet.exe"; Command = "msbuild"; Tool = "dotnet"; Framework = "netcoreapp2.1" } $buildTool = @{ Path = Join-Path $dotnetRoot 'dotnet.exe'; Command = 'msbuild'; Tool = 'dotnet'; Framework = 'netcoreapp2.1' }
} elseif ($msbuildEngine -eq "vs") { } elseif ($msbuildEngine -eq "vs") {
try { try {
$msbuildPath = InitializeVisualStudioMSBuild -install:$restore $msbuildPath = InitializeVisualStudioMSBuild -install:$restore
} catch { } catch {
Write-PipelineTelemetryError -Category "InitializeToolset" -Message $_ Write-PipelineTelemetryError -Category 'InitializeToolset' -Message $_
ExitWithExitCode 1 ExitWithExitCode 1
} }
$buildTool = @{ Path = $msbuildPath; Command = ""; Tool = "vs"; Framework = "net472" } $buildTool = @{ Path = $msbuildPath; Command = ""; Tool = "vs"; Framework = "net472" }
} else { } else {
Write-PipelineTelemetryError -Category "InitializeToolset" -Message "Unexpected value of -msbuildEngine: '$msbuildEngine'." Write-PipelineTelemetryError -Category 'InitializeToolset' -Message "Unexpected value of -msbuildEngine: '$msbuildEngine'."
ExitWithExitCode 1 ExitWithExitCode 1
} }
@ -395,15 +472,15 @@ function InitializeBuildTool() {
function GetDefaultMSBuildEngine() { function GetDefaultMSBuildEngine() {
# Presence of tools.vs indicates the repo needs to build using VS msbuild on Windows. # Presence of tools.vs indicates the repo needs to build using VS msbuild on Windows.
if (Get-Member -InputObject $GlobalJson.tools -Name "vs") { if (Get-Member -InputObject $GlobalJson.tools -Name 'vs') {
return "vs" return 'vs'
} }
if (Get-Member -InputObject $GlobalJson.tools -Name "dotnet") { if (Get-Member -InputObject $GlobalJson.tools -Name 'dotnet') {
return "dotnet" return 'dotnet'
} }
Write-PipelineTelemetryError -Category "InitializeToolset" -Message "-msbuildEngine must be specified, or /global.json must specify 'tools.dotnet' or 'tools.vs'." Write-PipelineTelemetryError -Category 'InitializeToolset' -Message "-msbuildEngine must be specified, or /global.json must specify 'tools.dotnet' or 'tools.vs'."
ExitWithExitCode 1 ExitWithExitCode 1
} }
@ -412,9 +489,9 @@ function GetNuGetPackageCachePath() {
# Use local cache on CI to ensure deterministic build, # Use local cache on CI to ensure deterministic build,
# use global cache in dev builds to avoid cost of downloading packages. # use global cache in dev builds to avoid cost of downloading packages.
if ($useGlobalNuGetCache) { if ($useGlobalNuGetCache) {
$env:NUGET_PACKAGES = Join-Path $env:UserProfile ".nuget\packages" $env:NUGET_PACKAGES = Join-Path $env:UserProfile '.nuget\packages'
} else { } else {
$env:NUGET_PACKAGES = Join-Path $RepoRoot ".packages" $env:NUGET_PACKAGES = Join-Path $RepoRoot '.packages'
} }
} }
@ -427,7 +504,7 @@ function GetSdkTaskProject([string]$taskName) {
} }
function InitializeNativeTools() { function InitializeNativeTools() {
if (Get-Member -InputObject $GlobalJson -Name "native-tools") { if (-Not (Test-Path variable:DisableNativeToolsetInstalls) -And (Get-Member -InputObject $GlobalJson -Name "native-tools")) {
$nativeArgs= @{} $nativeArgs= @{}
if ($ci) { if ($ci) {
$nativeArgs = @{ $nativeArgs = @{
@ -456,14 +533,14 @@ function InitializeToolset() {
} }
if (-not $restore) { if (-not $restore) {
Write-PipelineTelemetryError -Category "InitializeToolset" -Message "Toolset version $toolsetVersion has not been restored." Write-PipelineTelemetryError -Category 'InitializeToolset' -Message "Toolset version $toolsetVersion has not been restored."
ExitWithExitCode 1 ExitWithExitCode 1
} }
$buildTool = InitializeBuildTool $buildTool = InitializeBuildTool
$proj = Join-Path $ToolsetDir "restore.proj" $proj = Join-Path $ToolsetDir 'restore.proj'
$bl = if ($binaryLog) { "/bl:" + (Join-Path $LogDir "ToolsetRestore.binlog") } else { "" } $bl = if ($binaryLog) { '/bl:' + (Join-Path $LogDir 'ToolsetRestore.binlog') } else { '' }
'<Project Sdk="Microsoft.DotNet.Arcade.Sdk"/>' | Set-Content $proj '<Project Sdk="Microsoft.DotNet.Arcade.Sdk"/>' | Set-Content $proj
@ -485,7 +562,7 @@ function ExitWithExitCode([int] $exitCode) {
} }
function Stop-Processes() { function Stop-Processes() {
Write-Host "Killing running build processes..." Write-Host 'Killing running build processes...'
foreach ($processName in $processesToStopOnExit) { foreach ($processName in $processesToStopOnExit) {
Get-Process -Name $processName -ErrorAction SilentlyContinue | Stop-Process Get-Process -Name $processName -ErrorAction SilentlyContinue | Stop-Process
} }
@ -502,13 +579,18 @@ function MSBuild() {
# Work around issues with Azure Artifacts credential provider # Work around issues with Azure Artifacts credential provider
# https://github.com/dotnet/arcade/issues/3932 # https://github.com/dotnet/arcade/issues/3932
if ($ci -and $buildTool.Tool -eq "dotnet") { if ($ci -and $buildTool.Tool -eq 'dotnet') {
dotnet nuget locals http-cache -c dotnet nuget locals http-cache -c
$env:NUGET_PLUGIN_HANDSHAKE_TIMEOUT_IN_SECONDS = 20
$env:NUGET_PLUGIN_REQUEST_TIMEOUT_IN_SECONDS = 20
Write-PipelineSetVariable -Name 'NUGET_PLUGIN_HANDSHAKE_TIMEOUT_IN_SECONDS' -Value '20'
Write-PipelineSetVariable -Name 'NUGET_PLUGIN_REQUEST_TIMEOUT_IN_SECONDS' -Value '20'
} }
$toolsetBuildProject = InitializeToolset $toolsetBuildProject = InitializeToolset
$path = Split-Path -parent $toolsetBuildProject $path = Split-Path -parent $toolsetBuildProject
$path = Join-Path $path (Join-Path $buildTool.Framework "Microsoft.DotNet.Arcade.Sdk.dll") $path = Join-Path $path (Join-Path $buildTool.Framework 'Microsoft.DotNet.Arcade.Sdk.dll')
$args += "/logger:$path" $args += "/logger:$path"
} }
@ -523,12 +605,12 @@ function MSBuild() {
function MSBuild-Core() { function MSBuild-Core() {
if ($ci) { if ($ci) {
if (!$binaryLog) { if (!$binaryLog) {
Write-PipelineTaskError -Message "Binary log must be enabled in CI build." Write-PipelineTelemetryError -Category 'Build' -Message 'Binary log must be enabled in CI build.'
ExitWithExitCode 1 ExitWithExitCode 1
} }
if ($nodeReuse) { if ($nodeReuse) {
Write-PipelineTaskError -Message "Node reuse must be disabled in CI build." Write-PipelineTelemetryError -Category 'Build' -Message 'Node reuse must be disabled in CI build.'
ExitWithExitCode 1 ExitWithExitCode 1
} }
} }
@ -538,10 +620,10 @@ function MSBuild-Core() {
$cmdArgs = "$($buildTool.Command) /m /nologo /clp:Summary /v:$verbosity /nr:$nodeReuse /p:ContinuousIntegrationBuild=$ci" $cmdArgs = "$($buildTool.Command) /m /nologo /clp:Summary /v:$verbosity /nr:$nodeReuse /p:ContinuousIntegrationBuild=$ci"
if ($warnAsError) { if ($warnAsError) {
$cmdArgs += " /warnaserror /p:TreatWarningsAsErrors=true" $cmdArgs += ' /warnaserror /p:TreatWarningsAsErrors=true'
} }
else { else {
$cmdArgs += " /p:TreatWarningsAsErrors=false" $cmdArgs += ' /p:TreatWarningsAsErrors=false'
} }
foreach ($arg in $args) { foreach ($arg in $args) {
@ -553,7 +635,7 @@ function MSBuild-Core() {
$exitCode = Exec-Process $buildTool.Path $cmdArgs $exitCode = Exec-Process $buildTool.Path $cmdArgs
if ($exitCode -ne 0) { if ($exitCode -ne 0) {
Write-PipelineTaskError -Message "Build failed." Write-PipelineTelemetryError -Category 'Build' -Message 'Build failed.'
$buildLog = GetMSBuildBinaryLogCommandLineArgument $args $buildLog = GetMSBuildBinaryLogCommandLineArgument $args
if ($buildLog -ne $null) { if ($buildLog -ne $null) {
@ -568,12 +650,12 @@ function GetMSBuildBinaryLogCommandLineArgument($arguments) {
foreach ($argument in $arguments) { foreach ($argument in $arguments) {
if ($argument -ne $null) { if ($argument -ne $null) {
$arg = $argument.Trim() $arg = $argument.Trim()
if ($arg.StartsWith("/bl:", "OrdinalIgnoreCase")) { if ($arg.StartsWith('/bl:', "OrdinalIgnoreCase")) {
return $arg.Substring("/bl:".Length) return $arg.Substring('/bl:'.Length)
} }
if ($arg.StartsWith("/binaryLogger:", "OrdinalIgnoreCase")) { if ($arg.StartsWith('/binaryLogger:', 'OrdinalIgnoreCase')) {
return $arg.Substring("/binaryLogger:".Length) return $arg.Substring('/binaryLogger:'.Length)
} }
} }
} }
@ -583,14 +665,14 @@ function GetMSBuildBinaryLogCommandLineArgument($arguments) {
. $PSScriptRoot\pipeline-logging-functions.ps1 . $PSScriptRoot\pipeline-logging-functions.ps1
$RepoRoot = Resolve-Path (Join-Path $PSScriptRoot "..\..") $RepoRoot = Resolve-Path (Join-Path $PSScriptRoot '..\..')
$EngRoot = Resolve-Path (Join-Path $PSScriptRoot "..") $EngRoot = Resolve-Path (Join-Path $PSScriptRoot '..')
$ArtifactsDir = Join-Path $RepoRoot "artifacts" $ArtifactsDir = Join-Path $RepoRoot 'artifacts'
$ToolsetDir = Join-Path $ArtifactsDir "toolset" $ToolsetDir = Join-Path $ArtifactsDir 'toolset'
$ToolsDir = Join-Path $RepoRoot ".tools" $ToolsDir = Join-Path $RepoRoot '.tools'
$LogDir = Join-Path (Join-Path $ArtifactsDir "log") $configuration $LogDir = Join-Path (Join-Path $ArtifactsDir 'log') $configuration
$TempDir = Join-Path (Join-Path $ArtifactsDir "tmp") $configuration $TempDir = Join-Path (Join-Path $ArtifactsDir 'tmp') $configuration
$GlobalJson = Get-Content -Raw -Path (Join-Path $RepoRoot "global.json") | ConvertFrom-Json $GlobalJson = Get-Content -Raw -Path (Join-Path $RepoRoot 'global.json') | ConvertFrom-Json
# true if global.json contains a "runtimes" section # true if global.json contains a "runtimes" section
$globalJsonHasRuntimes = if ($GlobalJson.tools.PSObject.Properties.Name -Match 'runtimes') { $true } else { $false } $globalJsonHasRuntimes = if ($GlobalJson.tools.PSObject.Properties.Name -Match 'runtimes') { $true } else { $false }
@ -603,3 +685,18 @@ Write-PipelineSetVariable -Name 'Artifacts.Toolset' -Value $ToolsetDir
Write-PipelineSetVariable -Name 'Artifacts.Log' -Value $LogDir Write-PipelineSetVariable -Name 'Artifacts.Log' -Value $LogDir
Write-PipelineSetVariable -Name 'TEMP' -Value $TempDir Write-PipelineSetVariable -Name 'TEMP' -Value $TempDir
Write-PipelineSetVariable -Name 'TMP' -Value $TempDir Write-PipelineSetVariable -Name 'TMP' -Value $TempDir
# Import custom tools configuration, if present in the repo.
# Note: Import in global scope so that the script set top-level variables without qualification.
if (!$disableConfigureToolsetImport) {
$configureToolsetScript = Join-Path $EngRoot 'configure-toolset.ps1'
if (Test-Path $configureToolsetScript) {
. $configureToolsetScript
if ((Test-Path variable:failOnConfigureToolsetError) -And $failOnConfigureToolsetError) {
if ((Test-Path variable:LastExitCode) -And ($LastExitCode -ne 0)) {
Write-PipelineTelemetryError -Category 'Build' -Message 'configure-toolset.ps1 returned a non-zero exit code'
ExitWithExitCode $LastExitCode
}
}
}
}

108
eng/common/tools.sh

@ -41,7 +41,7 @@ fi
# Configures warning treatment in msbuild. # Configures warning treatment in msbuild.
warn_as_error=${warn_as_error:-true} warn_as_error=${warn_as_error:-true}
# True to attempt using .NET Core already that meets requirements specified in global.json # True to attempt using .NET Core already that meets requirements specified in global.json
# installed on the machine instead of downloading one. # installed on the machine instead of downloading one.
use_installed_dotnet_cli=${use_installed_dotnet_cli:-true} use_installed_dotnet_cli=${use_installed_dotnet_cli:-true}
@ -81,7 +81,7 @@ function ReadGlobalVersion {
local pattern="\"$key\" *: *\"(.*)\"" local pattern="\"$key\" *: *\"(.*)\""
if [[ ! $line =~ $pattern ]]; then if [[ ! $line =~ $pattern ]]; then
Write-PipelineTelemetryError -category 'InitializeToolset' "Error: Cannot find \"$key\" in $global_json_file" Write-PipelineTelemetryError -category 'Build' "Error: Cannot find \"$key\" in $global_json_file"
ExitWithExitCode 1 ExitWithExitCode 1
fi fi
@ -152,15 +152,6 @@ function InitializeDotNetCli {
# build steps from using anything other than what we've downloaded. # build steps from using anything other than what we've downloaded.
Write-PipelinePrependPath -path "$dotnet_root" Write-PipelinePrependPath -path "$dotnet_root"
# Work around issues with Azure Artifacts credential provider
# https://github.com/dotnet/arcade/issues/3932
if [[ "$ci" == true ]]; then
export NUGET_PLUGIN_HANDSHAKE_TIMEOUT_IN_SECONDS=20
export NUGET_PLUGIN_REQUEST_TIMEOUT_IN_SECONDS=20
Write-PipelineSetVariable -name "NUGET_PLUGIN_HANDSHAKE_TIMEOUT_IN_SECONDS" -value "20"
Write-PipelineSetVariable -name "NUGET_PLUGIN_REQUEST_TIMEOUT_IN_SECONDS" -value "20"
fi
Write-PipelineSetVariable -name "DOTNET_MULTILEVEL_LOOKUP" -value "0" Write-PipelineSetVariable -name "DOTNET_MULTILEVEL_LOOKUP" -value "0"
Write-PipelineSetVariable -name "DOTNET_SKIP_FIRST_TIME_EXPERIENCE" -value "1" Write-PipelineSetVariable -name "DOTNET_SKIP_FIRST_TIME_EXPERIENCE" -value "1"
@ -181,7 +172,7 @@ function InstallDotNetSdk {
function InstallDotNet { function InstallDotNet {
local root=$1 local root=$1
local version=$2 local version=$2
GetDotNetInstallScript "$root" GetDotNetInstallScript "$root"
local install_script=$_GetDotNetInstallScript local install_script=$_GetDotNetInstallScript
@ -200,11 +191,55 @@ function InstallDotNet {
fi fi
bash "$install_script" --version $version --install-dir "$root" $archArg $runtimeArg $skipNonVersionedFilesArg || { bash "$install_script" --version $version --install-dir "$root" $archArg $runtimeArg $skipNonVersionedFilesArg || {
local exit_code=$? local exit_code=$?
Write-PipelineTelemetryError -category 'InitializeToolset' "Failed to install dotnet SDK (exit code '$exit_code')." Write-PipelineTelemetryError -category 'InitializeToolset' "Failed to install dotnet SDK from public location (exit code '$exit_code')."
ExitWithExitCode $exit_code
if [[ -n "$runtimeArg" ]]; then
local runtimeSourceFeed=''
if [[ -n "${6:-}" ]]; then
runtimeSourceFeed="--azure-feed $6"
fi
local runtimeSourceFeedKey=''
if [[ -n "${7:-}" ]]; then
decodedFeedKey=`echo $7 | base64 --decode`
runtimeSourceFeedKey="--feed-credential $decodedFeedKey"
fi
if [[ -n "$runtimeSourceFeed" || -n "$runtimeSourceFeedKey" ]]; then
bash "$install_script" --version $version --install-dir "$root" $archArg $runtimeArg $skipNonVersionedFilesArg $runtimeSourceFeed $runtimeSourceFeedKey || {
local exit_code=$?
Write-PipelineTelemetryError -category 'InitializeToolset' "Failed to install dotnet SDK from custom location '$runtimeSourceFeed' (exit code '$exit_code')."
ExitWithExitCode $exit_code
}
else
ExitWithExitCode $exit_code
fi
fi
} }
} }
function with_retries {
local maxRetries=5
local retries=1
echo "Trying to run '$@' for maximum of $maxRetries attempts."
while [[ $((retries++)) -le $maxRetries ]]; do
"$@"
if [[ $? == 0 ]]; then
echo "Ran '$@' successfully."
return 0
fi
timeout=$((2**$retries-1))
echo "Failed to execute '$@'. Waiting $timeout seconds before next attempt ($retries out of $maxRetries)." 1>&2
sleep $timeout
done
echo "Failed to execute '$@' for $maxRetries times." 1>&2
return 1
}
function GetDotNetInstallScript { function GetDotNetInstallScript {
local root=$1 local root=$1
local install_script="$root/dotnet-install.sh" local install_script="$root/dotnet-install.sh"
@ -217,13 +252,13 @@ function GetDotNetInstallScript {
# Use curl if available, otherwise use wget # Use curl if available, otherwise use wget
if command -v curl > /dev/null; then if command -v curl > /dev/null; then
curl "$install_script_url" -sSL --retry 10 --create-dirs -o "$install_script" || { with_retries curl "$install_script_url" -sSL --retry 10 --create-dirs -o "$install_script" || {
local exit_code=$? local exit_code=$?
Write-PipelineTelemetryError -category 'InitializeToolset' "Failed to acquire dotnet install script (exit code '$exit_code')." Write-PipelineTelemetryError -category 'InitializeToolset' "Failed to acquire dotnet install script (exit code '$exit_code')."
ExitWithExitCode $exit_code ExitWithExitCode $exit_code
} }
else else
wget -q -O "$install_script" "$install_script_url" || { with_retries wget -v -O "$install_script" "$install_script_url" || {
local exit_code=$? local exit_code=$?
Write-PipelineTelemetryError -category 'InitializeToolset' "Failed to acquire dotnet install script (exit code '$exit_code')." Write-PipelineTelemetryError -category 'InitializeToolset' "Failed to acquire dotnet install script (exit code '$exit_code')."
ExitWithExitCode $exit_code ExitWithExitCode $exit_code
@ -238,11 +273,11 @@ function InitializeBuildTool {
if [[ -n "${_InitializeBuildTool:-}" ]]; then if [[ -n "${_InitializeBuildTool:-}" ]]; then
return return
fi fi
InitializeDotNetCli $restore InitializeDotNetCli $restore
# return values # return values
_InitializeBuildTool="$_InitializeDotNetCli/dotnet" _InitializeBuildTool="$_InitializeDotNetCli/dotnet"
_InitializeBuildToolCommand="msbuild" _InitializeBuildToolCommand="msbuild"
_InitializeBuildToolFramework="netcoreapp2.1" _InitializeBuildToolFramework="netcoreapp2.1"
} }
@ -261,6 +296,9 @@ function GetNuGetPackageCachePath {
} }
function InitializeNativeTools() { function InitializeNativeTools() {
if [[ -n "${DisableNativeToolsetInstalls:-}" ]]; then
return
fi
if grep -Fq "native-tools" $global_json_file if grep -Fq "native-tools" $global_json_file
then then
local nativeArgs="" local nativeArgs=""
@ -303,14 +341,14 @@ function InitializeToolset {
if [[ "$binary_log" == true ]]; then if [[ "$binary_log" == true ]]; then
bl="/bl:$log_dir/ToolsetRestore.binlog" bl="/bl:$log_dir/ToolsetRestore.binlog"
fi fi
echo '<Project Sdk="Microsoft.DotNet.Arcade.Sdk"/>' > "$proj" echo '<Project Sdk="Microsoft.DotNet.Arcade.Sdk"/>' > "$proj"
MSBuild-Core "$proj" $bl /t:__WriteToolsetLocation /clp:ErrorsOnly\;NoSummary /p:__ToolsetLocationOutputFile="$toolset_location_file" MSBuild-Core "$proj" $bl /t:__WriteToolsetLocation /clp:ErrorsOnly\;NoSummary /p:__ToolsetLocationOutputFile="$toolset_location_file"
local toolset_build_proj=`cat "$toolset_location_file"` local toolset_build_proj=`cat "$toolset_location_file"`
if [[ ! -a "$toolset_build_proj" ]]; then if [[ ! -a "$toolset_build_proj" ]]; then
Write-PipelineTelemetryError -category 'InitializeToolset' "Invalid toolset path: $toolset_build_proj" Write-PipelineTelemetryError -category 'Build' "Invalid toolset path: $toolset_build_proj"
ExitWithExitCode 3 ExitWithExitCode 3
fi fi
@ -341,7 +379,12 @@ function MSBuild {
# Work around issues with Azure Artifacts credential provider # Work around issues with Azure Artifacts credential provider
# https://github.com/dotnet/arcade/issues/3932 # https://github.com/dotnet/arcade/issues/3932
if [[ "$ci" == true ]]; then if [[ "$ci" == true ]]; then
dotnet nuget locals http-cache -c "$_InitializeBuildTool" nuget locals http-cache -c
export NUGET_PLUGIN_HANDSHAKE_TIMEOUT_IN_SECONDS=20
export NUGET_PLUGIN_REQUEST_TIMEOUT_IN_SECONDS=20
Write-PipelineSetVariable -name "NUGET_PLUGIN_HANDSHAKE_TIMEOUT_IN_SECONDS" -value "20"
Write-PipelineSetVariable -name "NUGET_PLUGIN_REQUEST_TIMEOUT_IN_SECONDS" -value "20"
fi fi
local toolset_dir="${_InitializeToolset%/*}" local toolset_dir="${_InitializeToolset%/*}"
@ -355,12 +398,12 @@ function MSBuild {
function MSBuild-Core { function MSBuild-Core {
if [[ "$ci" == true ]]; then if [[ "$ci" == true ]]; then
if [[ "$binary_log" != true ]]; then if [[ "$binary_log" != true ]]; then
Write-PipelineTaskError "Binary log must be enabled in CI build." Write-PipelineTelemetryError -category 'Build' "Binary log must be enabled in CI build."
ExitWithExitCode 1 ExitWithExitCode 1
fi fi
if [[ "$node_reuse" == true ]]; then if [[ "$node_reuse" == true ]]; then
Write-PipelineTaskError "Node reuse must be disabled in CI build." Write-PipelineTelemetryError -category 'Build' "Node reuse must be disabled in CI build."
ExitWithExitCode 1 ExitWithExitCode 1
fi fi
fi fi
@ -374,7 +417,7 @@ function MSBuild-Core {
"$_InitializeBuildTool" "$_InitializeBuildToolCommand" /m /nologo /clp:Summary /v:$verbosity /nr:$node_reuse $warnaserror_switch /p:TreatWarningsAsErrors=$warn_as_error /p:ContinuousIntegrationBuild=$ci "$@" || { "$_InitializeBuildTool" "$_InitializeBuildToolCommand" /m /nologo /clp:Summary /v:$verbosity /nr:$node_reuse $warnaserror_switch /p:TreatWarningsAsErrors=$warn_as_error /p:ContinuousIntegrationBuild=$ci "$@" || {
local exit_code=$? local exit_code=$?
Write-PipelineTaskError "Build failed (exit code '$exit_code')." Write-PipelineTelemetryError -category 'Build' "Build failed (exit code '$exit_code')."
ExitWithExitCode $exit_code ExitWithExitCode $exit_code
} }
} }
@ -415,3 +458,18 @@ Write-PipelineSetVariable -name "Artifacts.Toolset" -value "$toolset_dir"
Write-PipelineSetVariable -name "Artifacts.Log" -value "$log_dir" Write-PipelineSetVariable -name "Artifacts.Log" -value "$log_dir"
Write-PipelineSetVariable -name "Temp" -value "$temp_dir" Write-PipelineSetVariable -name "Temp" -value "$temp_dir"
Write-PipelineSetVariable -name "TMP" -value "$temp_dir" Write-PipelineSetVariable -name "TMP" -value "$temp_dir"
# Import custom tools configuration, if present in the repo.
if [ -z "${disable_configure_toolset_import:-}" ]; then
configure_toolset_script="$eng_root/configure-toolset.sh"
if [[ -a "$configure_toolset_script" ]]; then
. "$configure_toolset_script"
fi
fi
# TODO: https://github.com/dotnet/arcade/issues/1468
# Temporary workaround to avoid breaking change.
# Remove once repos are updated.
if [[ -n "${useInstalledDotNetCli:-}" ]]; then
use_installed_dotnet_cli="$useInstalledDotNetCli"
fi

4
global.json

@ -7,7 +7,7 @@
}, },
"msbuild-sdks": { "msbuild-sdks": {
"Microsoft.DotNet.Arcade.Sdk": "1.0.0-beta.19517.3", "Microsoft.DotNet.Arcade.Sdk": "5.0.0-beta.20180.5",
"Microsoft.DotNet.Helix.Sdk": "2.0.0-beta.19517.3" "Microsoft.DotNet.Helix.Sdk": "5.0.0-beta.20180.5"
} }
} }

Loading…
Cancel
Save