Browse Source

Merge branch 'main' into philliphoff-arm64

philliphoff-arm64
Phillip Hoff 5 years ago
parent
commit
edf0a9fc19
  1. 8
      docs/getting_started.md
  2. 20
      eng/Version.Details.xml
  3. 2
      eng/Versions.props
  4. 6
      eng/common/cross/build-rootfs.sh
  5. 109
      eng/common/sdl/configure-sdl-tool.ps1
  6. 71
      eng/common/sdl/execute-all-sdl-tools.ps1
  7. 63
      eng/common/sdl/extract-artifact-archives.ps1
  8. 50
      eng/common/sdl/run-sdl.ps1
  9. 108
      eng/common/templates/job/execute-sdl.yml
  10. 55
      eng/common/tools.ps1
  11. 4
      eng/common/tools.sh
  12. 2
      global.json
  13. 1
      src/Microsoft.Tye.Core/ContainerEngine.cs
  14. 6
      src/Microsoft.Tye.Core/ExecutableServiceBuilder.cs
  15. 20
      src/Microsoft.Tye.Core/LaunchedServiceBuilder.cs
  16. 6
      src/Microsoft.Tye.Core/ProjectServiceBuilder.cs
  17. 25
      src/Microsoft.Tye.Extensions/Dapr/DaprExtension.cs
  18. 9
      src/Microsoft.Tye.Hosting/DockerRunner.cs

8
docs/getting_started.md

@ -8,13 +8,13 @@ Tye is a tool that makes developing, testing, and deploying microservices and di
1. Install tye via the following command: 1. Install tye via the following command:
```text ```text
dotnet tool install -g Microsoft.Tye --version "0.7.0-alpha.21279.2" dotnet tool install -g Microsoft.Tye --version "0.8.0-alpha.21352.1"
``` ```
OR if you already have Tye installed and want to update: OR if you already have Tye installed and want to update:
```text ```text
dotnet tool update -g Microsoft.Tye --version "0.7.0-alpha.21279.2" dotnet tool update -g Microsoft.Tye --version "0.8.0-alpha.21352.1"
``` ```
> If using Mac and, if getting "command not found" errors when running `tye`, you may need to ensure that the `$HOME/.dotnet/tools` directory has been added to `PATH`. > If using Mac and, if getting "command not found" errors when running `tye`, you may need to ensure that the `$HOME/.dotnet/tools` directory has been added to `PATH`.
@ -37,13 +37,13 @@ Tye is a tool that makes developing, testing, and deploying microservices and di
This will install the newest available build from our CI. This will install the newest available build from our CI.
```txt ```txt
dotnet tool install -g Microsoft.Tye --version "0.8.0-*" --add-source https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet5/nuget/v3/index.json dotnet tool install -g Microsoft.Tye --version "0.9.0-*" --add-source https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet5/nuget/v3/index.json
``` ```
If you already have a build installed and you want to update, replace `install` with `update`: If you already have a build installed and you want to update, replace `install` with `update`:
```txt ```txt
dotnet tool update -g Microsoft.Tye --version "0.8.0-*" --add-source https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet5/nuget/v3/index.json dotnet tool update -g Microsoft.Tye --version "0.9.0-*" --add-source https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet5/nuget/v3/index.json
``` ```
> :bulb: Note that the version numbers for our CI builds and released packages will usually be different. > :bulb: Note that the version numbers for our CI builds and released packages will usually be different.

20
eng/Version.Details.xml

@ -3,25 +3,25 @@
<ProductDependencies> <ProductDependencies>
</ProductDependencies> </ProductDependencies>
<ToolsetDependencies> <ToolsetDependencies>
<Dependency Name="Microsoft.DotNet.Arcade.Sdk" Version="6.0.0-beta.21357.3"> <Dependency Name="Microsoft.DotNet.Arcade.Sdk" Version="6.0.0-beta.21366.1">
<Uri>https://github.com/dotnet/arcade</Uri> <Uri>https://github.com/dotnet/arcade</Uri>
<Sha>286d98094b830b8dad769542b2669cb1b75f7097</Sha> <Sha>b03966cd85285e425ffe56003c0ab57e103dd14e</Sha>
</Dependency> </Dependency>
<Dependency Name="Microsoft.DotNet.Build.Tasks.Feed" Version="6.0.0-beta.21357.3"> <Dependency Name="Microsoft.DotNet.Build.Tasks.Feed" Version="6.0.0-beta.21366.1">
<Uri>https://github.com/dotnet/arcade</Uri> <Uri>https://github.com/dotnet/arcade</Uri>
<Sha>286d98094b830b8dad769542b2669cb1b75f7097</Sha> <Sha>b03966cd85285e425ffe56003c0ab57e103dd14e</Sha>
</Dependency> </Dependency>
<Dependency Name="Microsoft.DotNet.SignTool" Version="6.0.0-beta.21357.3"> <Dependency Name="Microsoft.DotNet.SignTool" Version="6.0.0-beta.21366.1">
<Uri>https://github.com/dotnet/arcade</Uri> <Uri>https://github.com/dotnet/arcade</Uri>
<Sha>286d98094b830b8dad769542b2669cb1b75f7097</Sha> <Sha>b03966cd85285e425ffe56003c0ab57e103dd14e</Sha>
</Dependency> </Dependency>
<Dependency Name="Microsoft.DotNet.Helix.Sdk" Version="6.0.0-beta.21357.3"> <Dependency Name="Microsoft.DotNet.Helix.Sdk" Version="6.0.0-beta.21366.1">
<Uri>https://github.com/dotnet/arcade</Uri> <Uri>https://github.com/dotnet/arcade</Uri>
<Sha>286d98094b830b8dad769542b2669cb1b75f7097</Sha> <Sha>b03966cd85285e425ffe56003c0ab57e103dd14e</Sha>
</Dependency> </Dependency>
<Dependency Name="Microsoft.DotNet.SwaggerGenerator.MSBuild" Version="6.0.0-beta.21357.3"> <Dependency Name="Microsoft.DotNet.SwaggerGenerator.MSBuild" Version="6.0.0-beta.21366.1">
<Uri>https://github.com/dotnet/arcade</Uri> <Uri>https://github.com/dotnet/arcade</Uri>
<Sha>286d98094b830b8dad769542b2669cb1b75f7097</Sha> <Sha>b03966cd85285e425ffe56003c0ab57e103dd14e</Sha>
</Dependency> </Dependency>
<Dependency Name="Microsoft.DotNet.Maestro.Client" Version="1.1.0-beta.19556.4"> <Dependency Name="Microsoft.DotNet.Maestro.Client" Version="1.1.0-beta.19556.4">
<Uri>https://github.com/dotnet/arcade-services</Uri> <Uri>https://github.com/dotnet/arcade-services</Uri>

2
eng/Versions.props

@ -2,7 +2,7 @@
<Project> <Project>
<PropertyGroup> <PropertyGroup>
<!-- This repo version --> <!-- This repo version -->
<VersionPrefix>0.8.0</VersionPrefix> <VersionPrefix>0.9.0</VersionPrefix>
<PreReleaseVersionLabel>alpha</PreReleaseVersionLabel> <PreReleaseVersionLabel>alpha</PreReleaseVersionLabel>
<!-- Opt-out repo features --> <!-- Opt-out repo features -->
<UsingToolXliff>false</UsingToolXliff> <UsingToolXliff>false</UsingToolXliff>

6
eng/common/cross/build-rootfs.sh

@ -33,7 +33,6 @@ __AlpinePackages="alpine-base"
__AlpinePackages+=" build-base" __AlpinePackages+=" build-base"
__AlpinePackages+=" linux-headers" __AlpinePackages+=" linux-headers"
__AlpinePackagesEdgeCommunity=" lldb-dev" __AlpinePackagesEdgeCommunity=" lldb-dev"
__AlpinePackagesEdgeMain=" llvm10-libs"
__AlpinePackagesEdgeMain+=" python3" __AlpinePackagesEdgeMain+=" python3"
__AlpinePackagesEdgeMain+=" libedit" __AlpinePackagesEdgeMain+=" libedit"
@ -115,6 +114,8 @@ while :; do
__UbuntuArch=s390x __UbuntuArch=s390x
__UbuntuRepo="http://ports.ubuntu.com/ubuntu-ports/" __UbuntuRepo="http://ports.ubuntu.com/ubuntu-ports/"
__UbuntuPackages=$(echo ${__UbuntuPackages} | sed 's/ libunwind8-dev//') __UbuntuPackages=$(echo ${__UbuntuPackages} | sed 's/ libunwind8-dev//')
__UbuntuPackages=$(echo ${__UbuntuPackages} | sed 's/ libomp-dev//')
__UbuntuPackages=$(echo ${__UbuntuPackages} | sed 's/ libomp5//')
unset __LLDB_Package unset __LLDB_Package
;; ;;
x86) x86)
@ -191,6 +192,8 @@ while :; do
__CodeName=alpine __CodeName=alpine
__UbuntuRepo= __UbuntuRepo=
__AlpineVersion=3.9 __AlpineVersion=3.9
__AlpinePackagesEdgeMain+=" llvm11-libs"
__AlpinePackagesEdgeMain+=" clang-libs"
;; ;;
alpine3.13) alpine3.13)
__CodeName=alpine __CodeName=alpine
@ -201,6 +204,7 @@ while :; do
__AlpinePackagesEdgeCommunity= __AlpinePackagesEdgeCommunity=
__AlpinePackages+=$__AlpinePackagesEdgeMain __AlpinePackages+=$__AlpinePackagesEdgeMain
__AlpinePackagesEdgeMain= __AlpinePackagesEdgeMain=
__AlpinePackages+=" llvm10-libs"
;; ;;
freebsd11) freebsd11)
__FreeBSDBase="11.3-RELEASE" __FreeBSDBase="11.3-RELEASE"

109
eng/common/sdl/configure-sdl-tool.ps1

@ -0,0 +1,109 @@
Param(
[string] $GuardianCliLocation,
[string] $WorkingDirectory,
[string] $TargetDirectory,
[string] $GdnFolder,
# The list of Guardian tools to configure. For each object in the array:
# - If the item is a [hashtable], it must contain these entries:
# - Name = The tool name as Guardian knows it.
# - Scenario = (Optional) Scenario-specific name for this configuration entry. It must be unique
# among all tool entries with the same Name.
# - Args = (Optional) Array of Guardian tool configuration args, like '@("Target > C:\temp")'
# - If the item is a [string] $v, it is treated as '@{ Name="$v" }'
[object[]] $ToolsList,
[string] $GuardianLoggerLevel='Standard',
# Optional: Additional params to add to any tool using CredScan.
[string[]] $CrScanAdditionalRunConfigParams,
# Optional: Additional params to add to any tool using PoliCheck.
[string[]] $PoliCheckAdditionalRunConfigParams
)
$ErrorActionPreference = 'Stop'
Set-StrictMode -Version 2.0
$disableConfigureToolsetImport = $true
$global:LASTEXITCODE = 0
try {
# `tools.ps1` checks $ci to perform some actions. Since the SDL
# scripts don't necessarily execute in the same agent that run the
# build.ps1/sh script this variable isn't automatically set.
$ci = $true
. $PSScriptRoot\..\tools.ps1
# Normalize tools list: all in [hashtable] form with defined values for each key.
$ToolsList = $ToolsList |
ForEach-Object {
if ($_ -is [string]) {
$_ = @{ Name = $_ }
}
if (-not ($_['Scenario'])) { $_.Scenario = "" }
if (-not ($_['Args'])) { $_.Args = @() }
$_
}
Write-Host "List of tools to configure:"
$ToolsList | ForEach-Object { $_ | Out-String | Write-Host }
# We store config files in the r directory of .gdn
$gdnConfigPath = Join-Path $GdnFolder 'r'
$ValidPath = Test-Path $GuardianCliLocation
if ($ValidPath -eq $False)
{
Write-PipelineTelemetryError -Force -Category 'Sdl' -Message "Invalid Guardian CLI Location."
ExitWithExitCode 1
}
foreach ($tool in $ToolsList) {
# Put together the name and scenario to make a unique key.
$toolConfigName = $tool.Name
if ($tool.Scenario) {
$toolConfigName += "_" + $tool.Scenario
}
Write-Host "=== Configuring $toolConfigName..."
$gdnConfigFile = Join-Path $gdnConfigPath "$toolConfigName-configure.gdnconfig"
# For some tools, add default and automatic args.
if ($tool.Name -eq 'credscan') {
if ($targetDirectory) {
$tool.Args += "TargetDirectory < $TargetDirectory"
}
$tool.Args += "OutputType < pre"
$tool.Args += $CrScanAdditionalRunConfigParams
} elseif ($tool.Name -eq 'policheck') {
if ($targetDirectory) {
$tool.Args += "Target < $TargetDirectory"
}
$tool.Args += $PoliCheckAdditionalRunConfigParams
}
# Create variable pointing to the args array directly so we can use splat syntax later.
$toolArgs = $tool.Args
# Configure the tool. If args array is provided or the current tool has some default arguments
# defined, add "--args" and splat each element on the end. Arg format is "{Arg id} < {Value}",
# one per parameter. Doc page for "guardian configure":
# https://dev.azure.com/securitytools/SecurityIntegration/_wiki/wikis/Guardian/1395/configure
Exec-BlockVerbosely {
& $GuardianCliLocation configure `
--working-directory $WorkingDirectory `
--tool $tool.Name `
--output-path $gdnConfigFile `
--logger-level $GuardianLoggerLevel `
--noninteractive `
--force `
$(if ($toolArgs) { "--args" }) @toolArgs
Exit-IfNZEC "Sdl"
}
Write-Host "Created '$toolConfigName' configuration file: $gdnConfigFile"
}
}
catch {
Write-Host $_.ScriptStackTrace
Write-PipelineTelemetryError -Force -Category 'Sdl' -Message $_
ExitWithExitCode 1
}

71
eng/common/sdl/execute-all-sdl-tools.ps1

@ -7,8 +7,17 @@ Param(
[string] $SourceDirectory=$env:BUILD_SOURCESDIRECTORY, # Required: the directory where source files are located [string] $SourceDirectory=$env:BUILD_SOURCESDIRECTORY, # Required: the directory where source files are located
[string] $ArtifactsDirectory = (Join-Path $env:BUILD_ARTIFACTSTAGINGDIRECTORY ('artifacts')), # Required: the directory where build artifacts are located [string] $ArtifactsDirectory = (Join-Path $env:BUILD_ARTIFACTSTAGINGDIRECTORY ('artifacts')), # Required: the directory where build artifacts are located
[string] $AzureDevOpsAccessToken, # Required: access token for dnceng; should be provided via KeyVault [string] $AzureDevOpsAccessToken, # Required: access token for dnceng; should be provided via KeyVault
[string[]] $SourceToolsList, # Optional: list of SDL tools to run on source code
[string[]] $ArtifactToolsList, # Optional: list of SDL tools to run on built artifacts # Optional: list of SDL tools to run on source code. See 'configure-sdl-tool.ps1' for tools list
# format.
[object[]] $SourceToolsList,
# Optional: list of SDL tools to run on built artifacts. See 'configure-sdl-tool.ps1' for tools
# list format.
[object[]] $ArtifactToolsList,
# Optional: list of SDL tools to run without automatically specifying a target directory. See
# 'configure-sdl-tool.ps1' for tools list format.
[object[]] $CustomToolsList,
[bool] $TsaPublish=$False, # Optional: true will publish results to TSA; only set to true after onboarding to TSA; TSA is the automated framework used to upload test results as bugs. [bool] $TsaPublish=$False, # Optional: true will publish results to TSA; only set to true after onboarding to TSA; TSA is the automated framework used to upload test results as bugs.
[string] $TsaBranchName=$env:BUILD_SOURCEBRANCH, # Optional: required for TSA publish; defaults to $(Build.SourceBranchName); TSA is the automated framework used to upload test results as bugs. [string] $TsaBranchName=$env:BUILD_SOURCEBRANCH, # Optional: required for TSA publish; defaults to $(Build.SourceBranchName); TSA is the automated framework used to upload test results as bugs.
[string] $TsaRepositoryName=$env:BUILD_REPOSITORY_NAME, # Optional: TSA repository name; will be generated automatically if not submitted; TSA is the automated framework used to upload test results as bugs. [string] $TsaRepositoryName=$env:BUILD_REPOSITORY_NAME, # Optional: TSA repository name; will be generated automatically if not submitted; TSA is the automated framework used to upload test results as bugs.
@ -63,13 +72,16 @@ try {
ExitWithExitCode 1 ExitWithExitCode 1
} }
& $(Join-Path $PSScriptRoot 'init-sdl.ps1') -GuardianCliLocation $guardianCliLocation -Repository $RepoName -BranchName $BranchName -WorkingDirectory $workingDirectory -AzureDevOpsAccessToken $AzureDevOpsAccessToken -GuardianLoggerLevel $GuardianLoggerLevel Exec-BlockVerbosely {
& $(Join-Path $PSScriptRoot 'init-sdl.ps1') -GuardianCliLocation $guardianCliLocation -Repository $RepoName -BranchName $BranchName -WorkingDirectory $workingDirectory -AzureDevOpsAccessToken $AzureDevOpsAccessToken -GuardianLoggerLevel $GuardianLoggerLevel
}
$gdnFolder = Join-Path $workingDirectory '.gdn' $gdnFolder = Join-Path $workingDirectory '.gdn'
if ($TsaOnboard) { if ($TsaOnboard) {
if ($TsaCodebaseName -and $TsaNotificationEmail -and $TsaCodebaseAdmin -and $TsaBugAreaPath) { if ($TsaCodebaseName -and $TsaNotificationEmail -and $TsaCodebaseAdmin -and $TsaBugAreaPath) {
Write-Host "$guardianCliLocation tsa-onboard --codebase-name `"$TsaCodebaseName`" --notification-alias `"$TsaNotificationEmail`" --codebase-admin `"$TsaCodebaseAdmin`" --instance-url `"$TsaInstanceUrl`" --project-name `"$TsaProjectName`" --area-path `"$TsaBugAreaPath`" --iteration-path `"$TsaIterationPath`" --working-directory $workingDirectory --logger-level $GuardianLoggerLevel" Exec-BlockVerbosely {
& $guardianCliLocation tsa-onboard --codebase-name "$TsaCodebaseName" --notification-alias "$TsaNotificationEmail" --codebase-admin "$TsaCodebaseAdmin" --instance-url "$TsaInstanceUrl" --project-name "$TsaProjectName" --area-path "$TsaBugAreaPath" --iteration-path "$TsaIterationPath" --working-directory $workingDirectory --logger-level $GuardianLoggerLevel & $guardianCliLocation tsa-onboard --codebase-name "$TsaCodebaseName" --notification-alias "$TsaNotificationEmail" --codebase-admin "$TsaCodebaseAdmin" --instance-url "$TsaInstanceUrl" --project-name "$TsaProjectName" --area-path "$TsaBugAreaPath" --iteration-path "$TsaIterationPath" --working-directory $workingDirectory --logger-level $GuardianLoggerLevel
}
if ($LASTEXITCODE -ne 0) { if ($LASTEXITCODE -ne 0) {
Write-PipelineTelemetryError -Force -Category 'Sdl' -Message "Guardian tsa-onboard failed with exit code $LASTEXITCODE." Write-PipelineTelemetryError -Force -Category 'Sdl' -Message "Guardian tsa-onboard failed with exit code $LASTEXITCODE."
ExitWithExitCode $LASTEXITCODE ExitWithExitCode $LASTEXITCODE
@ -80,11 +92,41 @@ try {
} }
} }
if ($ArtifactToolsList -and $ArtifactToolsList.Count -gt 0) { # Configure a list of tools with a default target directory. Populates the ".gdn/r" directory.
& $(Join-Path $PSScriptRoot 'run-sdl.ps1') -GuardianCliLocation $guardianCliLocation -WorkingDirectory $workingDirectory -TargetDirectory $ArtifactsDirectory -GdnFolder $gdnFolder -ToolsList $ArtifactToolsList -AzureDevOpsAccessToken $AzureDevOpsAccessToken -UpdateBaseline $UpdateBaseline -GuardianLoggerLevel $GuardianLoggerLevel -CrScanAdditionalRunConfigParams $CrScanAdditionalRunConfigParams -PoliCheckAdditionalRunConfigParams $PoliCheckAdditionalRunConfigParams function Configure-ToolsList([object[]] $tools, [string] $targetDirectory) {
if ($tools -and $tools.Count -gt 0) {
Exec-BlockVerbosely {
& $(Join-Path $PSScriptRoot 'configure-sdl-tool.ps1') `
-GuardianCliLocation $guardianCliLocation `
-WorkingDirectory $workingDirectory `
-TargetDirectory $targetDirectory `
-GdnFolder $gdnFolder `
-ToolsList $tools `
-AzureDevOpsAccessToken $AzureDevOpsAccessToken `
-GuardianLoggerLevel $GuardianLoggerLevel `
-CrScanAdditionalRunConfigParams $CrScanAdditionalRunConfigParams `
-PoliCheckAdditionalRunConfigParams $PoliCheckAdditionalRunConfigParams
if ($BreakOnFailure) {
Exit-IfNZEC "Sdl"
}
}
}
} }
if ($SourceToolsList -and $SourceToolsList.Count -gt 0) {
& $(Join-Path $PSScriptRoot 'run-sdl.ps1') -GuardianCliLocation $guardianCliLocation -WorkingDirectory $workingDirectory -TargetDirectory $SourceDirectory -GdnFolder $gdnFolder -ToolsList $SourceToolsList -AzureDevOpsAccessToken $AzureDevOpsAccessToken -UpdateBaseline $UpdateBaseline -GuardianLoggerLevel $GuardianLoggerLevel -CrScanAdditionalRunConfigParams $CrScanAdditionalRunConfigParams -PoliCheckAdditionalRunConfigParams $PoliCheckAdditionalRunConfigParams # Configure Artifact and Source tools with default Target directories.
Configure-ToolsList $ArtifactToolsList $ArtifactsDirectory
Configure-ToolsList $SourceToolsList $SourceDirectory
# Configure custom tools with no default Target directory.
Configure-ToolsList $CustomToolsList $null
# At this point, all tools are configured in the ".gdn" directory. Run them all in a single call.
# (If we used "run" multiple times, each run would overwrite data from earlier runs.)
Exec-BlockVerbosely {
& $(Join-Path $PSScriptRoot 'run-sdl.ps1') `
-GuardianCliLocation $guardianCliLocation `
-WorkingDirectory $workingDirectory `
-UpdateBaseline $UpdateBaseline `
-GdnFolder $gdnFolder
} }
if ($TsaPublish) { if ($TsaPublish) {
@ -92,8 +134,9 @@ try {
if (-not $TsaRepositoryName) { if (-not $TsaRepositoryName) {
$TsaRepositoryName = "$($Repository)-$($BranchName)" $TsaRepositoryName = "$($Repository)-$($BranchName)"
} }
Write-Host "$guardianCliLocation tsa-publish --all-tools --repository-name `"$TsaRepositoryName`" --branch-name `"$TsaBranchName`" --build-number `"$BuildNumber`" --codebase-name `"$TsaCodebaseName`" --notification-alias `"$TsaNotificationEmail`" --codebase-admin `"$TsaCodebaseAdmin`" --instance-url `"$TsaInstanceUrl`" --project-name `"$TsaProjectName`" --area-path `"$TsaBugAreaPath`" --iteration-path `"$TsaIterationPath`" --working-directory $workingDirectory --logger-level $GuardianLoggerLevel" Exec-BlockVerbosely {
& $guardianCliLocation tsa-publish --all-tools --repository-name "$TsaRepositoryName" --branch-name "$TsaBranchName" --build-number "$BuildNumber" --onboard $True --codebase-name "$TsaCodebaseName" --notification-alias "$TsaNotificationEmail" --codebase-admin "$TsaCodebaseAdmin" --instance-url "$TsaInstanceUrl" --project-name "$TsaProjectName" --area-path "$TsaBugAreaPath" --iteration-path "$TsaIterationPath" --working-directory $workingDirectory --logger-level $GuardianLoggerLevel & $guardianCliLocation tsa-publish --all-tools --repository-name "$TsaRepositoryName" --branch-name "$TsaBranchName" --build-number "$BuildNumber" --onboard $True --codebase-name "$TsaCodebaseName" --notification-alias "$TsaNotificationEmail" --codebase-admin "$TsaCodebaseAdmin" --instance-url "$TsaInstanceUrl" --project-name "$TsaProjectName" --area-path "$TsaBugAreaPath" --iteration-path "$TsaIterationPath" --working-directory $workingDirectory --logger-level $GuardianLoggerLevel
}
if ($LASTEXITCODE -ne 0) { if ($LASTEXITCODE -ne 0) {
Write-PipelineTelemetryError -Force -Category 'Sdl' -Message "Guardian tsa-publish failed with exit code $LASTEXITCODE." Write-PipelineTelemetryError -Force -Category 'Sdl' -Message "Guardian tsa-publish failed with exit code $LASTEXITCODE."
ExitWithExitCode $LASTEXITCODE ExitWithExitCode $LASTEXITCODE
@ -106,7 +149,11 @@ try {
if ($BreakOnFailure) { if ($BreakOnFailure) {
Write-Host "Failing the build in case of breaking results..." Write-Host "Failing the build in case of breaking results..."
& $guardianCliLocation break Exec-BlockVerbosely {
& $guardianCliLocation break --working-directory $workingDirectory --logger-level $GuardianLoggerLevel
}
} else {
Write-Host "Letting the build pass even if there were breaking results..."
} }
} }
catch { catch {

63
eng/common/sdl/extract-artifact-archives.ps1

@ -0,0 +1,63 @@
# This script looks for each archive file in a directory and extracts it into the target directory.
# For example, the file "$InputPath/bin.tar.gz" extracts to "$ExtractPath/bin.tar.gz.extracted/**".
# Uses the "tar" utility added to Windows 10 / Windows 2019 that supports tar.gz and zip.
param(
# Full path to directory where archives are stored.
[Parameter(Mandatory=$true)][string] $InputPath,
# Full path to directory to extract archives into. May be the same as $InputPath.
[Parameter(Mandatory=$true)][string] $ExtractPath
)
$ErrorActionPreference = 'Stop'
Set-StrictMode -Version 2.0
$disableConfigureToolsetImport = $true
try {
# `tools.ps1` checks $ci to perform some actions. Since the SDL
# scripts don't necessarily execute in the same agent that run the
# build.ps1/sh script this variable isn't automatically set.
$ci = $true
. $PSScriptRoot\..\tools.ps1
Measure-Command {
$jobs = @()
# Find archive files for non-Windows and Windows builds.
$archiveFiles = @(
Get-ChildItem (Join-Path $InputPath "*.tar.gz")
Get-ChildItem (Join-Path $InputPath "*.zip")
)
foreach ($targzFile in $archiveFiles) {
$jobs += Start-Job -ScriptBlock {
$file = $using:targzFile
$fileName = [System.IO.Path]::GetFileName($file)
$extractDir = Join-Path $using:ExtractPath "$fileName.extracted"
New-Item $extractDir -ItemType Directory -Force | Out-Null
Write-Host "Extracting '$file' to '$extractDir'..."
# Pipe errors to stdout to prevent PowerShell detecting them and quitting the job early.
# This type of quit skips the catch, so we wouldn't be able to tell which file triggered the
# error. Save output so it can be stored in the exception string along with context.
$output = tar -xf $file -C $extractDir 2>&1
# Handle NZEC manually rather than using Exit-IfNZEC: we are in a background job, so we
# don't have access to the outer scope.
if ($LASTEXITCODE -ne 0) {
throw "Error extracting '$file': non-zero exit code ($LASTEXITCODE). Output: '$output'"
}
Write-Host "Extracted to $extractDir"
}
}
Receive-Job $jobs -Wait
}
}
catch {
Write-Host $_
Write-PipelineTelemetryError -Force -Category 'Sdl' -Message $_
ExitWithExitCode 1
}

50
eng/common/sdl/run-sdl.ps1

@ -1,13 +1,9 @@
Param( Param(
[string] $GuardianCliLocation, [string] $GuardianCliLocation,
[string] $WorkingDirectory, [string] $WorkingDirectory,
[string] $TargetDirectory,
[string] $GdnFolder, [string] $GdnFolder,
[string[]] $ToolsList,
[string] $UpdateBaseline, [string] $UpdateBaseline,
[string] $GuardianLoggerLevel='Standard', [string] $GuardianLoggerLevel='Standard'
[string[]] $CrScanAdditionalRunConfigParams,
[string[]] $PoliCheckAdditionalRunConfigParams
) )
$ErrorActionPreference = 'Stop' $ErrorActionPreference = 'Stop'
@ -23,7 +19,6 @@ try {
. $PSScriptRoot\..\tools.ps1 . $PSScriptRoot\..\tools.ps1
# We store config files in the r directory of .gdn # We store config files in the r directory of .gdn
Write-Host $ToolsList
$gdnConfigPath = Join-Path $GdnFolder 'r' $gdnConfigPath = Join-Path $GdnFolder 'r'
$ValidPath = Test-Path $GuardianCliLocation $ValidPath = Test-Path $GuardianCliLocation
@ -33,37 +28,18 @@ try {
ExitWithExitCode 1 ExitWithExitCode 1
} }
$configParam = @('--config') $gdnConfigFiles = Get-ChildItem $gdnConfigPath -Recurse -Include '*.gdnconfig'
Write-Host "Discovered Guardian config files:"
foreach ($tool in $ToolsList) { $gdnConfigFiles | Out-String | Write-Host
$gdnConfigFile = Join-Path $gdnConfigPath "$tool-configure.gdnconfig"
Write-Host $tool Exec-BlockVerbosely {
# We have to manually configure tools that run on source to look at the source directory only & $GuardianCliLocation run `
if ($tool -eq 'credscan') { --working-directory $WorkingDirectory `
Write-Host "$GuardianCliLocation configure --working-directory $WorkingDirectory --tool $tool --output-path $gdnConfigFile --logger-level $GuardianLoggerLevel --noninteractive --force --args `" TargetDirectory < $TargetDirectory `" `" OutputType < pre `" $(If ($CrScanAdditionalRunConfigParams) {$CrScanAdditionalRunConfigParams})" --baseline mainbaseline `
& $GuardianCliLocation configure --working-directory $WorkingDirectory --tool $tool --output-path $gdnConfigFile --logger-level $GuardianLoggerLevel --noninteractive --force --args " TargetDirectory < $TargetDirectory " "OutputType < pre" $(If ($CrScanAdditionalRunConfigParams) {$CrScanAdditionalRunConfigParams}) --update-baseline $UpdateBaseline `
if ($LASTEXITCODE -ne 0) { --logger-level $GuardianLoggerLevel `
Write-PipelineTelemetryError -Force -Category 'Sdl' -Message "Guardian configure for $tool failed with exit code $LASTEXITCODE." --config @gdnConfigFiles
ExitWithExitCode $LASTEXITCODE Exit-IfNZEC "Sdl"
}
}
if ($tool -eq 'policheck') {
Write-Host "$GuardianCliLocation configure --working-directory $WorkingDirectory --tool $tool --output-path $gdnConfigFile --logger-level $GuardianLoggerLevel --noninteractive --force --args `" Target < $TargetDirectory `" $(If ($PoliCheckAdditionalRunConfigParams) {$PoliCheckAdditionalRunConfigParams})"
& $GuardianCliLocation configure --working-directory $WorkingDirectory --tool $tool --output-path $gdnConfigFile --logger-level $GuardianLoggerLevel --noninteractive --force --args " Target < $TargetDirectory " $(If ($PoliCheckAdditionalRunConfigParams) {$PoliCheckAdditionalRunConfigParams})
if ($LASTEXITCODE -ne 0) {
Write-PipelineTelemetryError -Force -Category 'Sdl' -Message "Guardian configure for $tool failed with exit code $LASTEXITCODE."
ExitWithExitCode $LASTEXITCODE
}
}
$configParam+=$gdnConfigFile
}
Write-Host "$GuardianCliLocation run --working-directory $WorkingDirectory --baseline mainbaseline --update-baseline $UpdateBaseline --logger-level $GuardianLoggerLevel $configParam"
& $GuardianCliLocation run --working-directory $WorkingDirectory --tool $tool --baseline mainbaseline --update-baseline $UpdateBaseline --logger-level $GuardianLoggerLevel $configParam
if ($LASTEXITCODE -ne 0) {
Write-PipelineTelemetryError -Force -Category 'Sdl' -Message "Guardian run for $ToolsList using $configParam failed with exit code $LASTEXITCODE."
ExitWithExitCode $LASTEXITCODE
} }
} }
catch { catch {

108
eng/common/templates/job/execute-sdl.yml

@ -2,17 +2,41 @@ parameters:
enable: 'false' # Whether the SDL validation job should execute or not enable: 'false' # Whether the SDL validation job should execute or not
overrideParameters: '' # Optional: to override values for parameters. overrideParameters: '' # Optional: to override values for parameters.
additionalParameters: '' # Optional: parameters that need user specific values eg: '-SourceToolsList @("abc","def") -ArtifactToolsList @("ghi","jkl")' additionalParameters: '' # Optional: parameters that need user specific values eg: '-SourceToolsList @("abc","def") -ArtifactToolsList @("ghi","jkl")'
# Optional: if specified, restore and use this version of Guardian instead of the default.
overrideGuardianVersion: ''
# Optional: if true, publish the '.gdn' folder as a pipeline artifact. This can help with in-depth
# diagnosis of problems with specific tool configurations.
publishGuardianDirectoryToPipeline: false
# The script to run to execute all SDL tools. Use this if you want to use a script to define SDL
# parameters rather than relying on YAML. It may be better to use a local script, because you can
# reproduce results locally without piecing together a command based on the YAML.
executeAllSdlToolsScript: 'eng/common/sdl/execute-all-sdl-tools.ps1'
# There is some sort of bug (has been reported) in Azure DevOps where if this parameter is named # There is some sort of bug (has been reported) in Azure DevOps where if this parameter is named
# 'continueOnError', the parameter value is not correctly picked up. # 'continueOnError', the parameter value is not correctly picked up.
# This can also be remedied by the caller (post-build.yml) if it does not use a nested parameter # This can also be remedied by the caller (post-build.yml) if it does not use a nested parameter
sdlContinueOnError: false # optional: determines whether to continue the build if the step errors; sdlContinueOnError: false # optional: determines whether to continue the build if the step errors;
downloadArtifacts: true # optional: determines if the artifacts should be dowloaded # optional: determines if build artifacts should be downloaded.
downloadArtifacts: true
# optional: determines if this job should search the directory of downloaded artifacts for
# 'tar.gz' and 'zip' archive files and extract them before running SDL validation tasks.
extractArchiveArtifacts: false
dependsOn: '' # Optional: dependencies of the job dependsOn: '' # Optional: dependencies of the job
artifactNames: '' # Optional: patterns supplied to DownloadBuildArtifacts artifactNames: '' # Optional: patterns supplied to DownloadBuildArtifacts
# Usage: # Usage:
# artifactNames: # artifactNames:
# - 'BlobArtifacts' # - 'BlobArtifacts'
# - 'Artifacts_Windows_NT_Release' # - 'Artifacts_Windows_NT_Release'
# Optional: download a list of pipeline artifacts. 'downloadArtifacts' controls build artifacts,
# not pipeline artifacts, so doesn't affect the use of this parameter.
pipelineArtifactNames: []
# Optional: location and ID of the AzDO build that the build/pipeline artifacts should be
# downloaded from. By default, uses runtime expressions to decide based on the variables set by
# the 'setupMaestroVars' dependency. Overriding this parameter is necessary if SDL tasks are
# running without Maestro++/BAR involved, or to download artifacts from a specific existing build
# to iterate quickly on SDL changes.
AzDOProjectName: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOProjectName'] ]
AzDOPipelineId: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOPipelineId'] ]
AzDOBuildId: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOBuildId'] ]
jobs: jobs:
- job: Run_SDL - job: Run_SDL
@ -22,16 +46,29 @@ jobs:
variables: variables:
- group: DotNet-VSTS-Bot - group: DotNet-VSTS-Bot
- name: AzDOProjectName - name: AzDOProjectName
value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOProjectName'] ] value: ${{ parameters.AzDOProjectName }}
- name: AzDOPipelineId - name: AzDOPipelineId
value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOPipelineId'] ] value: ${{ parameters.AzDOPipelineId }}
- name: AzDOBuildId - name: AzDOBuildId
value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOBuildId'] ] value: ${{ parameters.AzDOBuildId }}
# The Guardian version specified in 'eng/common/sdl/packages.config'. This value must be kept in
# sync with the packages.config file.
- name: DefaultGuardianVersion
value: 0.53.3
- name: GuardianVersion
value: ${{ coalesce(parameters.overrideGuardianVersion, '$(DefaultGuardianVersion)') }}
- name: GuardianPackagesConfigFile
value: $(Build.SourcesDirectory)\eng\common\sdl\packages.config
pool: pool:
name: Hosted VS2017 # To extract archives (.tar.gz, .zip), we need access to "tar", added in Windows 10/2019.
${{ if eq(parameters.extractArchiveArtifacts, 'false') }}:
name: Hosted VS2017
${{ if ne(parameters.extractArchiveArtifacts, 'false') }}:
vmImage: windows-2019
steps: steps:
- checkout: self - checkout: self
clean: true clean: true
- ${{ if ne(parameters.downloadArtifacts, 'false')}}: - ${{ if ne(parameters.downloadArtifacts, 'false')}}:
- ${{ if ne(parameters.artifactNames, '') }}: - ${{ if ne(parameters.artifactNames, '') }}:
- ${{ each artifactName in parameters.artifactNames }}: - ${{ each artifactName in parameters.artifactNames }}:
@ -59,16 +96,51 @@ jobs:
itemPattern: "**" itemPattern: "**"
downloadPath: $(Build.ArtifactStagingDirectory)\artifacts downloadPath: $(Build.ArtifactStagingDirectory)\artifacts
checkDownloadedFiles: true checkDownloadedFiles: true
- ${{ each artifactName in parameters.pipelineArtifactNames }}:
- task: DownloadPipelineArtifact@2
displayName: Download Pipeline Artifacts
inputs:
buildType: specific
buildVersionToDownload: specific
project: $(AzDOProjectName)
pipeline: $(AzDOPipelineId)
buildId: $(AzDOBuildId)
artifactName: ${{ artifactName }}
downloadPath: $(Build.ArtifactStagingDirectory)\artifacts
checkDownloadedFiles: true
- powershell: eng/common/sdl/extract-artifact-packages.ps1 - powershell: eng/common/sdl/extract-artifact-packages.ps1
-InputPath $(Build.ArtifactStagingDirectory)\artifacts\BlobArtifacts -InputPath $(Build.ArtifactStagingDirectory)\artifacts\BlobArtifacts
-ExtractPath $(Build.ArtifactStagingDirectory)\artifacts\BlobArtifacts -ExtractPath $(Build.ArtifactStagingDirectory)\artifacts\BlobArtifacts
displayName: Extract Blob Artifacts displayName: Extract Blob Artifacts
continueOnError: ${{ parameters.sdlContinueOnError }} continueOnError: ${{ parameters.sdlContinueOnError }}
- powershell: eng/common/sdl/extract-artifact-packages.ps1 - powershell: eng/common/sdl/extract-artifact-packages.ps1
-InputPath $(Build.ArtifactStagingDirectory)\artifacts\PackageArtifacts -InputPath $(Build.ArtifactStagingDirectory)\artifacts\PackageArtifacts
-ExtractPath $(Build.ArtifactStagingDirectory)\artifacts\PackageArtifacts -ExtractPath $(Build.ArtifactStagingDirectory)\artifacts\PackageArtifacts
displayName: Extract Package Artifacts displayName: Extract Package Artifacts
continueOnError: ${{ parameters.sdlContinueOnError }} continueOnError: ${{ parameters.sdlContinueOnError }}
- ${{ if ne(parameters.extractArchiveArtifacts, 'false') }}:
- powershell: eng/common/sdl/extract-artifact-archives.ps1
-InputPath $(Build.ArtifactStagingDirectory)\artifacts
-ExtractPath $(Build.ArtifactStagingDirectory)\artifacts
displayName: Extract Archive Artifacts
continueOnError: ${{ parameters.sdlContinueOnError }}
- ${{ if ne(parameters.overrideGuardianVersion, '') }}:
- powershell: |
$content = Get-Content $(GuardianPackagesConfigFile)
Write-Host "packages.config content was:`n$content"
$content = $content.Replace('$(DefaultGuardianVersion)', '$(GuardianVersion)')
$content | Set-Content $(GuardianPackagesConfigFile)
Write-Host "packages.config content updated to:`n$content"
displayName: Use overridden Guardian version ${{ parameters.overrideGuardianVersion }}
- task: NuGetToolInstaller@1 - task: NuGetToolInstaller@1
displayName: 'Install NuGet.exe' displayName: 'Install NuGet.exe'
- task: NuGetCommand@2 - task: NuGetCommand@2
@ -79,15 +151,35 @@ jobs:
nugetConfigPath: $(Build.SourcesDirectory)\eng\common\sdl\NuGet.config nugetConfigPath: $(Build.SourcesDirectory)\eng\common\sdl\NuGet.config
externalFeedCredentials: GuardianConnect externalFeedCredentials: GuardianConnect
restoreDirectory: $(Build.SourcesDirectory)\.packages restoreDirectory: $(Build.SourcesDirectory)\.packages
- ${{ if ne(parameters.overrideParameters, '') }}: - ${{ if ne(parameters.overrideParameters, '') }}:
- powershell: eng/common/sdl/execute-all-sdl-tools.ps1 ${{ parameters.overrideParameters }} - powershell: ${{ parameters.executeAllSdlToolsScript }} ${{ parameters.overrideParameters }}
displayName: Execute SDL displayName: Execute SDL
continueOnError: ${{ parameters.sdlContinueOnError }} continueOnError: ${{ parameters.sdlContinueOnError }}
- ${{ if eq(parameters.overrideParameters, '') }}: - ${{ if eq(parameters.overrideParameters, '') }}:
- powershell: eng/common/sdl/execute-all-sdl-tools.ps1 - powershell: ${{ parameters.executeAllSdlToolsScript }}
-GuardianPackageName Microsoft.Guardian.Cli.0.53.3 -GuardianPackageName Microsoft.Guardian.Cli.$(GuardianVersion)
-NugetPackageDirectory $(Build.SourcesDirectory)\.packages -NugetPackageDirectory $(Build.SourcesDirectory)\.packages
-AzureDevOpsAccessToken $(dn-bot-dotnet-build-rw-code-rw) -AzureDevOpsAccessToken $(dn-bot-dotnet-build-rw-code-rw)
${{ parameters.additionalParameters }} ${{ parameters.additionalParameters }}
displayName: Execute SDL displayName: Execute SDL
continueOnError: ${{ parameters.sdlContinueOnError }} continueOnError: ${{ parameters.sdlContinueOnError }}
- ${{ if ne(parameters.publishGuardianDirectoryToPipeline, 'false') }}:
# We want to publish the Guardian results and configuration for easy diagnosis. However, the
# '.gdn' dir is a mix of configuration, results, extracted dependencies, and Guardian default
# tooling files. Some of these files are large and aren't useful during an investigation, so
# exclude them by simply deleting them before publishing. (As of writing, there is no documented
# way to selectively exclude a dir from the pipeline artifact publish task.)
- task: DeleteFiles@1
displayName: Delete Guardian dependencies to avoid uploading
inputs:
SourceFolder: $(Agent.BuildDirectory)/.gdn
Contents: |
c
i
condition: succeededOrFailed()
- publish: $(Agent.BuildDirectory)/.gdn
artifact: GuardianConfiguration
displayName: Publish GuardianConfiguration
condition: succeededOrFailed()

55
eng/common/tools.ps1

@ -42,7 +42,7 @@
[bool]$useInstalledDotNetCli = if (Test-Path variable:useInstalledDotNetCli) { $useInstalledDotNetCli } else { $true } [bool]$useInstalledDotNetCli = if (Test-Path variable:useInstalledDotNetCli) { $useInstalledDotNetCli } else { $true }
# Enable repos to use a particular version of the on-line dotnet-install scripts. # Enable repos to use a particular version of the on-line dotnet-install scripts.
# default URL: https://dot.net/v1/dotnet-install.ps1 # default URL: https://dotnet.microsoft.com/download/dotnet/scripts/v1/dotnet-install.ps1
[string]$dotnetInstallScriptVersion = if (Test-Path variable:dotnetInstallScriptVersion) { $dotnetInstallScriptVersion } else { 'v1' } [string]$dotnetInstallScriptVersion = if (Test-Path variable:dotnetInstallScriptVersion) { $dotnetInstallScriptVersion } else { 'v1' }
# True to use global NuGet cache instead of restoring packages to repository-local directory. # True to use global NuGet cache instead of restoring packages to repository-local directory.
@ -106,6 +106,46 @@ function Exec-Process([string]$command, [string]$commandArgs) {
} }
} }
# Take the given block, print it, print what the block probably references from the current set of
# variables using low-effort string matching, then run the block.
#
# This is intended to replace the pattern of manually copy-pasting a command, wrapping it in quotes,
# and printing it using "Write-Host". The copy-paste method is more readable in build logs, but less
# maintainable and less reliable. It is easy to make a mistake and modify the command without
# properly updating the "Write-Host" line, resulting in misleading build logs. The probability of
# this mistake makes the pattern hard to trust when it shows up in build logs. Finding the bug in
# existing source code can also be difficult, because the strings are not aligned to each other and
# the line may be 300+ columns long.
#
# By removing the need to maintain two copies of the command, Exec-BlockVerbosely avoids the issues.
#
# In Bash (or any posix-like shell), "set -x" prints usable verbose output automatically.
# "Set-PSDebug" appears to be similar at first glance, but unfortunately, it isn't very useful: it
# doesn't print any info about the variables being used by the command, which is normally the
# interesting part to diagnose.
function Exec-BlockVerbosely([scriptblock] $block) {
Write-Host "--- Running script block:"
$blockString = $block.ToString().Trim()
Write-Host $blockString
Write-Host "--- List of variables that might be used:"
# For each variable x in the environment, check the block for a reference to x via simple "$x" or
# "@x" syntax. This doesn't detect other ways to reference variables ("${x}" nor "$variable:x",
# among others). It only catches what this function was originally written for: simple
# command-line commands.
$variableTable = Get-Variable |
Where-Object {
$blockString.Contains("`$$($_.Name)") -or $blockString.Contains("@$($_.Name)")
} |
Format-Table -AutoSize -HideTableHeaders -Wrap |
Out-String
Write-Host $variableTable.Trim()
Write-Host "--- Executing:"
& $block
Write-Host "--- Done running script block!"
}
# createSdkLocationFile parameter enables a file being generated under the toolset directory # createSdkLocationFile parameter enables a file being generated under the toolset directory
# which writes the sdk's location into. This is only necessary for cmd --> powershell invocations # which writes the sdk's location into. This is only necessary for cmd --> powershell invocations
# as dot sourcing isn't possible. # as dot sourcing isn't possible.
@ -223,7 +263,7 @@ function GetDotNetInstallScript([string] $dotnetRoot) {
if (!(Test-Path $installScript)) { if (!(Test-Path $installScript)) {
Create-Directory $dotnetRoot Create-Directory $dotnetRoot
$ProgressPreference = 'SilentlyContinue' # Don't display the console progress UI - it's a huge perf hit $ProgressPreference = 'SilentlyContinue' # Don't display the console progress UI - it's a huge perf hit
$uri = "https://dot.net/$dotnetInstallScriptVersion/dotnet-install.ps1" $uri = "https://dotnet.microsoft.com/download/dotnet/scripts/$dotnetInstallScriptVersion/dotnet-install.ps1"
Retry({ Retry({
Write-Host "GET $uri" Write-Host "GET $uri"
@ -632,6 +672,17 @@ function ExitWithExitCode([int] $exitCode) {
exit $exitCode exit $exitCode
} }
# Check if $LASTEXITCODE is a nonzero exit code (NZEC). If so, print a Azure Pipeline error for
# diagnostics, then exit the script with the $LASTEXITCODE.
function Exit-IfNZEC([string] $category = "General") {
Write-Host "Exit code $LASTEXITCODE"
if ($LASTEXITCODE -ne 0) {
$message = "Last command failed with exit code $LASTEXITCODE."
Write-PipelineTelemetryError -Force -Category $category -Message $message
ExitWithExitCode $LASTEXITCODE
}
}
function Stop-Processes() { function Stop-Processes() {
Write-Host 'Killing running build processes...' Write-Host 'Killing running build processes...'
foreach ($processName in $processesToStopOnExit) { foreach ($processName in $processesToStopOnExit) {

4
eng/common/tools.sh

@ -54,7 +54,7 @@ warn_as_error=${warn_as_error:-true}
use_installed_dotnet_cli=${use_installed_dotnet_cli:-true} use_installed_dotnet_cli=${use_installed_dotnet_cli:-true}
# Enable repos to use a particular version of the on-line dotnet-install scripts. # Enable repos to use a particular version of the on-line dotnet-install scripts.
# default URL: https://dot.net/v1/dotnet-install.sh # default URL: https://dotnet.microsoft.com/download/dotnet/scripts/v1/dotnet-install.sh
dotnetInstallScriptVersion=${dotnetInstallScriptVersion:-'v1'} dotnetInstallScriptVersion=${dotnetInstallScriptVersion:-'v1'}
# True to use global NuGet cache instead of restoring packages to repository-local directory. # True to use global NuGet cache instead of restoring packages to repository-local directory.
@ -262,7 +262,7 @@ function with_retries {
function GetDotNetInstallScript { function GetDotNetInstallScript {
local root=$1 local root=$1
local install_script="$root/dotnet-install.sh" local install_script="$root/dotnet-install.sh"
local install_script_url="https://dot.net/$dotnetInstallScriptVersion/dotnet-install.sh" local install_script_url="https://dotnet.microsoft.com/download/dotnet/scripts/$dotnetInstallScriptVersion/dotnet-install.sh"
if [[ ! -a "$install_script" ]]; then if [[ ! -a "$install_script" ]]; then
mkdir -p "$root" mkdir -p "$root"

2
global.json

@ -11,6 +11,6 @@
} }
}, },
"msbuild-sdks": { "msbuild-sdks": {
"Microsoft.DotNet.Arcade.Sdk": "6.0.0-beta.21357.3" "Microsoft.DotNet.Arcade.Sdk": "6.0.0-beta.21366.1"
} }
} }

1
src/Microsoft.Tye.Core/ContainerEngine.cs

@ -31,6 +31,7 @@ namespace Microsoft.Tye
public string AspNetUrlsHost => _aspnetUrlsHost; public string AspNetUrlsHost => _aspnetUrlsHost;
public string? ContainerHost => _containerHost; public string? ContainerHost => _containerHost;
public bool IsPodman => _isPodman;
public Task<int> ExecuteAsync( public Task<int> ExecuteAsync(
string args, string args,

6
src/Microsoft.Tye.Core/ExecutableServiceBuilder.cs

@ -6,7 +6,7 @@ using System.Collections.Generic;
namespace Microsoft.Tye namespace Microsoft.Tye
{ {
public sealed class ExecutableServiceBuilder : ServiceBuilder public sealed class ExecutableServiceBuilder : LaunchedServiceBuilder
{ {
public ExecutableServiceBuilder(string name, string executable) public ExecutableServiceBuilder(string name, string executable)
: base(name) : base(name)
@ -20,10 +20,6 @@ namespace Microsoft.Tye
public string? Args { get; set; } public string? Args { get; set; }
public int Replicas { get; set; } = 1;
public List<EnvironmentVariableBuilder> EnvironmentVariables { get; } = new List<EnvironmentVariableBuilder>();
public ProbeBuilder? Liveness { get; set; } public ProbeBuilder? Liveness { get; set; }
public ProbeBuilder? Readiness { get; set; } public ProbeBuilder? Readiness { get; set; }

20
src/Microsoft.Tye.Core/LaunchedServiceBuilder.cs

@ -0,0 +1,20 @@
// Licensed to the .NET Foundation under one or more agreements.
// The .NET Foundation licenses this file to you under the MIT license.
// See the LICENSE file in the project root for more information.
using System.Collections.Generic;
namespace Microsoft.Tye
{
public abstract class LaunchedServiceBuilder : ServiceBuilder
{
public LaunchedServiceBuilder(string name)
: base(name)
{
}
public List<EnvironmentVariableBuilder> EnvironmentVariables { get; } = new List<EnvironmentVariableBuilder>();
public int Replicas { get; set; } = 1;
}
}

6
src/Microsoft.Tye.Core/ProjectServiceBuilder.cs

@ -6,7 +6,7 @@ using System.Collections.Generic;
namespace Microsoft.Tye namespace Microsoft.Tye
{ {
public class ProjectServiceBuilder : ServiceBuilder public class ProjectServiceBuilder : LaunchedServiceBuilder
{ {
public ProjectServiceBuilder(string name) public ProjectServiceBuilder(string name)
: base(name) : base(name)
@ -14,8 +14,6 @@ namespace Microsoft.Tye
} }
public bool IsAspNet { get; set; } public bool IsAspNet { get; set; }
public int Replicas { get; set; } = 1;
public bool Build { get; set; } public bool Build { get; set; }
public string? Args { get; set; } public string? Args { get; set; }
@ -26,8 +24,6 @@ namespace Microsoft.Tye
// Data used for building Kubernetes manifests // Data used for building Kubernetes manifests
public KubernetesManifestInfo? ManifestInfo { get; set; } public KubernetesManifestInfo? ManifestInfo { get; set; }
public List<EnvironmentVariableBuilder> EnvironmentVariables { get; } = new List<EnvironmentVariableBuilder>();
// Used when running in a container locally. // Used when running in a container locally.
public List<VolumeBuilder> Volumes { get; } = new List<VolumeBuilder>(); public List<VolumeBuilder> Volumes { get; } = new List<VolumeBuilder>();

25
src/Microsoft.Tye.Extensions/Dapr/DaprExtension.cs

@ -77,8 +77,11 @@ namespace Microsoft.Tye.Extensions.Dapr
} }
// For local run, enumerate all projects, and add services for each dapr proxy. // For local run, enumerate all projects, and add services for each dapr proxy.
var projects = context.Application.Services.OfType<ProjectServiceBuilder>().ToList(); var projects = context.Application.Services.OfType<ProjectServiceBuilder>().Cast<LaunchedServiceBuilder>();
foreach (var project in projects) var executables = context.Application.Services.OfType<ExecutableServiceBuilder>().Cast<LaunchedServiceBuilder>();
var services = projects.Concat(executables).ToList();
foreach (var project in services)
{ {
// Dapr requires http. If this project isn't listening to HTTP then it's not daprized. // Dapr requires http. If this project isn't listening to HTTP then it's not daprized.
var httpBinding = project.Bindings.FirstOrDefault(b => b.Protocol == "http"); var httpBinding = project.Bindings.FirstOrDefault(b => b.Protocol == "http");
@ -107,7 +110,7 @@ namespace Microsoft.Tye.Extensions.Dapr
// These environment variables are replaced with environment variables // These environment variables are replaced with environment variables
// defined for this service. // defined for this service.
Args = $"-app-id {project.Name} -app-port %APP_PORT% -dapr-grpc-port %DAPR_GRPC_PORT% --dapr-http-port %DAPR_HTTP_PORT% --metrics-port %METRICS_PORT% --placement-host-address localhost:{daprPlacementPort}", Args = $"run --app-id {project.Name} --app-port %APP_PORT% --dapr-grpc-port %DAPR_GRPC_PORT% --dapr-http-port %DAPR_HTTP_PORT% --metrics-port %METRICS_PORT% --placement-host-address localhost:{daprPlacementPort}",
}; };
// When running locally `-config` specifies a filename, not a configuration name. By convention // When running locally `-config` specifies a filename, not a configuration name. By convention
@ -117,7 +120,7 @@ namespace Microsoft.Tye.Extensions.Dapr
var configFile = Path.Combine(context.Application.Source.DirectoryName!, "components", $"{daprConfig}.yaml"); var configFile = Path.Combine(context.Application.Source.DirectoryName!, "components", $"{daprConfig}.yaml");
if (File.Exists(configFile)) if (File.Exists(configFile))
{ {
proxy.Args += $" -config \"{configFile}\""; proxy.Args += $" --config \"{configFile}\"";
} }
else else
{ {
@ -127,12 +130,12 @@ namespace Microsoft.Tye.Extensions.Dapr
if (config.Data.TryGetValue("log-level", out obj) && obj?.ToString() is string logLevel) if (config.Data.TryGetValue("log-level", out obj) && obj?.ToString() is string logLevel)
{ {
proxy.Args += $" -log-level {logLevel}"; proxy.Args += $" --log-level {logLevel}";
} }
if (config.Data.TryGetValue("components-path", out obj) && obj?.ToString() is string componentsPath) if (config.Data.TryGetValue("components-path", out obj) && obj?.ToString() is string componentsPath)
{ {
proxy.Args += $" -components-path {componentsPath}"; proxy.Args += $" --components-path {componentsPath}";
} }
// Add dapr proxy as a service available to everyone. // Add dapr proxy as a service available to everyone.
proxy.Dependencies.UnionWith(context.Application.Services.Select(s => s.Name)); proxy.Dependencies.UnionWith(context.Application.Services.Select(s => s.Name));
@ -268,10 +271,10 @@ namespace Microsoft.Tye.Extensions.Dapr
private string GetDaprExecutablePath() private string GetDaprExecutablePath()
{ {
// Starting with dapr version 11, dapr is installed in user profile/home.
if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows))
{ {
var windowsPath = Environment.ExpandEnvironmentVariables("%USERPROFILE%/.dapr/bin/daprd.exe"); // The Dapr Windows installation script defaults to "C:\dapr".
var windowsPath = Environment.ExpandEnvironmentVariables("C:/dapr/dapr.exe");
if (File.Exists(windowsPath)) if (File.Exists(windowsPath))
{ {
return windowsPath; return windowsPath;
@ -279,15 +282,15 @@ namespace Microsoft.Tye.Extensions.Dapr
} }
else else
{ {
var nixpath = Environment.ExpandEnvironmentVariables("%HOME%/.dapr/bin/daprd"); var nixpath = Environment.ExpandEnvironmentVariables("/usr/local/bin/dapr");
if (File.Exists(nixpath)) if (File.Exists(nixpath))
{ {
return nixpath; return nixpath;
} }
} }
// Older version of dapr don't have dapr in the bin directory, but it is usually on the path. // Dapr is usually on the path.
return "daprd"; return "dapr";
} }
} }
} }

9
src/Microsoft.Tye.Hosting/DockerRunner.cs

@ -285,6 +285,15 @@ namespace Microsoft.Tye.Hosting
if (volumeMapping.Source != null) if (volumeMapping.Source != null)
{ {
var sourcePath = Path.GetFullPath(Path.Combine(application.ContextDirectory, volumeMapping.Source)); var sourcePath = Path.GetFullPath(Path.Combine(application.ContextDirectory, volumeMapping.Source));
if (application.ContainerEngine.IsPodman)
{
// unlike docker, podman doesn't create the host directory when it doesn't exist.
// https://github.com/containers/podman/issues/10471
if (!File.Exists(sourcePath) && !Directory.Exists(sourcePath))
{
Directory.CreateDirectory(sourcePath);
}
}
volumes += $"-v \"{sourcePath}:{volumeMapping.Target}:{(volumeMapping.ReadOnly ? "ro," : "")}z\" "; volumes += $"-v \"{sourcePath}:{volumeMapping.Target}:{(volumeMapping.ReadOnly ? "ro," : "")}z\" ";
} }
else if (volumeMapping.Name != null) else if (volumeMapping.Name != null)

Loading…
Cancel
Save