Browse Source

Merge branch 'main' into feature/dashboard-enhancements

pull/1566/head
razvan.goga@gmail.com 4 years ago
parent
commit
a707932f15
  1. 20
      eng/Version.Details.xml
  2. 161
      eng/common/cross/build-rootfs.sh
  3. 9
      eng/common/generate-locproject.ps1
  4. 9
      eng/common/init-tools-native.ps1
  5. 8
      eng/common/templates/job/job.yml
  6. 2
      eng/common/templates/job/source-index-stage1.yml
  7. 14
      eng/common/tools.ps1
  8. 4
      global.json

20
eng/Version.Details.xml

@ -3,25 +3,25 @@
<ProductDependencies>
</ProductDependencies>
<ToolsetDependencies>
<Dependency Name="Microsoft.DotNet.Arcade.Sdk" Version="8.0.0-beta.23106.4">
<Dependency Name="Microsoft.DotNet.Arcade.Sdk" Version="8.0.0-beta.23168.1">
<Uri>https://github.com/dotnet/arcade</Uri>
<Sha>02980a654d20995c38eca6ceabf729c4c13412c4</Sha>
<Sha>b12f035e893c34ec2c965d75f6e21b7a2667e98d</Sha>
</Dependency>
<Dependency Name="Microsoft.DotNet.Build.Tasks.Feed" Version="8.0.0-beta.23106.4">
<Dependency Name="Microsoft.DotNet.Build.Tasks.Feed" Version="8.0.0-beta.23168.1">
<Uri>https://github.com/dotnet/arcade</Uri>
<Sha>02980a654d20995c38eca6ceabf729c4c13412c4</Sha>
<Sha>b12f035e893c34ec2c965d75f6e21b7a2667e98d</Sha>
</Dependency>
<Dependency Name="Microsoft.DotNet.SignTool" Version="8.0.0-beta.23106.4">
<Dependency Name="Microsoft.DotNet.SignTool" Version="8.0.0-beta.23168.1">
<Uri>https://github.com/dotnet/arcade</Uri>
<Sha>02980a654d20995c38eca6ceabf729c4c13412c4</Sha>
<Sha>b12f035e893c34ec2c965d75f6e21b7a2667e98d</Sha>
</Dependency>
<Dependency Name="Microsoft.DotNet.Helix.Sdk" Version="8.0.0-beta.23106.4">
<Dependency Name="Microsoft.DotNet.Helix.Sdk" Version="8.0.0-beta.23168.1">
<Uri>https://github.com/dotnet/arcade</Uri>
<Sha>02980a654d20995c38eca6ceabf729c4c13412c4</Sha>
<Sha>b12f035e893c34ec2c965d75f6e21b7a2667e98d</Sha>
</Dependency>
<Dependency Name="Microsoft.DotNet.SwaggerGenerator.MSBuild" Version="8.0.0-beta.23106.4">
<Dependency Name="Microsoft.DotNet.SwaggerGenerator.MSBuild" Version="8.0.0-beta.23168.1">
<Uri>https://github.com/dotnet/arcade</Uri>
<Sha>02980a654d20995c38eca6ceabf729c4c13412c4</Sha>
<Sha>b12f035e893c34ec2c965d75f6e21b7a2667e98d</Sha>
</Dependency>
<Dependency Name="Microsoft.DotNet.Maestro.Client" Version="1.1.0-beta.19556.4">
<Uri>https://github.com/dotnet/arcade-services</Uri>

161
eng/common/cross/build-rootfs.sh

@ -6,10 +6,11 @@ usage()
{
echo "Usage: $0 [BuildArch] [CodeName] [lldbx.y] [llvmx[.y]] [--skipunmount] --rootfsdir <directory>]"
echo "BuildArch can be: arm(default), arm64, armel, armv6, ppc64le, riscv64, s390x, x64, x86"
echo "CodeName - optional, Code name for Linux, can be: xenial(default), zesty, bionic, alpine, alpine3.13 or alpine3.14. If BuildArch is armel, LinuxCodeName is jessie(default) or tizen."
echo " for FreeBSD can be: freebsd12, freebsd13"
echo " for illumos can be: illumos"
echo " for Haiku can be: haiku."
echo "CodeName - optional, Code name for Linux, can be: xenial(default), zesty, bionic, alpine"
echo " for alpine can be specified with version: alpineX.YY or alpineedge"
echo " for FreeBSD can be: freebsd12, freebsd13"
echo " for illumos can be: illumos"
echo " for Haiku can be: haiku."
echo "lldbx.y - optional, LLDB version, can be: lldb3.9(default), lldb4.0, lldb5.0, lldb6.0 no-lldb. Ignored for alpine and FreeBSD"
echo "llvmx[.y] - optional, LLVM version for LLVM related packages."
echo "--skipunmount - optional, will skip the unmount of rootfs folder."
@ -54,7 +55,7 @@ __AlpinePackages+=" gettext-dev"
__AlpinePackages+=" icu-dev"
__AlpinePackages+=" libunwind-dev"
__AlpinePackages+=" lttng-ust-dev"
__AlpinePackages+=" compiler-rt-static"
__AlpinePackages+=" compiler-rt"
__AlpinePackages+=" numactl-dev"
# runtime libraries' dependencies
@ -145,20 +146,13 @@ while :; do
__Keyring="--keyring /usr/share/keyrings/raspbian-archive-keyring.gpg"
fi
;;
ppc64le)
__BuildArch=ppc64le
__UbuntuArch=ppc64el
__UbuntuRepo="http://ports.ubuntu.com/ubuntu-ports/"
__UbuntuPackages="${__UbuntuPackages// libunwind8-dev/}"
__UbuntuPackages="${__UbuntuPackages// libomp-dev/}"
__UbuntuPackages="${__UbuntuPackages// libomp5/}"
unset __LLDB_Package
;;
riscv64)
__BuildArch=riscv64
__AlpineArch=riscv64
__AlpinePackages="${__AlpinePackages// lldb-dev/}"
__QEMUArch=riscv64
__UbuntuArch=riscv64
__UbuntuRepo="http://deb.debian.org/debian-ports"
__CodeName=sid
__UbuntuPackages="${__UbuntuPackages// libunwind8-dev/}"
unset __LLDB_Package
@ -166,8 +160,21 @@ while :; do
__Keyring="--keyring /usr/share/keyrings/debian-ports-archive-keyring.gpg --include=debian-ports-archive-keyring"
fi
;;
ppc64le)
__BuildArch=ppc64le
__AlpineArch=ppc64le
__QEMUArch=ppc64le
__UbuntuArch=ppc64el
__UbuntuRepo="http://ports.ubuntu.com/ubuntu-ports/"
__UbuntuPackages="${__UbuntuPackages// libunwind8-dev/}"
__UbuntuPackages="${__UbuntuPackages// libomp-dev/}"
__UbuntuPackages="${__UbuntuPackages// libomp5/}"
unset __LLDB_Package
;;
s390x)
__BuildArch=s390x
__AlpineArch=s390x
__QEMUArch=s390x
__UbuntuArch=s390x
__UbuntuRepo="http://ports.ubuntu.com/ubuntu-ports/"
__UbuntuPackages="${__UbuntuPackages// libunwind8-dev/}"
@ -186,6 +193,7 @@ while :; do
x86)
__BuildArch=x86
__UbuntuArch=i386
__AlpineArch=x86
__UbuntuRepo="http://archive.ubuntu.com/ubuntu/"
;;
lldb*)
@ -240,34 +248,59 @@ while :; do
;;
jessie) # Debian 8
__CodeName=jessie
__UbuntuRepo="http://ftp.debian.org/debian/"
if [[ -z "$__UbuntuRepo" ]]; then
__UbuntuRepo="http://ftp.debian.org/debian/"
fi
;;
stretch) # Debian 9
__CodeName=stretch
__UbuntuRepo="http://ftp.debian.org/debian/"
__LLDB_Package="liblldb-6.0-dev"
if [[ -z "$__UbuntuRepo" ]]; then
__UbuntuRepo="http://ftp.debian.org/debian/"
fi
;;
buster) # Debian 10
__CodeName=buster
__UbuntuRepo="http://ftp.debian.org/debian/"
__LLDB_Package="liblldb-6.0-dev"
if [[ -z "$__UbuntuRepo" ]]; then
__UbuntuRepo="http://ftp.debian.org/debian/"
fi
;;
bullseye) # Debian 11
__CodeName=bullseye
if [[ -z "$__UbuntuRepo" ]]; then
__UbuntuRepo="http://ftp.debian.org/debian/"
fi
;;
sid) # Debian sid
__CodeName=sid
if [[ -z "$__UbuntuRepo" ]]; then
__UbuntuRepo="http://ftp.debian.org/debian/"
fi
;;
tizen)
__CodeName=
__UbuntuRepo=
__Tizen=tizen
;;
alpine|alpine3.13)
alpine*)
__CodeName=alpine
__UbuntuRepo=
__AlpineVersion=3.13
__AlpinePackages+=" llvm10-libs"
;;
alpine3.14)
__CodeName=alpine
__UbuntuRepo=
__AlpineVersion=3.14
__AlpinePackages+=" llvm11-libs"
version="${lowerI/alpine/}"
if [[ "$version" == "edge" ]]; then
__AlpineVersion=edge
else
parts=(${version//./ })
__AlpineMajorVersion="${parts[0]}"
__AlpineMinoVersion="${parts[1]}"
__AlpineVersion="$__AlpineMajorVersion.$__AlpineMinoVersion"
fi
;;
freebsd12)
__CodeName=freebsd
@ -310,10 +343,41 @@ while :; do
shift
done
case "$__AlpineVersion" in
3.14) __AlpinePackages+=" llvm11-libs" ;;
3.15) __AlpinePackages+=" llvm12-libs" ;;
3.16) __AlpinePackages+=" llvm13-libs" ;;
3.17) __AlpinePackages+=" llvm15-libs" ;;
edge) __AlpineLlvmLibsLookup=1 ;;
*)
if [[ "$__AlpineArch" =~ s390x|ppc64le ]]; then
echo boo
__AlpineVersion=3.15 # minimum version that supports lldb-dev
__AlpinePackages+=" llvm12-libs"
elif [[ "$__AlpineArch" == "x86" ]]; then
__AlpineVersion=3.17 # minimum version that supports lldb-dev
__AlpinePackages+=" llvm15-libs"
elif [[ "$__AlpineArch" == "riscv64" ]]; then
__AlpineLlvmLibsLookup=1
__AlpineVersion=edge # minimum version with APKINDEX.tar.gz (packages archive)
else
__AlpineVersion=3.13 # 3.13 to maximize compatibility
__AlpinePackages+=" llvm10-libs"
if [[ "$__AlpineArch" == "armv7" ]]; then
__AlpinePackages="${__AlpinePackages//numactl-dev/}"
fi
fi
esac
if [[ "$__AlpineVersion" =~ 3\.1[345] ]]; then
# compiler-rt--static was merged in compiler-rt package in alpine 3.16
# for older versions, we need compiler-rt--static, so replace the name
__AlpinePackages="${__AlpinePackages/compiler-rt/compiler-rt-static}"
fi
if [[ "$__BuildArch" == "armel" ]]; then
__LLDB_Package="lldb-3.5-dev"
elif [[ "$__BuildArch" == "arm" && "$__AlpineVersion" == "3.13" ]]; then
__AlpinePackages="${__AlpinePackages//numactl-dev/}"
fi
__UbuntuPackages+=" ${__LLDB_Package:-}"
@ -341,18 +405,41 @@ mkdir -p "$__RootfsDir"
__RootfsDir="$( cd "$__RootfsDir" && pwd )"
if [[ "$__CodeName" == "alpine" ]]; then
__ApkToolsVersion=2.9.1
__ApkToolsVersion=2.12.11
__ApkToolsDir="$(mktemp -d)"
wget "https://github.com/alpinelinux/apk-tools/releases/download/v$__ApkToolsVersion/apk-tools-$__ApkToolsVersion-x86_64-linux.tar.gz" -P "$__ApkToolsDir"
tar -xf "$__ApkToolsDir/apk-tools-$__ApkToolsVersion-x86_64-linux.tar.gz" -C "$__ApkToolsDir"
wget "https://gitlab.alpinelinux.org/api/v4/projects/5/packages/generic//v$__ApkToolsVersion/x86_64/apk.static" -P "$__ApkToolsDir"
chmod +x "$__ApkToolsDir/apk.static"
mkdir -p "$__RootfsDir"/usr/bin
cp -v "/usr/bin/qemu-$__QEMUArch-static" "$__RootfsDir/usr/bin"
"$__ApkToolsDir/apk-tools-$__ApkToolsVersion/apk" \
-X "http://dl-cdn.alpinelinux.org/alpine/v$__AlpineVersion/main" \
-X "http://dl-cdn.alpinelinux.org/alpine/v$__AlpineVersion/community" \
-U --allow-untrusted --root "$__RootfsDir" --arch "$__AlpineArch" --initdb \
add $__AlpinePackages
if [[ "$__AlpineVersion" == "edge" ]]; then
version=edge
else
version="v$__AlpineVersion"
fi
# initialize DB
"$__ApkToolsDir/apk.static" \
-X "http://dl-cdn.alpinelinux.org/alpine/$version/main" \
-X "http://dl-cdn.alpinelinux.org/alpine/$version/community" \
-U --allow-untrusted --root "$__RootfsDir" --arch "$__AlpineArch" --initdb add
if [[ "$__AlpineLlvmLibsLookup" == 1 ]]; then
__AlpinePackages+=" $("$__ApkToolsDir/apk.static" \
-X "http://dl-cdn.alpinelinux.org/alpine/$version/main" \
-X "http://dl-cdn.alpinelinux.org/alpine/$version/community" \
-U --allow-untrusted --root "$__RootfsDir" --arch "$__AlpineArch" \
search 'llvm*-libs' | sort | tail -1 | sed 's/-[^-]*//2g')"
fi
# install all packages in one go
"$__ApkToolsDir/apk.static" \
-X "http://dl-cdn.alpinelinux.org/alpine/$version/main" \
-X "http://dl-cdn.alpinelinux.org/alpine/$version/community" \
-U --allow-untrusted --root "$__RootfsDir" --arch "$__AlpineArch" \
add $__AlpinePackages
rm -r "$__ApkToolsDir"
elif [[ "$__CodeName" == "freebsd" ]]; then

9
eng/common/generate-locproject.ps1

@ -45,7 +45,7 @@ if (-not $wxlFiles) {
}
}
$macosHtmlEnFiles = Get-ChildItem -Recurse -Path "$SourcesDirectory" | Where-Object { $_.FullName -Match "en\.lproj\\.+\.html" } # add installer HTML files
$macosHtmlEnFiles = Get-ChildItem -Recurse -Path "$SourcesDirectory" | Where-Object { $_.FullName -Match "en\.lproj\\.+\.html$" } # add installer HTML files
$macosHtmlFiles = @()
if ($macosHtmlEnFiles) {
$macosHtmlEnFiles | ForEach-Object {
@ -148,12 +148,17 @@ $locJson = @{
}
}
$sourceFile = ($_.FullName | Resolve-Path -Relative)
$lciFile = $sourceFile + ".lci"
if ($continue) {
return @{
$result = @{
SourceFile = $sourceFile
CopyOption = "LangIDOnPath"
OutputPath = $outputPath
}
if (Test-Path $lciFile -PathType Leaf) {
$result["LciFile"] = $lciFile
}
return $result
}
}
)

9
eng/common/init-tools-native.ps1

@ -83,7 +83,8 @@ try {
Select-Object -Expand 'native-tools' -ErrorAction SilentlyContinue
if ($NativeTools) {
if ($PathPromotion -eq $True) {
if ($env:SYSTEM_TEAMPROJECT) { # check to see if we're in an Azure pipelines build
$ArcadeToolsDirectory = "$env:SYSTEMDRIVE\arcade-tools"
if (Test-Path $ArcadeToolsDirectory) { # if this directory exists, we should use native tools on machine
$NativeTools.PSObject.Properties | ForEach-Object {
$ToolName = $_.Name
$ToolVersion = $_.Value
@ -93,11 +94,6 @@ try {
if ($ToolVersion -eq "latest") {
$ToolVersion = ""
}
$ArcadeToolsDirectory = "C:\arcade-tools"
if (-not (Test-Path $ArcadeToolsDirectory)) {
Write-Error "Arcade tools directory '$ArcadeToolsDirectory' was not found; artifacts were not properly installed."
exit 1
}
$ToolDirectories = (Get-ChildItem -Path "$ArcadeToolsDirectory" -Filter "$ToolName-$ToolVersion*" | Sort-Object -Descending)
if ($ToolDirectories -eq $null) {
Write-Error "Unable to find directory for $ToolName $ToolVersion; please make sure the tool is installed on this image."
@ -125,6 +121,7 @@ try {
if ((Get-Command "$ToolName" -ErrorAction SilentlyContinue) -eq $null) {
Write-PipelineTelemetryError -Category 'NativeToolsBootstrap' -Message "$ToolName not found on path. Please install $ToolName $ToolVersion before proceeding."
Write-PipelineTelemetryError -Category 'NativeToolsBootstrap' -Message "If this is running on a build machine, the arcade-tools directory was not found, which means there's an error with the image."
}
}
exit 0

8
eng/common/templates/job/job.yml

@ -101,7 +101,7 @@ jobs:
# handle key-value variable syntax.
# example:
# - [key]: [value]
- ${{ if and(eq(variable.name, ''), eq(variable.group, '')) }}:
- ${{ if and(eq(variable.name, ''), eq(variable.group, ''), eq(variable.template, '')) }}:
- ${{ each pair in variable }}:
- name: ${{ pair.key }}
value: ${{ pair.value }}
@ -134,7 +134,7 @@ jobs:
- ${{ if and(eq(parameters.runAsPublic, 'false'), eq(variables['System.TeamProject'], 'internal')) }}:
- task: NuGetAuthenticate@0
- ${{ if or(eq(parameters.artifacts.download, 'true'), ne(parameters.artifacts.download, '')) }}:
- ${{ if and(ne(parameters.artifacts.download, 'false'), ne(parameters.artifacts.download, '')) }}:
- task: DownloadPipelineArtifact@2
inputs:
buildType: current
@ -171,7 +171,7 @@ jobs:
TeamName: $(_TeamName)
- ${{ if ne(parameters.artifacts.publish, '') }}:
- ${{ if or(eq(parameters.artifacts.publish.artifacts, 'true'), ne(parameters.artifacts.publish.artifacts, '')) }}:
- ${{ if and(ne(parameters.artifacts.publish.artifacts, 'false'), ne(parameters.artifacts.publish.artifacts, '')) }}:
- task: CopyFiles@2
displayName: Gather binaries for publish to artifacts
inputs:
@ -192,7 +192,7 @@ jobs:
ArtifactName: ${{ coalesce(parameters.artifacts.publish.artifacts.name , 'Artifacts_$(Agent.Os)_$(_BuildConfig)') }}
continueOnError: true
condition: always()
- ${{ if or(eq(parameters.artifacts.publish.logs, 'true'), ne(parameters.artifacts.publish.logs, '')) }}:
- ${{ if and(ne(parameters.artifacts.publish.logs, 'false'), ne(parameters.artifacts.publish.logs, '')) }}:
- publish: artifacts/log
artifact: ${{ coalesce(parameters.artifacts.publish.logs.name, 'Logs_Build_$(Agent.Os)_$(_BuildConfig)') }}
displayName: Publish logs

2
eng/common/templates/job/source-index-stage1.yml

@ -1,6 +1,6 @@
parameters:
runAsPublic: false
sourceIndexPackageVersion: 1.0.1-20221220.2
sourceIndexPackageVersion: 1.0.1-20230228.2
sourceIndexPackageSource: https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools/nuget/v3/index.json
sourceIndexBuildCommand: powershell -NoLogo -NoProfile -ExecutionPolicy Bypass -Command "eng/common/build.ps1 -restore -build -binarylog -ci"
preSteps: []

14
eng/common/tools.ps1

@ -913,11 +913,13 @@ if (!$disableConfigureToolsetImport) {
function Enable-Nuget-EnhancedRetry() {
if ($ci) {
Write-Host "Setting NUGET enhanced retry environment variables"
$env:NUGET_ENABLE_EXPERIMENTAL_HTTP_RETRY = 'true'
$env:NUGET_EXPERIMENTAL_MAX_NETWORK_TRY_COUNT = 6
$env:NUGET_EXPERIMENTAL_NETWORK_RETRY_DELAY_MILLISECONDS = 1000
Write-PipelineSetVariable -Name 'NUGET_ENABLE_EXPERIMENTAL_HTTP_RETRY' -Value 'true'
Write-PipelineSetVariable -Name 'NUGET_EXPERIMENTAL_MAX_NETWORK_TRY_COUNT' -Value '6'
Write-PipelineSetVariable -Name 'NUGET_EXPERIMENTAL_NETWORK_RETRY_DELAY_MILLISECONDS' -Value '1000'
$env:NUGET_ENABLE_ENHANCED_HTTP_RETRY = 'true'
$env:NUGET_ENHANCED_MAX_NETWORK_TRY_COUNT = 6
$env:NUGET_ENHANCED_NETWORK_RETRY_DELAY_MILLISECONDS = 1000
$env:NUGET_RETRY_HTTP_429 = 'true'
Write-PipelineSetVariable -Name 'NUGET_ENABLE_ENHANCED_HTTP_RETRY' -Value 'true'
Write-PipelineSetVariable -Name 'NUGET_ENHANCED_MAX_NETWORK_TRY_COUNT' -Value '6'
Write-PipelineSetVariable -Name 'NUGET_ENHANCED_NETWORK_RETRY_DELAY_MILLISECONDS' -Value '1000'
Write-PipelineSetVariable -Name 'NUGET_RETRY_HTTP_429' -Value 'true'
}
}

4
global.json

@ -3,7 +3,7 @@
"allowPrerelease": true
},
"tools": {
"dotnet": "8.0.100-alpha.1.23061.8",
"dotnet": "8.0.100-preview.1.23115.2",
"runtimes": {
"dotnet": [
"6.0.0"
@ -14,6 +14,6 @@
}
},
"msbuild-sdks": {
"Microsoft.DotNet.Arcade.Sdk": "8.0.0-beta.23106.4"
"Microsoft.DotNet.Arcade.Sdk": "8.0.0-beta.23168.1"
}
}

Loading…
Cancel
Save