diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index 40665ec2..022724dd 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -3,25 +3,25 @@ - + https://github.com/dotnet/arcade - 02980a654d20995c38eca6ceabf729c4c13412c4 + b12f035e893c34ec2c965d75f6e21b7a2667e98d - + https://github.com/dotnet/arcade - 02980a654d20995c38eca6ceabf729c4c13412c4 + b12f035e893c34ec2c965d75f6e21b7a2667e98d - + https://github.com/dotnet/arcade - 02980a654d20995c38eca6ceabf729c4c13412c4 + b12f035e893c34ec2c965d75f6e21b7a2667e98d - + https://github.com/dotnet/arcade - 02980a654d20995c38eca6ceabf729c4c13412c4 + b12f035e893c34ec2c965d75f6e21b7a2667e98d - + https://github.com/dotnet/arcade - 02980a654d20995c38eca6ceabf729c4c13412c4 + b12f035e893c34ec2c965d75f6e21b7a2667e98d https://github.com/dotnet/arcade-services diff --git a/eng/common/cross/build-rootfs.sh b/eng/common/cross/build-rootfs.sh index 1ebf454f..ff113f73 100755 --- a/eng/common/cross/build-rootfs.sh +++ b/eng/common/cross/build-rootfs.sh @@ -6,10 +6,11 @@ usage() { echo "Usage: $0 [BuildArch] [CodeName] [lldbx.y] [llvmx[.y]] [--skipunmount] --rootfsdir ]" echo "BuildArch can be: arm(default), arm64, armel, armv6, ppc64le, riscv64, s390x, x64, x86" - echo "CodeName - optional, Code name for Linux, can be: xenial(default), zesty, bionic, alpine, alpine3.13 or alpine3.14. If BuildArch is armel, LinuxCodeName is jessie(default) or tizen." - echo " for FreeBSD can be: freebsd12, freebsd13" - echo " for illumos can be: illumos" - echo " for Haiku can be: haiku." + echo "CodeName - optional, Code name for Linux, can be: xenial(default), zesty, bionic, alpine" + echo " for alpine can be specified with version: alpineX.YY or alpineedge" + echo " for FreeBSD can be: freebsd12, freebsd13" + echo " for illumos can be: illumos" + echo " for Haiku can be: haiku." echo "lldbx.y - optional, LLDB version, can be: lldb3.9(default), lldb4.0, lldb5.0, lldb6.0 no-lldb. Ignored for alpine and FreeBSD" echo "llvmx[.y] - optional, LLVM version for LLVM related packages." echo "--skipunmount - optional, will skip the unmount of rootfs folder." @@ -54,7 +55,7 @@ __AlpinePackages+=" gettext-dev" __AlpinePackages+=" icu-dev" __AlpinePackages+=" libunwind-dev" __AlpinePackages+=" lttng-ust-dev" -__AlpinePackages+=" compiler-rt-static" +__AlpinePackages+=" compiler-rt" __AlpinePackages+=" numactl-dev" # runtime libraries' dependencies @@ -145,20 +146,13 @@ while :; do __Keyring="--keyring /usr/share/keyrings/raspbian-archive-keyring.gpg" fi ;; - ppc64le) - __BuildArch=ppc64le - __UbuntuArch=ppc64el - __UbuntuRepo="http://ports.ubuntu.com/ubuntu-ports/" - __UbuntuPackages="${__UbuntuPackages// libunwind8-dev/}" - __UbuntuPackages="${__UbuntuPackages// libomp-dev/}" - __UbuntuPackages="${__UbuntuPackages// libomp5/}" - unset __LLDB_Package - ;; riscv64) __BuildArch=riscv64 + __AlpineArch=riscv64 + __AlpinePackages="${__AlpinePackages// lldb-dev/}" + __QEMUArch=riscv64 __UbuntuArch=riscv64 __UbuntuRepo="http://deb.debian.org/debian-ports" - __CodeName=sid __UbuntuPackages="${__UbuntuPackages// libunwind8-dev/}" unset __LLDB_Package @@ -166,8 +160,21 @@ while :; do __Keyring="--keyring /usr/share/keyrings/debian-ports-archive-keyring.gpg --include=debian-ports-archive-keyring" fi ;; + ppc64le) + __BuildArch=ppc64le + __AlpineArch=ppc64le + __QEMUArch=ppc64le + __UbuntuArch=ppc64el + __UbuntuRepo="http://ports.ubuntu.com/ubuntu-ports/" + __UbuntuPackages="${__UbuntuPackages// libunwind8-dev/}" + __UbuntuPackages="${__UbuntuPackages// libomp-dev/}" + __UbuntuPackages="${__UbuntuPackages// libomp5/}" + unset __LLDB_Package + ;; s390x) __BuildArch=s390x + __AlpineArch=s390x + __QEMUArch=s390x __UbuntuArch=s390x __UbuntuRepo="http://ports.ubuntu.com/ubuntu-ports/" __UbuntuPackages="${__UbuntuPackages// libunwind8-dev/}" @@ -186,6 +193,7 @@ while :; do x86) __BuildArch=x86 __UbuntuArch=i386 + __AlpineArch=x86 __UbuntuRepo="http://archive.ubuntu.com/ubuntu/" ;; lldb*) @@ -240,34 +248,59 @@ while :; do ;; jessie) # Debian 8 __CodeName=jessie - __UbuntuRepo="http://ftp.debian.org/debian/" + + if [[ -z "$__UbuntuRepo" ]]; then + __UbuntuRepo="http://ftp.debian.org/debian/" + fi ;; stretch) # Debian 9 __CodeName=stretch - __UbuntuRepo="http://ftp.debian.org/debian/" __LLDB_Package="liblldb-6.0-dev" + + if [[ -z "$__UbuntuRepo" ]]; then + __UbuntuRepo="http://ftp.debian.org/debian/" + fi ;; buster) # Debian 10 __CodeName=buster - __UbuntuRepo="http://ftp.debian.org/debian/" __LLDB_Package="liblldb-6.0-dev" + + if [[ -z "$__UbuntuRepo" ]]; then + __UbuntuRepo="http://ftp.debian.org/debian/" + fi + ;; + bullseye) # Debian 11 + __CodeName=bullseye + + if [[ -z "$__UbuntuRepo" ]]; then + __UbuntuRepo="http://ftp.debian.org/debian/" + fi + ;; + sid) # Debian sid + __CodeName=sid + + if [[ -z "$__UbuntuRepo" ]]; then + __UbuntuRepo="http://ftp.debian.org/debian/" + fi ;; tizen) __CodeName= __UbuntuRepo= __Tizen=tizen ;; - alpine|alpine3.13) + alpine*) __CodeName=alpine __UbuntuRepo= - __AlpineVersion=3.13 - __AlpinePackages+=" llvm10-libs" - ;; - alpine3.14) - __CodeName=alpine - __UbuntuRepo= - __AlpineVersion=3.14 - __AlpinePackages+=" llvm11-libs" + version="${lowerI/alpine/}" + + if [[ "$version" == "edge" ]]; then + __AlpineVersion=edge + else + parts=(${version//./ }) + __AlpineMajorVersion="${parts[0]}" + __AlpineMinoVersion="${parts[1]}" + __AlpineVersion="$__AlpineMajorVersion.$__AlpineMinoVersion" + fi ;; freebsd12) __CodeName=freebsd @@ -310,10 +343,41 @@ while :; do shift done +case "$__AlpineVersion" in + 3.14) __AlpinePackages+=" llvm11-libs" ;; + 3.15) __AlpinePackages+=" llvm12-libs" ;; + 3.16) __AlpinePackages+=" llvm13-libs" ;; + 3.17) __AlpinePackages+=" llvm15-libs" ;; + edge) __AlpineLlvmLibsLookup=1 ;; + *) + if [[ "$__AlpineArch" =~ s390x|ppc64le ]]; then + echo boo + __AlpineVersion=3.15 # minimum version that supports lldb-dev + __AlpinePackages+=" llvm12-libs" + elif [[ "$__AlpineArch" == "x86" ]]; then + __AlpineVersion=3.17 # minimum version that supports lldb-dev + __AlpinePackages+=" llvm15-libs" + elif [[ "$__AlpineArch" == "riscv64" ]]; then + __AlpineLlvmLibsLookup=1 + __AlpineVersion=edge # minimum version with APKINDEX.tar.gz (packages archive) + else + __AlpineVersion=3.13 # 3.13 to maximize compatibility + __AlpinePackages+=" llvm10-libs" + + if [[ "$__AlpineArch" == "armv7" ]]; then + __AlpinePackages="${__AlpinePackages//numactl-dev/}" + fi + fi +esac + +if [[ "$__AlpineVersion" =~ 3\.1[345] ]]; then + # compiler-rt--static was merged in compiler-rt package in alpine 3.16 + # for older versions, we need compiler-rt--static, so replace the name + __AlpinePackages="${__AlpinePackages/compiler-rt/compiler-rt-static}" +fi + if [[ "$__BuildArch" == "armel" ]]; then __LLDB_Package="lldb-3.5-dev" -elif [[ "$__BuildArch" == "arm" && "$__AlpineVersion" == "3.13" ]]; then - __AlpinePackages="${__AlpinePackages//numactl-dev/}" fi __UbuntuPackages+=" ${__LLDB_Package:-}" @@ -341,18 +405,41 @@ mkdir -p "$__RootfsDir" __RootfsDir="$( cd "$__RootfsDir" && pwd )" if [[ "$__CodeName" == "alpine" ]]; then - __ApkToolsVersion=2.9.1 + __ApkToolsVersion=2.12.11 __ApkToolsDir="$(mktemp -d)" - wget "https://github.com/alpinelinux/apk-tools/releases/download/v$__ApkToolsVersion/apk-tools-$__ApkToolsVersion-x86_64-linux.tar.gz" -P "$__ApkToolsDir" - tar -xf "$__ApkToolsDir/apk-tools-$__ApkToolsVersion-x86_64-linux.tar.gz" -C "$__ApkToolsDir" + + wget "https://gitlab.alpinelinux.org/api/v4/projects/5/packages/generic//v$__ApkToolsVersion/x86_64/apk.static" -P "$__ApkToolsDir" + chmod +x "$__ApkToolsDir/apk.static" + mkdir -p "$__RootfsDir"/usr/bin cp -v "/usr/bin/qemu-$__QEMUArch-static" "$__RootfsDir/usr/bin" - "$__ApkToolsDir/apk-tools-$__ApkToolsVersion/apk" \ - -X "http://dl-cdn.alpinelinux.org/alpine/v$__AlpineVersion/main" \ - -X "http://dl-cdn.alpinelinux.org/alpine/v$__AlpineVersion/community" \ - -U --allow-untrusted --root "$__RootfsDir" --arch "$__AlpineArch" --initdb \ - add $__AlpinePackages + if [[ "$__AlpineVersion" == "edge" ]]; then + version=edge + else + version="v$__AlpineVersion" + fi + + # initialize DB + "$__ApkToolsDir/apk.static" \ + -X "http://dl-cdn.alpinelinux.org/alpine/$version/main" \ + -X "http://dl-cdn.alpinelinux.org/alpine/$version/community" \ + -U --allow-untrusted --root "$__RootfsDir" --arch "$__AlpineArch" --initdb add + + if [[ "$__AlpineLlvmLibsLookup" == 1 ]]; then + __AlpinePackages+=" $("$__ApkToolsDir/apk.static" \ + -X "http://dl-cdn.alpinelinux.org/alpine/$version/main" \ + -X "http://dl-cdn.alpinelinux.org/alpine/$version/community" \ + -U --allow-untrusted --root "$__RootfsDir" --arch "$__AlpineArch" \ + search 'llvm*-libs' | sort | tail -1 | sed 's/-[^-]*//2g')" + fi + + # install all packages in one go + "$__ApkToolsDir/apk.static" \ + -X "http://dl-cdn.alpinelinux.org/alpine/$version/main" \ + -X "http://dl-cdn.alpinelinux.org/alpine/$version/community" \ + -U --allow-untrusted --root "$__RootfsDir" --arch "$__AlpineArch" \ + add $__AlpinePackages rm -r "$__ApkToolsDir" elif [[ "$__CodeName" == "freebsd" ]]; then diff --git a/eng/common/generate-locproject.ps1 b/eng/common/generate-locproject.ps1 index bcb579e3..524aaa57 100644 --- a/eng/common/generate-locproject.ps1 +++ b/eng/common/generate-locproject.ps1 @@ -45,7 +45,7 @@ if (-not $wxlFiles) { } } -$macosHtmlEnFiles = Get-ChildItem -Recurse -Path "$SourcesDirectory" | Where-Object { $_.FullName -Match "en\.lproj\\.+\.html" } # add installer HTML files +$macosHtmlEnFiles = Get-ChildItem -Recurse -Path "$SourcesDirectory" | Where-Object { $_.FullName -Match "en\.lproj\\.+\.html$" } # add installer HTML files $macosHtmlFiles = @() if ($macosHtmlEnFiles) { $macosHtmlEnFiles | ForEach-Object { @@ -148,12 +148,17 @@ $locJson = @{ } } $sourceFile = ($_.FullName | Resolve-Path -Relative) + $lciFile = $sourceFile + ".lci" if ($continue) { - return @{ + $result = @{ SourceFile = $sourceFile CopyOption = "LangIDOnPath" OutputPath = $outputPath } + if (Test-Path $lciFile -PathType Leaf) { + $result["LciFile"] = $lciFile + } + return $result } } ) diff --git a/eng/common/init-tools-native.ps1 b/eng/common/init-tools-native.ps1 index fbc67eff..27ccdb9e 100644 --- a/eng/common/init-tools-native.ps1 +++ b/eng/common/init-tools-native.ps1 @@ -83,7 +83,8 @@ try { Select-Object -Expand 'native-tools' -ErrorAction SilentlyContinue if ($NativeTools) { if ($PathPromotion -eq $True) { - if ($env:SYSTEM_TEAMPROJECT) { # check to see if we're in an Azure pipelines build + $ArcadeToolsDirectory = "$env:SYSTEMDRIVE\arcade-tools" + if (Test-Path $ArcadeToolsDirectory) { # if this directory exists, we should use native tools on machine $NativeTools.PSObject.Properties | ForEach-Object { $ToolName = $_.Name $ToolVersion = $_.Value @@ -93,11 +94,6 @@ try { if ($ToolVersion -eq "latest") { $ToolVersion = "" } - $ArcadeToolsDirectory = "C:\arcade-tools" - if (-not (Test-Path $ArcadeToolsDirectory)) { - Write-Error "Arcade tools directory '$ArcadeToolsDirectory' was not found; artifacts were not properly installed." - exit 1 - } $ToolDirectories = (Get-ChildItem -Path "$ArcadeToolsDirectory" -Filter "$ToolName-$ToolVersion*" | Sort-Object -Descending) if ($ToolDirectories -eq $null) { Write-Error "Unable to find directory for $ToolName $ToolVersion; please make sure the tool is installed on this image." @@ -125,6 +121,7 @@ try { if ((Get-Command "$ToolName" -ErrorAction SilentlyContinue) -eq $null) { Write-PipelineTelemetryError -Category 'NativeToolsBootstrap' -Message "$ToolName not found on path. Please install $ToolName $ToolVersion before proceeding." + Write-PipelineTelemetryError -Category 'NativeToolsBootstrap' -Message "If this is running on a build machine, the arcade-tools directory was not found, which means there's an error with the image." } } exit 0 diff --git a/eng/common/templates/job/job.yml b/eng/common/templates/job/job.yml index f0af425d..b214a31d 100644 --- a/eng/common/templates/job/job.yml +++ b/eng/common/templates/job/job.yml @@ -101,7 +101,7 @@ jobs: # handle key-value variable syntax. # example: # - [key]: [value] - - ${{ if and(eq(variable.name, ''), eq(variable.group, '')) }}: + - ${{ if and(eq(variable.name, ''), eq(variable.group, ''), eq(variable.template, '')) }}: - ${{ each pair in variable }}: - name: ${{ pair.key }} value: ${{ pair.value }} @@ -134,7 +134,7 @@ jobs: - ${{ if and(eq(parameters.runAsPublic, 'false'), eq(variables['System.TeamProject'], 'internal')) }}: - task: NuGetAuthenticate@0 - - ${{ if or(eq(parameters.artifacts.download, 'true'), ne(parameters.artifacts.download, '')) }}: + - ${{ if and(ne(parameters.artifacts.download, 'false'), ne(parameters.artifacts.download, '')) }}: - task: DownloadPipelineArtifact@2 inputs: buildType: current @@ -171,7 +171,7 @@ jobs: TeamName: $(_TeamName) - ${{ if ne(parameters.artifacts.publish, '') }}: - - ${{ if or(eq(parameters.artifacts.publish.artifacts, 'true'), ne(parameters.artifacts.publish.artifacts, '')) }}: + - ${{ if and(ne(parameters.artifacts.publish.artifacts, 'false'), ne(parameters.artifacts.publish.artifacts, '')) }}: - task: CopyFiles@2 displayName: Gather binaries for publish to artifacts inputs: @@ -192,7 +192,7 @@ jobs: ArtifactName: ${{ coalesce(parameters.artifacts.publish.artifacts.name , 'Artifacts_$(Agent.Os)_$(_BuildConfig)') }} continueOnError: true condition: always() - - ${{ if or(eq(parameters.artifacts.publish.logs, 'true'), ne(parameters.artifacts.publish.logs, '')) }}: + - ${{ if and(ne(parameters.artifacts.publish.logs, 'false'), ne(parameters.artifacts.publish.logs, '')) }}: - publish: artifacts/log artifact: ${{ coalesce(parameters.artifacts.publish.logs.name, 'Logs_Build_$(Agent.Os)_$(_BuildConfig)') }} displayName: Publish logs diff --git a/eng/common/templates/job/source-index-stage1.yml b/eng/common/templates/job/source-index-stage1.yml index 71b66aeb..b98202aa 100644 --- a/eng/common/templates/job/source-index-stage1.yml +++ b/eng/common/templates/job/source-index-stage1.yml @@ -1,6 +1,6 @@ parameters: runAsPublic: false - sourceIndexPackageVersion: 1.0.1-20221220.2 + sourceIndexPackageVersion: 1.0.1-20230228.2 sourceIndexPackageSource: https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools/nuget/v3/index.json sourceIndexBuildCommand: powershell -NoLogo -NoProfile -ExecutionPolicy Bypass -Command "eng/common/build.ps1 -restore -build -binarylog -ci" preSteps: [] diff --git a/eng/common/tools.ps1 b/eng/common/tools.ps1 index 7caacc6d..8ad03be3 100644 --- a/eng/common/tools.ps1 +++ b/eng/common/tools.ps1 @@ -913,11 +913,13 @@ if (!$disableConfigureToolsetImport) { function Enable-Nuget-EnhancedRetry() { if ($ci) { Write-Host "Setting NUGET enhanced retry environment variables" - $env:NUGET_ENABLE_EXPERIMENTAL_HTTP_RETRY = 'true' - $env:NUGET_EXPERIMENTAL_MAX_NETWORK_TRY_COUNT = 6 - $env:NUGET_EXPERIMENTAL_NETWORK_RETRY_DELAY_MILLISECONDS = 1000 - Write-PipelineSetVariable -Name 'NUGET_ENABLE_EXPERIMENTAL_HTTP_RETRY' -Value 'true' - Write-PipelineSetVariable -Name 'NUGET_EXPERIMENTAL_MAX_NETWORK_TRY_COUNT' -Value '6' - Write-PipelineSetVariable -Name 'NUGET_EXPERIMENTAL_NETWORK_RETRY_DELAY_MILLISECONDS' -Value '1000' + $env:NUGET_ENABLE_ENHANCED_HTTP_RETRY = 'true' + $env:NUGET_ENHANCED_MAX_NETWORK_TRY_COUNT = 6 + $env:NUGET_ENHANCED_NETWORK_RETRY_DELAY_MILLISECONDS = 1000 + $env:NUGET_RETRY_HTTP_429 = 'true' + Write-PipelineSetVariable -Name 'NUGET_ENABLE_ENHANCED_HTTP_RETRY' -Value 'true' + Write-PipelineSetVariable -Name 'NUGET_ENHANCED_MAX_NETWORK_TRY_COUNT' -Value '6' + Write-PipelineSetVariable -Name 'NUGET_ENHANCED_NETWORK_RETRY_DELAY_MILLISECONDS' -Value '1000' + Write-PipelineSetVariable -Name 'NUGET_RETRY_HTTP_429' -Value 'true' } } diff --git a/global.json b/global.json index 092cdd5e..f3cfc534 100644 --- a/global.json +++ b/global.json @@ -3,7 +3,7 @@ "allowPrerelease": true }, "tools": { - "dotnet": "8.0.100-alpha.1.23061.8", + "dotnet": "8.0.100-preview.1.23115.2", "runtimes": { "dotnet": [ "6.0.0" @@ -14,6 +14,6 @@ } }, "msbuild-sdks": { - "Microsoft.DotNet.Arcade.Sdk": "8.0.0-beta.23106.4" + "Microsoft.DotNet.Arcade.Sdk": "8.0.0-beta.23168.1" } }