diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index 2e9942dd..df1ec7c3 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -5,14 +5,14 @@ - + https://github.com/dotnet/arcade - 85f3aa16d8797b5020f1fda11df1a958feb5f8df + 02e28316bf35d1028683ee313f0794776bff18d1 - + https://github.com/dotnet/arcade - 85f3aa16d8797b5020f1fda11df1a958feb5f8df + 02e28316bf35d1028683ee313f0794776bff18d1 diff --git a/eng/common/SetupNugetSources.ps1 b/eng/common/SetupNugetSources.ps1 index 18823840..6e997239 100644 --- a/eng/common/SetupNugetSources.ps1 +++ b/eng/common/SetupNugetSources.ps1 @@ -146,22 +146,22 @@ $userName = "dn-bot" # Insert credential nodes for Maestro's private feeds InsertMaestroPrivateFeedCredentials -Sources $sources -Creds $creds -Username $userName -Password $Password +# 3.1 uses a different feed url format so it's handled differently here $dotnet31Source = $sources.SelectSingleNode("add[@key='dotnet3.1']") if ($dotnet31Source -ne $null) { AddPackageSource -Sources $sources -SourceName "dotnet3.1-internal" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/_packaging/dotnet3.1-internal/nuget/v2" -Creds $creds -Username $userName -Password $Password AddPackageSource -Sources $sources -SourceName "dotnet3.1-internal-transport" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/_packaging/dotnet3.1-internal-transport/nuget/v2" -Creds $creds -Username $userName -Password $Password } -$dotnet5Source = $sources.SelectSingleNode("add[@key='dotnet5']") -if ($dotnet5Source -ne $null) { - AddPackageSource -Sources $sources -SourceName "dotnet5-internal" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/internal/_packaging/dotnet5-internal/nuget/v2" -Creds $creds -Username $userName -Password $Password - AddPackageSource -Sources $sources -SourceName "dotnet5-internal-transport" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/internal/_packaging/dotnet5-internal-transport/nuget/v2" -Creds $creds -Username $userName -Password $Password -} +$dotnetVersions = @('5','6','7') -$dotnet6Source = $sources.SelectSingleNode("add[@key='dotnet6']") -if ($dotnet6Source -ne $null) { - AddPackageSource -Sources $sources -SourceName "dotnet6-internal" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/internal/_packaging/dotnet6-internal/nuget/v2" -Creds $creds -Username $userName -Password $Password - AddPackageSource -Sources $sources -SourceName "dotnet6-internal-transport" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/internal/_packaging/dotnet6-internal-transport/nuget/v2" -Creds $creds -Username $userName -Password $Password +foreach ($dotnetVersion in $dotnetVersions) { + $feedPrefix = "dotnet" + $dotnetVersion; + $dotnetSource = $sources.SelectSingleNode("add[@key='$feedPrefix']") + if ($dotnetSource -ne $null) { + AddPackageSource -Sources $sources -SourceName "$feedPrefix-internal" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/internal/_packaging/$feedPrefix-internal/nuget/v2" -Creds $creds -Username $userName -Password $Password + AddPackageSource -Sources $sources -SourceName "$feedPrefix-internal-transport" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/internal/_packaging/$feedPrefix-internal-transport/nuget/v2" -Creds $creds -Username $userName -Password $Password + } } $doc.Save($filename) diff --git a/eng/common/SetupNugetSources.sh b/eng/common/SetupNugetSources.sh index ad3fb74f..8af7d899 100755 --- a/eng/common/SetupNugetSources.sh +++ b/eng/common/SetupNugetSources.sh @@ -105,53 +105,33 @@ if [ "$?" == "0" ]; then PackageSources+=('dotnet3.1-internal-transport') fi -# Ensure dotnet5-internal and dotnet5-internal-transport are in the packageSources if the public dotnet5 feeds are present -grep -i "" - - sed -i.bak "s|$PackageSourcesNodeFooter|$PackageSourceTemplate${NL}$PackageSourcesNodeFooter|" $ConfigFile - fi - PackageSources+=('dotnet5-internal') - - grep -i "" $ConfigFile - if [ "$?" != "0" ]; then - echo "Adding dotnet5-internal-transport to the packageSources." - PackageSourcesNodeFooter="" - PackageSourceTemplate="${TB}" - - sed -i.bak "s|$PackageSourcesNodeFooter|$PackageSourceTemplate${NL}$PackageSourcesNodeFooter|" $ConfigFile - fi - PackageSources+=('dotnet5-internal-transport') -fi - -# Ensure dotnet6-internal and dotnet6-internal-transport are in the packageSources if the public dotnet6 feeds are present -grep -i "" +DotNetVersions=('5' '6' '7') + +for DotNetVersion in ${DotNetVersions[@]} ; do + FeedPrefix="dotnet${DotNetVersion}"; + grep -i "" + + sed -i.bak "s|$PackageSourcesNodeFooter|$PackageSourceTemplate${NL}$PackageSourcesNodeFooter|" $ConfigFile + fi + PackageSources+=("$FeedPrefix-internal") - sed -i.bak "s|$PackageSourcesNodeFooter|$PackageSourceTemplate${NL}$PackageSourcesNodeFooter|" $ConfigFile - fi - PackageSources+=('dotnet6-internal') + grep -i "" $ConfigFile + if [ "$?" != "0" ]; then + echo "Adding $FeedPrefix-internal-transport to the packageSources." + PackageSourcesNodeFooter="" + PackageSourceTemplate="${TB}" - grep -i "" $ConfigFile - if [ "$?" != "0" ]; then - echo "Adding dotnet6-internal-transport to the packageSources." - PackageSourcesNodeFooter="" - PackageSourceTemplate="${TB}" - - sed -i.bak "s|$PackageSourcesNodeFooter|$PackageSourceTemplate${NL}$PackageSourcesNodeFooter|" $ConfigFile + sed -i.bak "s|$PackageSourcesNodeFooter|$PackageSourceTemplate${NL}$PackageSourcesNodeFooter|" $ConfigFile + fi + PackageSources+=("$FeedPrefix-internal-transport") fi - PackageSources+=('dotnet6-internal-transport') -fi +done # I want things split line by line PrevIFS=$IFS diff --git a/eng/common/build.ps1 b/eng/common/build.ps1 index 8943da24..e0420a64 100644 --- a/eng/common/build.ps1 +++ b/eng/common/build.ps1 @@ -26,6 +26,7 @@ Param( [string] $runtimeSourceFeed = '', [string] $runtimeSourceFeedKey = '', [switch] $excludePrereleaseVS, + [switch] $nativeToolsOnMachine, [switch] $help, [Parameter(ValueFromRemainingArguments=$true)][String[]]$properties ) @@ -66,6 +67,7 @@ function Print-Usage() { Write-Host " -prepareMachine Prepare machine for CI run, clean up processes after build" Write-Host " -warnAsError Sets warnaserror msbuild parameter ('true' or 'false')" Write-Host " -msbuildEngine Msbuild engine to use to run build ('dotnet', 'vs', or unspecified)." + Write-Host " -nativeToolsOnMachine Sets the native tools on machine environment variable (indicating that the script should use native tools on machine)" Write-Host " -excludePrereleaseVS Set to exclude build engines in prerelease versions of Visual Studio" Write-Host "" @@ -146,6 +148,9 @@ try { $nodeReuse = $false } + if ($nativeToolsOnMachine) { + $env:NativeToolsOnMachine = $true + } if ($restore) { InitializeNativeTools } diff --git a/eng/common/build.sh b/eng/common/build.sh index bc07a1c6..50af40cd 100755 --- a/eng/common/build.sh +++ b/eng/common/build.sh @@ -19,6 +19,9 @@ usage() echo "Actions:" echo " --restore Restore dependencies (short: -r)" echo " --build Build solution (short: -b)" + echo " --sourceBuild Source-build the solution (short: -sb)" + echo " Will additionally trigger the following actions: --restore, --build, --pack" + echo " If --configuration is not set explicitly, will also set it to 'Release'" echo " --rebuild Rebuild solution" echo " --test Run all unit tests in the solution (short: -t)" echo " --integrationTest Run all integration tests in the solution" @@ -55,6 +58,7 @@ scriptroot="$( cd -P "$( dirname "$source" )" && pwd )" restore=false build=false +source_build=false rebuild=false test=false integration_test=false @@ -73,7 +77,7 @@ exclude_ci_binary_log=false pipelines_log=false projects='' -configuration='Debug' +configuration='' prepare_machine=false verbosity='minimal' runtime_source_feed='' @@ -119,6 +123,12 @@ while [[ $# > 0 ]]; do -pack) pack=true ;; + -sourcebuild|-sb) + build=true + source_build=true + restore=true + pack=true + ;; -test|-t) test=true ;; @@ -168,6 +178,10 @@ while [[ $# > 0 ]]; do shift done +if [[ -z "$configuration" ]]; then + if [[ "$source_build" = true ]]; then configuration="Release"; else configuration="Debug"; fi +fi + if [[ "$ci" == true ]]; then pipelines_log=true node_reuse=false @@ -187,10 +201,6 @@ function InitializeCustomToolset { } function Build { - - if [[ "$ci" == true ]]; then - TryLogClientIpAddress - fi InitializeToolset InitializeCustomToolset @@ -209,6 +219,7 @@ function Build { /p:RepoRoot="$repo_root" \ /p:Restore=$restore \ /p:Build=$build \ + /p:ArcadeBuildFromSource=$source_build \ /p:Rebuild=$rebuild \ /p:Test=$test \ /p:Pack=$pack \ diff --git a/eng/common/cross/arm/sources.list.focal b/eng/common/cross/arm/sources.list.focal new file mode 100644 index 00000000..4de2600c --- /dev/null +++ b/eng/common/cross/arm/sources.list.focal @@ -0,0 +1,11 @@ +deb http://ports.ubuntu.com/ubuntu-ports/ focal main restricted universe +deb-src http://ports.ubuntu.com/ubuntu-ports/ focal main restricted universe + +deb http://ports.ubuntu.com/ubuntu-ports/ focal-updates main restricted universe +deb-src http://ports.ubuntu.com/ubuntu-ports/ focal-updates main restricted universe + +deb http://ports.ubuntu.com/ubuntu-ports/ focal-backports main restricted +deb-src http://ports.ubuntu.com/ubuntu-ports/ focal-backports main restricted + +deb http://ports.ubuntu.com/ubuntu-ports/ focal-security main restricted universe multiverse +deb-src http://ports.ubuntu.com/ubuntu-ports/ focal-security main restricted universe multiverse diff --git a/eng/common/cross/arm/sources.list.jammy b/eng/common/cross/arm/sources.list.jammy new file mode 100644 index 00000000..6bb04530 --- /dev/null +++ b/eng/common/cross/arm/sources.list.jammy @@ -0,0 +1,11 @@ +deb http://ports.ubuntu.com/ubuntu-ports/ jammy main restricted universe +deb-src http://ports.ubuntu.com/ubuntu-ports/ jammy main restricted universe + +deb http://ports.ubuntu.com/ubuntu-ports/ jammy-updates main restricted universe +deb-src http://ports.ubuntu.com/ubuntu-ports/ jammy-updates main restricted universe + +deb http://ports.ubuntu.com/ubuntu-ports/ jammy-backports main restricted +deb-src http://ports.ubuntu.com/ubuntu-ports/ jammy-backports main restricted + +deb http://ports.ubuntu.com/ubuntu-ports/ jammy-security main restricted universe multiverse +deb-src http://ports.ubuntu.com/ubuntu-ports/ jammy-security main restricted universe multiverse diff --git a/eng/common/cross/arm/tizen-build-rootfs.sh b/eng/common/cross/arm/tizen-build-rootfs.sh new file mode 100644 index 00000000..9fdb32e9 --- /dev/null +++ b/eng/common/cross/arm/tizen-build-rootfs.sh @@ -0,0 +1,35 @@ +#!/usr/bin/env bash +set -e + +__ARM_HARDFP_CrossDir=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd ) +__TIZEN_CROSSDIR="$__ARM_HARDFP_CrossDir/tizen" + +if [[ -z "$ROOTFS_DIR" ]]; then + echo "ROOTFS_DIR is not defined." + exit 1; +fi + +TIZEN_TMP_DIR=$ROOTFS_DIR/tizen_tmp +mkdir -p $TIZEN_TMP_DIR + +# Download files +echo ">>Start downloading files" +VERBOSE=1 $__ARM_HARDFP_CrossDir/tizen-fetch.sh $TIZEN_TMP_DIR +echo "<>Start constructing Tizen rootfs" +TIZEN_RPM_FILES=`ls $TIZEN_TMP_DIR/*.rpm` +cd $ROOTFS_DIR +for f in $TIZEN_RPM_FILES; do + rpm2cpio $f | cpio -idm --quiet +done +echo "<>Start configuring Tizen rootfs" +ln -sfn asm-arm ./usr/include/asm +patch -p1 < $__TIZEN_CROSSDIR/tizen.patch +echo "</dev/null; then + VERBOSE=0 +fi + +Log() +{ + if [ $VERBOSE -ge $1 ]; then + echo ${@:2} + fi +} + +Inform() +{ + Log 1 -e "\x1B[0;34m$@\x1B[m" +} + +Debug() +{ + Log 2 -e "\x1B[0;32m$@\x1B[m" +} + +Error() +{ + >&2 Log 0 -e "\x1B[0;31m$@\x1B[m" +} + +Fetch() +{ + URL=$1 + FILE=$2 + PROGRESS=$3 + if [ $VERBOSE -ge 1 ] && [ $PROGRESS ]; then + CURL_OPT="--progress-bar" + else + CURL_OPT="--silent" + fi + curl $CURL_OPT $URL > $FILE +} + +hash curl 2> /dev/null || { Error "Require 'curl' Aborting."; exit 1; } +hash xmllint 2> /dev/null || { Error "Require 'xmllint' Aborting."; exit 1; } +hash sha256sum 2> /dev/null || { Error "Require 'sha256sum' Aborting."; exit 1; } + +TMPDIR=$1 +if [ ! -d $TMPDIR ]; then + TMPDIR=./tizen_tmp + Debug "Create temporary directory : $TMPDIR" + mkdir -p $TMPDIR +fi + +TIZEN_URL=http://download.tizen.org/snapshots/tizen +BUILD_XML=build.xml +REPOMD_XML=repomd.xml +PRIMARY_XML=primary.xml +TARGET_URL="http://__not_initialized" + +Xpath_get() +{ + XPATH_RESULT='' + XPATH=$1 + XML_FILE=$2 + RESULT=$(xmllint --xpath $XPATH $XML_FILE) + if [[ -z ${RESULT// } ]]; then + Error "Can not find target from $XML_FILE" + Debug "Xpath = $XPATH" + exit 1 + fi + XPATH_RESULT=$RESULT +} + +fetch_tizen_pkgs_init() +{ + TARGET=$1 + PROFILE=$2 + Debug "Initialize TARGET=$TARGET, PROFILE=$PROFILE" + + TMP_PKG_DIR=$TMPDIR/tizen_${PROFILE}_pkgs + if [ -d $TMP_PKG_DIR ]; then rm -rf $TMP_PKG_DIR; fi + mkdir -p $TMP_PKG_DIR + + PKG_URL=$TIZEN_URL/$PROFILE/latest + + BUILD_XML_URL=$PKG_URL/$BUILD_XML + TMP_BUILD=$TMP_PKG_DIR/$BUILD_XML + TMP_REPOMD=$TMP_PKG_DIR/$REPOMD_XML + TMP_PRIMARY=$TMP_PKG_DIR/$PRIMARY_XML + TMP_PRIMARYGZ=${TMP_PRIMARY}.gz + + Fetch $BUILD_XML_URL $TMP_BUILD + + Debug "fetch $BUILD_XML_URL to $TMP_BUILD" + + TARGET_XPATH="//build/buildtargets/buildtarget[@name=\"$TARGET\"]/repo[@type=\"binary\"]/text()" + Xpath_get $TARGET_XPATH $TMP_BUILD + TARGET_PATH=$XPATH_RESULT + TARGET_URL=$PKG_URL/$TARGET_PATH + + REPOMD_URL=$TARGET_URL/repodata/repomd.xml + PRIMARY_XPATH='string(//*[local-name()="data"][@type="primary"]/*[local-name()="location"]/@href)' + + Fetch $REPOMD_URL $TMP_REPOMD + + Debug "fetch $REPOMD_URL to $TMP_REPOMD" + + Xpath_get $PRIMARY_XPATH $TMP_REPOMD + PRIMARY_XML_PATH=$XPATH_RESULT + PRIMARY_URL=$TARGET_URL/$PRIMARY_XML_PATH + + Fetch $PRIMARY_URL $TMP_PRIMARYGZ + + Debug "fetch $PRIMARY_URL to $TMP_PRIMARYGZ" + + gunzip $TMP_PRIMARYGZ + + Debug "unzip $TMP_PRIMARYGZ to $TMP_PRIMARY" +} + +fetch_tizen_pkgs() +{ + ARCH=$1 + PACKAGE_XPATH_TPL='string(//*[local-name()="metadata"]/*[local-name()="package"][*[local-name()="name"][text()="_PKG_"]][*[local-name()="arch"][text()="_ARCH_"]]/*[local-name()="location"]/@href)' + + PACKAGE_CHECKSUM_XPATH_TPL='string(//*[local-name()="metadata"]/*[local-name()="package"][*[local-name()="name"][text()="_PKG_"]][*[local-name()="arch"][text()="_ARCH_"]]/*[local-name()="checksum"]/text())' + + for pkg in ${@:2} + do + Inform "Fetching... $pkg" + XPATH=${PACKAGE_XPATH_TPL/_PKG_/$pkg} + XPATH=${XPATH/_ARCH_/$ARCH} + Xpath_get $XPATH $TMP_PRIMARY + PKG_PATH=$XPATH_RESULT + + XPATH=${PACKAGE_CHECKSUM_XPATH_TPL/_PKG_/$pkg} + XPATH=${XPATH/_ARCH_/$ARCH} + Xpath_get $XPATH $TMP_PRIMARY + CHECKSUM=$XPATH_RESULT + + PKG_URL=$TARGET_URL/$PKG_PATH + PKG_FILE=$(basename $PKG_PATH) + PKG_PATH=$TMPDIR/$PKG_FILE + + Debug "Download $PKG_URL to $PKG_PATH" + Fetch $PKG_URL $PKG_PATH true + + echo "$CHECKSUM $PKG_PATH" | sha256sum -c - > /dev/null + if [ $? -ne 0 ]; then + Error "Fail to fetch $PKG_URL to $PKG_PATH" + Debug "Checksum = $CHECKSUM" + exit 1 + fi + done +} + +Inform "Initialize arm base" +fetch_tizen_pkgs_init standard base +Inform "fetch common packages" +fetch_tizen_pkgs armv7hl gcc gcc-devel-static glibc glibc-devel libicu libicu-devel libatomic linux-glibc-devel keyutils keyutils-devel libkeyutils +Inform "fetch coreclr packages" +fetch_tizen_pkgs armv7hl lldb lldb-devel libgcc libstdc++ libstdc++-devel libunwind libunwind-devel lttng-ust-devel lttng-ust userspace-rcu-devel userspace-rcu +Inform "fetch corefx packages" +fetch_tizen_pkgs armv7hl libcom_err libcom_err-devel zlib zlib-devel libopenssl11 libopenssl1.1-devel krb5 krb5-devel + +Inform "Initialize standard unified" +fetch_tizen_pkgs_init standard unified +Inform "fetch corefx packages" +fetch_tizen_pkgs armv7hl gssdp gssdp-devel tizen-release + diff --git a/eng/common/cross/arm/tizen/tizen.patch b/eng/common/cross/arm/tizen/tizen.patch new file mode 100644 index 00000000..fb12ade7 --- /dev/null +++ b/eng/common/cross/arm/tizen/tizen.patch @@ -0,0 +1,9 @@ +diff -u -r a/usr/lib/libc.so b/usr/lib/libc.so +--- a/usr/lib/libc.so 2016-12-30 23:00:08.284951863 +0900 ++++ b/usr/lib/libc.so 2016-12-30 23:00:32.140951815 +0900 +@@ -2,4 +2,4 @@ + Use the shared library, but some functions are only in + the static library, so try that secondarily. */ + OUTPUT_FORMAT(elf32-littlearm) +-GROUP ( /lib/libc.so.6 /usr/lib/libc_nonshared.a AS_NEEDED ( /lib/ld-linux-armhf.so.3 ) ) ++GROUP ( libc.so.6 libc_nonshared.a AS_NEEDED ( ld-linux-armhf.so.3 ) ) diff --git a/eng/common/cross/arm64/sources.list.focal b/eng/common/cross/arm64/sources.list.focal new file mode 100644 index 00000000..4de2600c --- /dev/null +++ b/eng/common/cross/arm64/sources.list.focal @@ -0,0 +1,11 @@ +deb http://ports.ubuntu.com/ubuntu-ports/ focal main restricted universe +deb-src http://ports.ubuntu.com/ubuntu-ports/ focal main restricted universe + +deb http://ports.ubuntu.com/ubuntu-ports/ focal-updates main restricted universe +deb-src http://ports.ubuntu.com/ubuntu-ports/ focal-updates main restricted universe + +deb http://ports.ubuntu.com/ubuntu-ports/ focal-backports main restricted +deb-src http://ports.ubuntu.com/ubuntu-ports/ focal-backports main restricted + +deb http://ports.ubuntu.com/ubuntu-ports/ focal-security main restricted universe multiverse +deb-src http://ports.ubuntu.com/ubuntu-ports/ focal-security main restricted universe multiverse diff --git a/eng/common/cross/arm64/sources.list.jammy b/eng/common/cross/arm64/sources.list.jammy new file mode 100644 index 00000000..6bb04530 --- /dev/null +++ b/eng/common/cross/arm64/sources.list.jammy @@ -0,0 +1,11 @@ +deb http://ports.ubuntu.com/ubuntu-ports/ jammy main restricted universe +deb-src http://ports.ubuntu.com/ubuntu-ports/ jammy main restricted universe + +deb http://ports.ubuntu.com/ubuntu-ports/ jammy-updates main restricted universe +deb-src http://ports.ubuntu.com/ubuntu-ports/ jammy-updates main restricted universe + +deb http://ports.ubuntu.com/ubuntu-ports/ jammy-backports main restricted +deb-src http://ports.ubuntu.com/ubuntu-ports/ jammy-backports main restricted + +deb http://ports.ubuntu.com/ubuntu-ports/ jammy-security main restricted universe multiverse +deb-src http://ports.ubuntu.com/ubuntu-ports/ jammy-security main restricted universe multiverse diff --git a/eng/common/cross/armv6/sources.list.buster b/eng/common/cross/armv6/sources.list.buster new file mode 100644 index 00000000..f27fc4fb --- /dev/null +++ b/eng/common/cross/armv6/sources.list.buster @@ -0,0 +1,2 @@ +deb http://raspbian.raspberrypi.org/raspbian/ buster main contrib non-free rpi +deb-src http://raspbian.raspberrypi.org/raspbian/ buster main contrib non-free rpi diff --git a/eng/common/cross/build-rootfs.sh b/eng/common/cross/build-rootfs.sh index 5c05b39f..5680980f 100755 --- a/eng/common/cross/build-rootfs.sh +++ b/eng/common/cross/build-rootfs.sh @@ -4,22 +4,27 @@ set -e usage() { - echo "Usage: $0 [BuildArch] [CodeName] [lldbx.y] [--skipunmount] --rootfsdir ]" - echo "BuildArch can be: arm(default), armel, arm64, x86" - echo "CodeName - optional, Code name for Linux, can be: xenial(default), zesty, bionic, alpine, alpine3.9 or alpine3.13. If BuildArch is armel, LinuxCodeName is jessie(default) or tizen." - echo " for FreeBSD can be: freebsd11, freebsd12, freebsd13" - echo " for illumos can be: illumos." + echo "Usage: $0 [BuildArch] [CodeName] [lldbx.y] [llvmx[.y]] [--skipunmount] --rootfsdir ]" + echo "BuildArch can be: arm(default), arm64, armel, armv6, ppc64le, riscv64, s390x, x64, x86" + echo "CodeName - optional, Code name for Linux, can be: xenial(default), zesty, bionic, alpine, alpine3.13 or alpine3.14. If BuildArch is armel, LinuxCodeName is jessie(default) or tizen." + echo " for FreeBSD can be: freebsd12, freebsd13" + echo " for illumos can be: illumos" + echo " for Haiku can be: haiku." echo "lldbx.y - optional, LLDB version, can be: lldb3.9(default), lldb4.0, lldb5.0, lldb6.0 no-lldb. Ignored for alpine and FreeBSD" + echo "llvmx[.y] - optional, LLVM version for LLVM related packages." echo "--skipunmount - optional, will skip the unmount of rootfs folder." echo "--use-mirror - optional, use mirror URL to fetch resources, when available." + echo "--jobs N - optional, restrict to N jobs." exit 1 } __CodeName=xenial __CrossDir=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd ) -__InitialDir=$PWD __BuildArch=arm __AlpineArch=armv7 +__FreeBSDArch=arm +__FreeBSDMachineArch=armv7 +__IllumosArch=arm7 __QEMUArch=arm __UbuntuArch=armhf __UbuntuRepo="http://ports.ubuntu.com/" @@ -32,14 +37,14 @@ __UbuntuPackages="build-essential" __AlpinePackages="alpine-base" __AlpinePackages+=" build-base" __AlpinePackages+=" linux-headers" -__AlpinePackagesEdgeCommunity=" lldb-dev" -__AlpinePackagesEdgeMain+=" python3" -__AlpinePackagesEdgeMain+=" libedit" +__AlpinePackages+=" lldb-dev" +__AlpinePackages+=" python3" +__AlpinePackages+=" libedit" # symlinks fixer __UbuntuPackages+=" symlinks" -# CoreCLR and CoreFX dependencies +# runtime dependencies __UbuntuPackages+=" libicu-dev" __UbuntuPackages+=" liblttng-ust-dev" __UbuntuPackages+=" libunwind8-dev" @@ -48,8 +53,9 @@ __AlpinePackages+=" gettext-dev" __AlpinePackages+=" icu-dev" __AlpinePackages+=" libunwind-dev" __AlpinePackages+=" lttng-ust-dev" +__AlpinePackages+=" compiler-rt-static" -# CoreFX dependencies +# runtime libraries' dependencies __UbuntuPackages+=" libcurl4-openssl-dev" __UbuntuPackages+=" libkrb5-dev" __UbuntuPackages+=" libssl-dev" @@ -60,36 +66,48 @@ __AlpinePackages+=" krb5-dev" __AlpinePackages+=" openssl-dev" __AlpinePackages+=" zlib-dev" -__FreeBSDBase="12.2-RELEASE" -__FreeBSDPkg="1.12.0" +__FreeBSDBase="12.3-RELEASE" +__FreeBSDPkg="1.17.0" __FreeBSDABI="12" __FreeBSDPackages="libunwind" __FreeBSDPackages+=" icu" __FreeBSDPackages+=" libinotify" -__FreeBSDPackages+=" lttng-ust" +__FreeBSDPackages+=" openssl" __FreeBSDPackages+=" krb5" __FreeBSDPackages+=" terminfo-db" -__IllumosPackages="icu-64.2nb2" -__IllumosPackages+=" mit-krb5-1.16.2nb4" -__IllumosPackages+=" openssl-1.1.1e" -__IllumosPackages+=" zlib-1.2.11" +__IllumosPackages="icu" +__IllumosPackages+=" mit-krb5" +__IllumosPackages+=" openssl" +__IllumosPackages+=" zlib" + +__HaikuPackages="gmp" +__HaikuPackages+=" gmp_devel" +__HaikuPackages+=" krb5" +__HaikuPackages+=" krb5_devel" +__HaikuPackages+=" libiconv" +__HaikuPackages+=" libiconv_devel" +__HaikuPackages+=" llvm12_libunwind" +__HaikuPackages+=" llvm12_libunwind_devel" +__HaikuPackages+=" mpfr" +__HaikuPackages+=" mpfr_devel" # ML.NET dependencies __UbuntuPackages+=" libomp5" __UbuntuPackages+=" libomp-dev" +__Keyring= __UseMirror=0 __UnprocessedBuildArgs= while :; do - if [ $# -le 0 ]; then + if [[ "$#" -le 0 ]]; then break fi - lowerI="$(echo $1 | tr "[:upper:]" "[:lower:]")" + lowerI="$(echo "$1" | tr "[:upper:]" "[:lower:]")" case $lowerI in - -?|-h|--help) + -\?|-h|--help) usage exit 1 ;; @@ -104,6 +122,8 @@ while :; do __UbuntuArch=arm64 __AlpineArch=aarch64 __QEMUArch=aarch64 + __FreeBSDArch=arm64 + __FreeBSDMachineArch=aarch64 ;; armel) __BuildArch=armel @@ -111,6 +131,39 @@ while :; do __UbuntuRepo="http://ftp.debian.org/debian/" __CodeName=jessie ;; + armv6) + __BuildArch=armv6 + __UbuntuArch=armhf + __QEMUArch=arm + __UbuntuRepo="http://raspbian.raspberrypi.org/raspbian/" + __CodeName=buster + __LLDB_Package="liblldb-6.0-dev" + + if [[ -e "/usr/share/keyrings/raspbian-archive-keyring.gpg" ]]; then + __Keyring="--keyring /usr/share/keyrings/raspbian-archive-keyring.gpg" + fi + ;; + ppc64le) + __BuildArch=ppc64le + __UbuntuArch=ppc64el + __UbuntuRepo="http://ports.ubuntu.com/ubuntu-ports/" + __UbuntuPackages=$(echo ${__UbuntuPackages} | sed 's/ libunwind8-dev//') + __UbuntuPackages=$(echo ${__UbuntuPackages} | sed 's/ libomp-dev//') + __UbuntuPackages=$(echo ${__UbuntuPackages} | sed 's/ libomp5//') + unset __LLDB_Package + ;; + riscv64) + __BuildArch=riscv64 + __UbuntuArch=riscv64 + __UbuntuRepo="http://deb.debian.org/debian-ports" + __CodeName=sid + __UbuntuPackages=$(echo ${__UbuntuPackages} | sed 's/ libunwind8-dev//') + unset __LLDB_Package + + if [[ -e "/usr/share/keyrings/debian-ports-archive-keyring.gpg" ]]; then + __Keyring="--keyring /usr/share/keyrings/debian-ports-archive-keyring.gpg --include=debian-ports-archive-keyring" + fi + ;; s390x) __BuildArch=s390x __UbuntuArch=s390x @@ -120,47 +173,69 @@ while :; do __UbuntuPackages=$(echo ${__UbuntuPackages} | sed 's/ libomp5//') unset __LLDB_Package ;; + x64) + __BuildArch=x64 + __UbuntuArch=amd64 + __FreeBSDArch=amd64 + __FreeBSDMachineArch=amd64 + __illumosArch=x86_64 + __UbuntuRepo= + ;; x86) __BuildArch=x86 __UbuntuArch=i386 __UbuntuRepo="http://archive.ubuntu.com/ubuntu/" ;; - lldb3.6) - __LLDB_Package="lldb-3.6-dev" - ;; - lldb3.8) - __LLDB_Package="lldb-3.8-dev" - ;; - lldb3.9) - __LLDB_Package="liblldb-3.9-dev" - ;; - lldb4.0) - __LLDB_Package="liblldb-4.0-dev" - ;; - lldb5.0) - __LLDB_Package="liblldb-5.0-dev" - ;; - lldb6.0) - __LLDB_Package="liblldb-6.0-dev" + lldb*) + version="${lowerI/lldb/}" + parts=(${version//./ }) + + # for versions > 6.0, lldb has dropped the minor version + if [[ "${parts[0]}" -gt 6 ]]; then + version="${parts[0]}" + fi + + __LLDB_Package="liblldb-${version}-dev" ;; no-lldb) unset __LLDB_Package ;; + llvm*) + version="${lowerI/llvm/}" + parts=(${version//./ }) + __LLVM_MajorVersion="${parts[0]}" + __LLVM_MinorVersion="${parts[1]}" + + # for versions > 6.0, llvm has dropped the minor version + if [[ -z "$__LLVM_MinorVersion" && "$__LLVM_MajorVersion" -le 6 ]]; then + __LLVM_MinorVersion=0; + fi + ;; xenial) # Ubuntu 16.04 - if [ "$__CodeName" != "jessie" ]; then + if [[ "$__CodeName" != "jessie" ]]; then __CodeName=xenial fi ;; zesty) # Ubuntu 17.04 - if [ "$__CodeName" != "jessie" ]; then + if [[ "$__CodeName" != "jessie" ]]; then __CodeName=zesty fi ;; bionic) # Ubuntu 18.04 - if [ "$__CodeName" != "jessie" ]; then + if [[ "$__CodeName" != "jessie" ]]; then __CodeName=bionic fi ;; + focal) # Ubuntu 20.04 + if [[ "$__CodeName" != "jessie" ]]; then + __CodeName=focal + fi + ;; + jammy) # Ubuntu 22.04 + if [[ "$__CodeName" != "jessie" ]]; then + __CodeName=jammy + fi + ;; jessie) # Debian 8 __CodeName=jessie __UbuntuRepo="http://ftp.debian.org/debian/" @@ -176,51 +251,38 @@ while :; do __LLDB_Package="liblldb-6.0-dev" ;; tizen) - if [ "$__BuildArch" != "armel" ] && [ "$__BuildArch" != "arm64" ]; then - echo "Tizen is available only for armel and arm64." - usage; - exit 1; - fi __CodeName= __UbuntuRepo= __Tizen=tizen ;; - alpine|alpine3.9) + alpine|alpine3.13) __CodeName=alpine __UbuntuRepo= - __AlpineVersion=3.9 - __AlpinePackagesEdgeMain+=" llvm11-libs" - __AlpinePackagesEdgeMain+=" clang-libs" + __AlpineVersion=3.13 + __AlpinePackages+=" llvm10-libs" ;; - alpine3.13) + alpine3.14) __CodeName=alpine __UbuntuRepo= - __AlpineVersion=3.13 - # Alpine 3.13 has all the packages we need in the 3.13 repository - __AlpinePackages+=$__AlpinePackagesEdgeCommunity - __AlpinePackagesEdgeCommunity= - __AlpinePackages+=$__AlpinePackagesEdgeMain - __AlpinePackagesEdgeMain= - __AlpinePackages+=" llvm10-libs" + __AlpineVersion=3.14 + __AlpinePackages+=" llvm11-libs" ;; - freebsd11) - __FreeBSDBase="11.3-RELEASE" - __FreeBSDABI="11" - ;& freebsd12) __CodeName=freebsd - __BuildArch=x64 __SkipUnmount=1 ;; freebsd13) __CodeName=freebsd __FreeBSDBase="13.0-RELEASE" __FreeBSDABI="13" - __BuildArch=x64 __SkipUnmount=1 ;; illumos) __CodeName=illumos + __SkipUnmount=1 + ;; + haiku) + __CodeName=haiku __BuildArch=x64 __SkipUnmount=1 ;; @@ -229,11 +291,15 @@ while :; do ;; --rootfsdir|-rootfsdir) shift - __RootfsDir=$1 + __RootfsDir="$1" ;; --use-mirror) __UseMirror=1 ;; + --use-jobs) + shift + MAXJOBS=$1 + ;; *) __UnprocessedBuildArgs="$__UnprocessedBuildArgs $1" ;; @@ -242,85 +308,76 @@ while :; do shift done -if [ "$__BuildArch" == "armel" ]; then +if [[ "$__BuildArch" == "armel" ]]; then __LLDB_Package="lldb-3.5-dev" fi + __UbuntuPackages+=" ${__LLDB_Package:-}" -if [ -z "$__RootfsDir" ] && [ ! -z "$ROOTFS_DIR" ]; then - __RootfsDir=$ROOTFS_DIR +if [[ -n "$__LLVM_MajorVersion" ]]; then + __UbuntuPackages+=" libclang-common-${__LLVM_MajorVersion}${__LLVM_MinorVersion:+.$__LLVM_MinorVersion}-dev" fi -if [ -z "$__RootfsDir" ]; then +if [[ -z "$__RootfsDir" && -n "$ROOTFS_DIR" ]]; then + __RootfsDir="$ROOTFS_DIR" +fi + +if [[ -z "$__RootfsDir" ]]; then __RootfsDir="$__CrossDir/../../../.tools/rootfs/$__BuildArch" fi -if [ -d "$__RootfsDir" ]; then - if [ $__SkipUnmount == 0 ]; then - umount $__RootfsDir/* || true +if [[ -d "$__RootfsDir" ]]; then + if [[ "$__SkipUnmount" == "0" ]]; then + umount "$__RootfsDir"/* || true fi - rm -rf $__RootfsDir + rm -rf "$__RootfsDir" fi -mkdir -p $__RootfsDir +mkdir -p "$__RootfsDir" __RootfsDir="$( cd "$__RootfsDir" && pwd )" if [[ "$__CodeName" == "alpine" ]]; then __ApkToolsVersion=2.9.1 - __ApkToolsDir=$(mktemp -d) - wget https://github.com/alpinelinux/apk-tools/releases/download/v$__ApkToolsVersion/apk-tools-$__ApkToolsVersion-x86_64-linux.tar.gz -P $__ApkToolsDir - tar -xf $__ApkToolsDir/apk-tools-$__ApkToolsVersion-x86_64-linux.tar.gz -C $__ApkToolsDir - mkdir -p $__RootfsDir/usr/bin - cp -v /usr/bin/qemu-$__QEMUArch-static $__RootfsDir/usr/bin - - $__ApkToolsDir/apk-tools-$__ApkToolsVersion/apk \ - -X http://dl-cdn.alpinelinux.org/alpine/v$__AlpineVersion/main \ - -X http://dl-cdn.alpinelinux.org/alpine/v$__AlpineVersion/community \ - -U --allow-untrusted --root $__RootfsDir --arch $__AlpineArch --initdb \ - add $__AlpinePackages - - if [[ -n "$__AlpinePackagesEdgeMain" ]]; then - $__ApkToolsDir/apk-tools-$__ApkToolsVersion/apk \ - -X http://dl-cdn.alpinelinux.org/alpine/edge/main \ - -U --allow-untrusted --root $__RootfsDir --arch $__AlpineArch --initdb \ - add $__AlpinePackagesEdgeMain - fi + __ApkToolsDir="$(mktemp -d)" + wget "https://github.com/alpinelinux/apk-tools/releases/download/v$__ApkToolsVersion/apk-tools-$__ApkToolsVersion-x86_64-linux.tar.gz" -P "$__ApkToolsDir" + tar -xf "$__ApkToolsDir/apk-tools-$__ApkToolsVersion-x86_64-linux.tar.gz" -C "$__ApkToolsDir" + mkdir -p "$__RootfsDir"/usr/bin + cp -v "/usr/bin/qemu-$__QEMUArch-static" "$__RootfsDir/usr/bin" - if [[ -n "$__AlpinePackagesEdgeCommunity" ]]; then - $__ApkToolsDir/apk-tools-$__ApkToolsVersion/apk \ - -X http://dl-cdn.alpinelinux.org/alpine/edge/community \ - -U --allow-untrusted --root $__RootfsDir --arch $__AlpineArch --initdb \ - add $__AlpinePackagesEdgeCommunity - fi + "$__ApkToolsDir/apk-tools-$__ApkToolsVersion/apk" \ + -X "http://dl-cdn.alpinelinux.org/alpine/v$__AlpineVersion/main" \ + -X "http://dl-cdn.alpinelinux.org/alpine/v$__AlpineVersion/community" \ + -U --allow-untrusted --root "$__RootfsDir" --arch "$__AlpineArch" --initdb \ + add $__AlpinePackages - rm -r $__ApkToolsDir + rm -r "$__ApkToolsDir" elif [[ "$__CodeName" == "freebsd" ]]; then - mkdir -p $__RootfsDir/usr/local/etc - JOBS="$(getconf _NPROCESSORS_ONLN)" - wget -O - https://download.freebsd.org/ftp/releases/amd64/${__FreeBSDBase}/base.txz | tar -C $__RootfsDir -Jxf - ./lib ./usr/lib ./usr/libdata ./usr/include ./usr/share/keys ./etc ./bin/freebsd-version - echo "ABI = \"FreeBSD:${__FreeBSDABI}:amd64\"; FINGERPRINTS = \"${__RootfsDir}/usr/share/keys\"; REPOS_DIR = [\"${__RootfsDir}/etc/pkg\"]; REPO_AUTOUPDATE = NO; RUN_SCRIPTS = NO;" > ${__RootfsDir}/usr/local/etc/pkg.conf - echo "FreeBSD: { url: "pkg+http://pkg.FreeBSD.org/\${ABI}/quarterly", mirror_type: \"srv\", signature_type: \"fingerprints\", fingerprints: \"${__RootfsDir}/usr/share/keys/pkg\", enabled: yes }" > ${__RootfsDir}/etc/pkg/FreeBSD.conf - mkdir -p $__RootfsDir/tmp + mkdir -p "$__RootfsDir"/usr/local/etc + JOBS=${MAXJOBS:="$(getconf _NPROCESSORS_ONLN)"} + wget -O - "https://download.freebsd.org/ftp/releases/${__FreeBSDArch}/${__FreeBSDMachineArch}/${__FreeBSDBase}/base.txz" | tar -C "$__RootfsDir" -Jxf - ./lib ./usr/lib ./usr/libdata ./usr/include ./usr/share/keys ./etc ./bin/freebsd-version + echo "ABI = \"FreeBSD:${__FreeBSDABI}:${__FreeBSDMachineArch}\"; FINGERPRINTS = \"${__RootfsDir}/usr/share/keys\"; REPOS_DIR = [\"${__RootfsDir}/etc/pkg\"]; REPO_AUTOUPDATE = NO; RUN_SCRIPTS = NO;" > "${__RootfsDir}"/usr/local/etc/pkg.conf + echo "FreeBSD: { url: \"pkg+http://pkg.FreeBSD.org/\${ABI}/quarterly\", mirror_type: \"srv\", signature_type: \"fingerprints\", fingerprints: \"${__RootfsDir}/usr/share/keys/pkg\", enabled: yes }" > "${__RootfsDir}"/etc/pkg/FreeBSD.conf + mkdir -p "$__RootfsDir"/tmp # get and build package manager - wget -O - https://github.com/freebsd/pkg/archive/${__FreeBSDPkg}.tar.gz | tar -C $__RootfsDir/tmp -zxf - - cd $__RootfsDir/tmp/pkg-${__FreeBSDPkg} + wget -O - "https://github.com/freebsd/pkg/archive/${__FreeBSDPkg}.tar.gz" | tar -C "$__RootfsDir"/tmp -zxf - + cd "$__RootfsDir/tmp/pkg-${__FreeBSDPkg}" # needed for install to succeed - mkdir -p $__RootfsDir/host/etc - ./autogen.sh && ./configure --prefix=$__RootfsDir/host && make -j "$JOBS" && make install - rm -rf $__RootfsDir/tmp/pkg-${__FreeBSDPkg} + mkdir -p "$__RootfsDir"/host/etc + ./autogen.sh && ./configure --prefix="$__RootfsDir"/host && make -j "$JOBS" && make install + rm -rf "$__RootfsDir/tmp/pkg-${__FreeBSDPkg}" # install packages we need. - INSTALL_AS_USER=$(whoami) $__RootfsDir/host/sbin/pkg -r $__RootfsDir -C $__RootfsDir/usr/local/etc/pkg.conf update - INSTALL_AS_USER=$(whoami) $__RootfsDir/host/sbin/pkg -r $__RootfsDir -C $__RootfsDir/usr/local/etc/pkg.conf install --yes $__FreeBSDPackages + INSTALL_AS_USER=$(whoami) "$__RootfsDir"/host/sbin/pkg -r "$__RootfsDir" -C "$__RootfsDir"/usr/local/etc/pkg.conf update + INSTALL_AS_USER=$(whoami) "$__RootfsDir"/host/sbin/pkg -r "$__RootfsDir" -C "$__RootfsDir"/usr/local/etc/pkg.conf install --yes $__FreeBSDPackages elif [[ "$__CodeName" == "illumos" ]]; then mkdir "$__RootfsDir/tmp" pushd "$__RootfsDir/tmp" - JOBS="$(getconf _NPROCESSORS_ONLN)" + JOBS=${MAXJOBS:="$(getconf _NPROCESSORS_ONLN)"} echo "Downloading sysroot." wget -O - https://github.com/illumos/sysroot/releases/download/20181213-de6af22ae73b-v1/illumos-sysroot-i386-20181213-de6af22ae73b-v1.tar.gz | tar -C "$__RootfsDir" -xzf - echo "Building binutils. Please wait.." wget -O - https://ftp.gnu.org/gnu/binutils/binutils-2.33.1.tar.bz2 | tar -xjf - mkdir build-binutils && cd build-binutils - ../binutils-2.33.1/configure --prefix="$__RootfsDir" --target="x86_64-sun-solaris2.10" --program-prefix="x86_64-illumos-" --with-sysroot="$__RootfsDir" + ../binutils-2.33.1/configure --prefix="$__RootfsDir" --target="${__illumosArch}-sun-solaris2.10" --program-prefix="${__illumosArch}-illumos-" --with-sysroot="$__RootfsDir" make -j "$JOBS" && make install && cd .. echo "Building gcc. Please wait.." wget -O - https://ftp.gnu.org/gnu/gcc/gcc-8.4.0/gcc-8.4.0.tar.xz | tar -xJf - @@ -330,7 +387,7 @@ elif [[ "$__CodeName" == "illumos" ]]; then CFLAGS_FOR_TARGET="-fPIC" export CFLAGS CXXFLAGS CXXFLAGS_FOR_TARGET CFLAGS_FOR_TARGET mkdir build-gcc && cd build-gcc - ../gcc-8.4.0/configure --prefix="$__RootfsDir" --target="x86_64-sun-solaris2.10" --program-prefix="x86_64-illumos-" --with-sysroot="$__RootfsDir" --with-gnu-as \ + ../gcc-8.4.0/configure --prefix="$__RootfsDir" --target="${__illumosArch}-sun-solaris2.10" --program-prefix="${__illumosArch}-illumos-" --with-sysroot="$__RootfsDir" --with-gnu-as \ --with-gnu-ld --disable-nls --disable-libgomp --disable-libquadmath --disable-libssp --disable-libvtv --disable-libcilkrts --disable-libada --disable-libsanitizer \ --disable-libquadmath-support --disable-shared --enable-tls make -j "$JOBS" && make install && cd .. @@ -338,14 +395,18 @@ elif [[ "$__CodeName" == "illumos" ]]; then if [[ "$__UseMirror" == 1 ]]; then BaseUrl=http://pkgsrc.smartos.skylime.net fi - BaseUrl="$BaseUrl"/packages/SmartOS/2020Q1/x86_64/All + BaseUrl="$BaseUrl/packages/SmartOS/trunk/${__illumosArch}/All" + echo "Downloading manifest" + wget "$BaseUrl" echo "Downloading dependencies." read -ra array <<<"$__IllumosPackages" for package in "${array[@]}"; do - echo "Installing $package..." + echo "Installing '$package'" + package="$(grep ">$package-[0-9]" All | sed -En 's/.*href="(.*)\.tgz".*/\1/p')" + echo "Resolved name '$package'" wget "$BaseUrl"/"$package".tgz ar -x "$package".tgz - tar --skip-old-files -xzf "$package".tmp.tgz -C "$__RootfsDir" 2>/dev/null + tar --skip-old-files -xzf "$package".tmp.tg* -C "$__RootfsDir" 2>/dev/null done echo "Cleaning up temporary files." popd @@ -356,26 +417,90 @@ elif [[ "$__CodeName" == "illumos" ]]; then wget -P "$__RootfsDir"/usr/include/net https://raw.githubusercontent.com/illumos/illumos-gate/master/usr/src/uts/common/io/bpf/net/dlt.h wget -P "$__RootfsDir"/usr/include/netpacket https://raw.githubusercontent.com/illumos/illumos-gate/master/usr/src/uts/common/inet/sockmods/netpacket/packet.h wget -P "$__RootfsDir"/usr/include/sys https://raw.githubusercontent.com/illumos/illumos-gate/master/usr/src/uts/common/sys/sdt.h -elif [[ -n $__CodeName ]]; then - qemu-debootstrap --arch $__UbuntuArch $__CodeName $__RootfsDir $__UbuntuRepo - cp $__CrossDir/$__BuildArch/sources.list.$__CodeName $__RootfsDir/etc/apt/sources.list - chroot $__RootfsDir apt-get update - chroot $__RootfsDir apt-get -f -y install - chroot $__RootfsDir apt-get -y install $__UbuntuPackages - chroot $__RootfsDir symlinks -cr /usr - chroot $__RootfsDir apt-get clean - - if [ $__SkipUnmount == 0 ]; then - umount $__RootfsDir/* || true +elif [[ "$__CodeName" == "haiku" ]]; then + JOBS=${MAXJOBS:="$(getconf _NPROCESSORS_ONLN)"} + + echo "Building Haiku sysroot for x86_64" + mkdir -p "$__RootfsDir/tmp" + cd "$__RootfsDir/tmp" + git clone -b hrev56235 https://review.haiku-os.org/haiku + git clone -b btrev43195 https://review.haiku-os.org/buildtools + cd "$__RootfsDir/tmp/buildtools" && git checkout 7487388f5110021d400b9f3b88e1a7f310dc066d + + # Fetch some unmerged patches + cd "$__RootfsDir/tmp/haiku" + ## Add development build profile (slimmer than nightly) + git fetch origin refs/changes/64/4164/1 && git -c commit.gpgsign=false cherry-pick FETCH_HEAD + + # Build jam + cd "$__RootfsDir/tmp/buildtools/jam" + make + + # Configure cross tools + echo "Building cross-compiler" + mkdir -p "$__RootfsDir/generated" + cd "$__RootfsDir/generated" + "$__RootfsDir/tmp/haiku/configure" -j"$JOBS" --sysroot "$__RootfsDir" --cross-tools-source "$__RootfsDir/tmp/buildtools" --build-cross-tools x86_64 + + # Build Haiku packages + echo "Building Haiku" + echo 'HAIKU_BUILD_PROFILE = "development-raw" ;' > UserProfileConfig + "$__RootfsDir/tmp/buildtools/jam/jam0" -j"$JOBS" -q 'package' 'Haiku' + + BaseUrl="https://depot.haiku-os.org/__api/v2/pkg/get-pkg" + + # Download additional packages + echo "Downloading additional required packages" + read -ra array <<<"$__HaikuPackages" + for package in "${array[@]}"; do + echo "Downloading $package..." + # API documented here: https://github.com/haiku/haikudepotserver/blob/master/haikudepotserver-api2/src/main/resources/api2/pkg.yaml#L60 + # The schema here: https://github.com/haiku/haikudepotserver/blob/master/haikudepotserver-api2/src/main/resources/api2/pkg.yaml#L598 + hpkgDownloadUrl="$(wget -qO- --post-data='{"name":"'"$package"'","repositorySourceCode":"haikuports_x86_64","versionType":"LATEST","naturalLanguageCode":"en"}' \ + --header='Content-Type:application/json' "$BaseUrl" | jq -r '.result.versions[].hpkgDownloadURL')" + wget -P "$__RootfsDir/generated/download" "$hpkgDownloadUrl" + done + + # Setup the sysroot + echo "Setting up sysroot and extracting needed packages" + mkdir -p "$__RootfsDir/boot/system" + for file in "$__RootfsDir/generated/objects/haiku/x86_64/packaging/packages/"*.hpkg; do + "$__RootfsDir/generated/objects/linux/x86_64/release/tools/package/package" extract -C "$__RootfsDir/boot/system" "$file" + done + for file in "$__RootfsDir/generated/download/"*.hpkg; do + "$__RootfsDir/generated/objects/linux/x86_64/release/tools/package/package" extract -C "$__RootfsDir/boot/system" "$file" + done + + # Cleaning up temporary files + echo "Cleaning up temporary files" + rm -rf "$__RootfsDir/tmp" + for name in "$__RootfsDir/generated/"*; do + if [[ "$name" =~ "cross-tools-" ]]; then + : # Keep the cross-compiler + else + rm -rf "$name" + fi + done +elif [[ -n "$__CodeName" ]]; then + qemu-debootstrap $__Keyring --arch "$__UbuntuArch" "$__CodeName" "$__RootfsDir" "$__UbuntuRepo" + cp "$__CrossDir/$__BuildArch/sources.list.$__CodeName" "$__RootfsDir/etc/apt/sources.list" + chroot "$__RootfsDir" apt-get update + chroot "$__RootfsDir" apt-get -f -y install + chroot "$__RootfsDir" apt-get -y install $__UbuntuPackages + chroot "$__RootfsDir" symlinks -cr /usr + chroot "$__RootfsDir" apt-get clean + + if [[ "$__SkipUnmount" == "0" ]]; then + umount "$__RootfsDir"/* || true fi if [[ "$__BuildArch" == "armel" && "$__CodeName" == "jessie" ]]; then - pushd $__RootfsDir - patch -p1 < $__CrossDir/$__BuildArch/armel.jessie.patch + pushd "$__RootfsDir" + patch -p1 < "$__CrossDir/$__BuildArch/armel.jessie.patch" popd fi elif [[ "$__Tizen" == "tizen" ]]; then - ROOTFS_DIR=$__RootfsDir $__CrossDir/$__BuildArch/tizen-build-rootfs.sh + ROOTFS_DIR="$__RootfsDir" "$__CrossDir/$__BuildArch/tizen-build-rootfs.sh" else echo "Unsupported target platform." usage; diff --git a/eng/common/cross/ppc64le/sources.list.bionic b/eng/common/cross/ppc64le/sources.list.bionic new file mode 100644 index 00000000..21095574 --- /dev/null +++ b/eng/common/cross/ppc64le/sources.list.bionic @@ -0,0 +1,11 @@ +deb http://ports.ubuntu.com/ubuntu-ports/ bionic main restricted universe +deb-src http://ports.ubuntu.com/ubuntu-ports/ bionic main restricted universe + +deb http://ports.ubuntu.com/ubuntu-ports/ bionic-updates main restricted universe +deb-src http://ports.ubuntu.com/ubuntu-ports/ bionic-updates main restricted universe + +deb http://ports.ubuntu.com/ubuntu-ports/ bionic-backports main restricted +deb-src http://ports.ubuntu.com/ubuntu-ports/ bionic-backports main restricted + +deb http://ports.ubuntu.com/ubuntu-ports/ bionic-security main restricted universe multiverse +deb-src http://ports.ubuntu.com/ubuntu-ports/ bionic-security main restricted universe multiverse diff --git a/eng/common/cross/riscv64/sources.list.sid b/eng/common/cross/riscv64/sources.list.sid new file mode 100644 index 00000000..65f730d2 --- /dev/null +++ b/eng/common/cross/riscv64/sources.list.sid @@ -0,0 +1 @@ +deb http://deb.debian.org/debian-ports sid main diff --git a/eng/common/cross/toolchain.cmake b/eng/common/cross/toolchain.cmake index ec8971eb..561576be 100644 --- a/eng/common/cross/toolchain.cmake +++ b/eng/common/cross/toolchain.cmake @@ -3,21 +3,25 @@ set(CROSS_ROOTFS $ENV{ROOTFS_DIR}) set(TARGET_ARCH_NAME $ENV{TARGET_BUILD_ARCH}) if(EXISTS ${CROSS_ROOTFS}/bin/freebsd-version) set(CMAKE_SYSTEM_NAME FreeBSD) + set(FREEBSD 1) elseif(EXISTS ${CROSS_ROOTFS}/usr/platform/i86pc) set(CMAKE_SYSTEM_NAME SunOS) set(ILLUMOS 1) +elseif(EXISTS ${CROSS_ROOTFS}/boot/system/develop/headers/config/HaikuConfig.h) + set(CMAKE_SYSTEM_NAME Haiku) else() set(CMAKE_SYSTEM_NAME Linux) + set(LINUX 1) endif() set(CMAKE_SYSTEM_VERSION 1) -if(TARGET_ARCH_NAME STREQUAL "armel") - set(CMAKE_SYSTEM_PROCESSOR armv7l) - set(TOOLCHAIN "arm-linux-gnueabi") - if("$ENV{__DistroRid}" MATCHES "tizen.*") - set(TIZEN_TOOLCHAIN "armv7l-tizen-linux-gnueabi/9.2.0") - endif() -elseif(TARGET_ARCH_NAME STREQUAL "arm") +if(EXISTS ${CROSS_ROOTFS}/etc/tizen-release) + set(TIZEN 1) +elseif(EXISTS ${CROSS_ROOTFS}/android_platform) + set(ANDROID 1) +endif() + +if(TARGET_ARCH_NAME STREQUAL "arm") set(CMAKE_SYSTEM_PROCESSOR armv7l) if(EXISTS ${CROSS_ROOTFS}/usr/lib/gcc/armv7-alpine-linux-musleabihf) set(TOOLCHAIN "armv7-alpine-linux-musleabihf") @@ -26,30 +30,65 @@ elseif(TARGET_ARCH_NAME STREQUAL "arm") else() set(TOOLCHAIN "arm-linux-gnueabihf") endif() + if(TIZEN) + set(TIZEN_TOOLCHAIN "armv7hl-tizen-linux-gnueabihf/9.2.0") + endif() elseif(TARGET_ARCH_NAME STREQUAL "arm64") set(CMAKE_SYSTEM_PROCESSOR aarch64) if(EXISTS ${CROSS_ROOTFS}/usr/lib/gcc/aarch64-alpine-linux-musl) set(TOOLCHAIN "aarch64-alpine-linux-musl") - else() + elseif(LINUX) set(TOOLCHAIN "aarch64-linux-gnu") + if(TIZEN) + set(TIZEN_TOOLCHAIN "aarch64-tizen-linux-gnu/9.2.0") + endif() + elseif(FREEBSD) + set(triple "aarch64-unknown-freebsd12") + endif() +elseif(TARGET_ARCH_NAME STREQUAL "armel") + set(CMAKE_SYSTEM_PROCESSOR armv7l) + set(TOOLCHAIN "arm-linux-gnueabi") + if(TIZEN) + set(TIZEN_TOOLCHAIN "armv7l-tizen-linux-gnueabi/9.2.0") endif() - if("$ENV{__DistroRid}" MATCHES "tizen.*") - set(TIZEN_TOOLCHAIN "aarch64-tizen-linux-gnu/9.2.0") +elseif(TARGET_ARCH_NAME STREQUAL "armv6") + set(CMAKE_SYSTEM_PROCESSOR armv6l) + if(EXISTS ${CROSS_ROOTFS}/usr/lib/gcc/armv6-alpine-linux-musleabihf) + set(TOOLCHAIN "armv6-alpine-linux-musleabihf") + else() + set(TOOLCHAIN "arm-linux-gnueabihf") endif() +elseif(TARGET_ARCH_NAME STREQUAL "ppc64le") + set(CMAKE_SYSTEM_PROCESSOR ppc64le) + set(TOOLCHAIN "powerpc64le-linux-gnu") +elseif(TARGET_ARCH_NAME STREQUAL "riscv64") + set(CMAKE_SYSTEM_PROCESSOR riscv64) + set(TOOLCHAIN "riscv64-linux-gnu") elseif(TARGET_ARCH_NAME STREQUAL "s390x") set(CMAKE_SYSTEM_PROCESSOR s390x) set(TOOLCHAIN "s390x-linux-gnu") +elseif(TARGET_ARCH_NAME STREQUAL "x64") + set(CMAKE_SYSTEM_PROCESSOR x86_64) + if(LINUX) + set(TOOLCHAIN "x86_64-linux-gnu") + if(TIZEN) + set(TIZEN_TOOLCHAIN "x86_64-tizen-linux-gnu/9.2.0") + endif() + elseif(FREEBSD) + set(triple "x86_64-unknown-freebsd12") + elseif(ILLUMOS) + set(TOOLCHAIN "x86_64-illumos") + elseif(HAIKU) + set(TOOLCHAIN "x64_64-unknown-haiku") + endif() elseif(TARGET_ARCH_NAME STREQUAL "x86") set(CMAKE_SYSTEM_PROCESSOR i686) set(TOOLCHAIN "i686-linux-gnu") -elseif (CMAKE_SYSTEM_NAME STREQUAL "FreeBSD") - set(CMAKE_SYSTEM_PROCESSOR "x86_64") - set(triple "x86_64-unknown-freebsd11") -elseif (ILLUMOS) - set(CMAKE_SYSTEM_PROCESSOR "x86_64") - set(TOOLCHAIN "x86_64-illumos") + if(TIZEN) + set(TIZEN_TOOLCHAIN "i586-tizen-linux-gnu/9.2.0") + endif() else() - message(FATAL_ERROR "Arch is ${TARGET_ARCH_NAME}. Only armel, arm, arm64, s390x and x86 are supported!") + message(FATAL_ERROR "Arch is ${TARGET_ARCH_NAME}. Only arm, arm64, armel, armv6, ppc64le, riscv64, s390x, x64 and x86 are supported!") endif() if(DEFINED ENV{TOOLCHAIN}) @@ -57,7 +96,11 @@ if(DEFINED ENV{TOOLCHAIN}) endif() # Specify include paths -if(DEFINED TIZEN_TOOLCHAIN) +if(TIZEN) + if(TARGET_ARCH_NAME STREQUAL "arm") + include_directories(SYSTEM ${CROSS_ROOTFS}/usr/lib/gcc/${TIZEN_TOOLCHAIN}/include/c++/) + include_directories(SYSTEM ${CROSS_ROOTFS}/usr/lib/gcc/${TIZEN_TOOLCHAIN}/include/c++/armv7hl-tizen-linux-gnueabihf) + endif() if(TARGET_ARCH_NAME STREQUAL "armel") include_directories(SYSTEM ${CROSS_ROOTFS}/usr/lib/gcc/${TIZEN_TOOLCHAIN}/include/c++/) include_directories(SYSTEM ${CROSS_ROOTFS}/usr/lib/gcc/${TIZEN_TOOLCHAIN}/include/c++/armv7l-tizen-linux-gnueabi) @@ -66,9 +109,13 @@ if(DEFINED TIZEN_TOOLCHAIN) include_directories(SYSTEM ${CROSS_ROOTFS}/usr/lib64/gcc/${TIZEN_TOOLCHAIN}/include/c++/) include_directories(SYSTEM ${CROSS_ROOTFS}/usr/lib64/gcc/${TIZEN_TOOLCHAIN}/include/c++/aarch64-tizen-linux-gnu) endif() + if(TARGET_ARCH_NAME STREQUAL "x86") + include_directories(SYSTEM ${CROSS_ROOTFS}/usr/lib/gcc/${TIZEN_TOOLCHAIN}/include/c++/) + include_directories(SYSTEM ${CROSS_ROOTFS}/usr/lib/gcc/${TIZEN_TOOLCHAIN}/include/c++/i586-tizen-linux-gnu) + endif() endif() -if("$ENV{__DistroRid}" MATCHES "android.*") +if(ANDROID) if(TARGET_ARCH_NAME STREQUAL "arm") set(ANDROID_ABI armeabi-v7a) elseif(TARGET_ARCH_NAME STREQUAL "arm64") @@ -76,7 +123,9 @@ if("$ENV{__DistroRid}" MATCHES "android.*") endif() # extract platform number required by the NDK's toolchain - string(REGEX REPLACE ".*\\.([0-9]+)-.*" "\\1" ANDROID_PLATFORM "$ENV{__DistroRid}") + file(READ "${CROSS_ROOTFS}/android_platform" RID_FILE_CONTENTS) + string(REPLACE "RID=" "" ANDROID_RID "${RID_FILE_CONTENTS}") + string(REGEX REPLACE ".*\\.([0-9]+)-.*" "\\1" ANDROID_PLATFORM "${ANDROID_RID}") set(ANDROID_TOOLCHAIN clang) set(FEATURE_EVENT_TRACE 0) # disable event trace as there is no lttng-ust package in termux repository @@ -85,12 +134,15 @@ if("$ENV{__DistroRid}" MATCHES "android.*") # include official NDK toolchain script include(${CROSS_ROOTFS}/../build/cmake/android.toolchain.cmake) -elseif(CMAKE_SYSTEM_NAME STREQUAL "FreeBSD") +elseif(FREEBSD) # we cross-compile by instructing clang set(CMAKE_C_COMPILER_TARGET ${triple}) set(CMAKE_CXX_COMPILER_TARGET ${triple}) set(CMAKE_ASM_COMPILER_TARGET ${triple}) set(CMAKE_SYSROOT "${CROSS_ROOTFS}") + set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -fuse-ld=lld") + set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} -fuse-ld=lld") + set(CMAKE_MODULE_LINKER_FLAGS "${CMAKE_MODULE_LINKER_FLAGS} -fuse-ld=lld") elseif(ILLUMOS) set(CMAKE_SYSROOT "${CROSS_ROOTFS}") @@ -122,6 +174,41 @@ elseif(ILLUMOS) set(CMAKE_C_STANDARD_LIBRARIES "${CMAKE_C_STANDARD_LIBRARIES} -lssp") set(CMAKE_CXX_STANDARD_LIBRARIES "${CMAKE_CXX_STANDARD_LIBRARIES} -lssp") +elseif(HAIKU) + set(CMAKE_SYSROOT "${CROSS_ROOTFS}") + + set(TOOLSET_PREFIX ${TOOLCHAIN}-) + function(locate_toolchain_exec exec var) + string(TOUPPER ${exec} EXEC_UPPERCASE) + if(NOT "$ENV{CLR_${EXEC_UPPERCASE}}" STREQUAL "") + set(${var} "$ENV{CLR_${EXEC_UPPERCASE}}" PARENT_SCOPE) + return() + endif() + + set(SEARCH_PATH "${CROSS_ROOTFS}/generated/cross-tools-x86_64/bin") + + find_program(EXEC_LOCATION_${exec} + PATHS ${SEARCH_PATH} + NAMES + "${TOOLSET_PREFIX}${exec}${CLR_CMAKE_COMPILER_FILE_NAME_VERSION}" + "${TOOLSET_PREFIX}${exec}") + + if (EXEC_LOCATION_${exec} STREQUAL "EXEC_LOCATION_${exec}-NOTFOUND") + message(FATAL_ERROR "Unable to find toolchain executable. Name: ${exec}, Prefix: ${TOOLSET_PREFIX}.") + endif() + set(${var} ${EXEC_LOCATION_${exec}} PARENT_SCOPE) + endfunction() + + set(CMAKE_SYSTEM_PREFIX_PATH "${CROSS_ROOTFS}") + + locate_toolchain_exec(gcc CMAKE_C_COMPILER) + locate_toolchain_exec(g++ CMAKE_CXX_COMPILER) + + set(CMAKE_C_STANDARD_LIBRARIES "${CMAKE_C_STANDARD_LIBRARIES} -lssp") + set(CMAKE_CXX_STANDARD_LIBRARIES "${CMAKE_CXX_STANDARD_LIBRARIES} -lssp") + + # let CMake set up the correct search paths + include(Platform/Haiku) else() set(CMAKE_SYSROOT "${CROSS_ROOTFS}") @@ -142,20 +229,20 @@ function(add_toolchain_linker_flag Flag) set("CMAKE_SHARED_LINKER_FLAGS${CONFIG_SUFFIX}_INIT" "${CMAKE_SHARED_LINKER_FLAGS${CONFIG_SUFFIX}_INIT} ${Flag}" PARENT_SCOPE) endfunction() -if(CMAKE_SYSTEM_NAME STREQUAL "Linux") +if(LINUX) add_toolchain_linker_flag("-Wl,--rpath-link=${CROSS_ROOTFS}/lib/${TOOLCHAIN}") add_toolchain_linker_flag("-Wl,--rpath-link=${CROSS_ROOTFS}/usr/lib/${TOOLCHAIN}") endif() -if(TARGET_ARCH_NAME STREQUAL "armel") - if(DEFINED TIZEN_TOOLCHAIN) # For Tizen only +if(TARGET_ARCH_NAME MATCHES "^(arm|armel)$") + if(TIZEN) add_toolchain_linker_flag("-B${CROSS_ROOTFS}/usr/lib/gcc/${TIZEN_TOOLCHAIN}") add_toolchain_linker_flag("-L${CROSS_ROOTFS}/lib") add_toolchain_linker_flag("-L${CROSS_ROOTFS}/usr/lib") add_toolchain_linker_flag("-L${CROSS_ROOTFS}/usr/lib/gcc/${TIZEN_TOOLCHAIN}") endif() elseif(TARGET_ARCH_NAME STREQUAL "arm64") - if(DEFINED TIZEN_TOOLCHAIN) # For Tizen only + if(TIZEN) add_toolchain_linker_flag("-B${CROSS_ROOTFS}/usr/lib64/gcc/${TIZEN_TOOLCHAIN}") add_toolchain_linker_flag("-L${CROSS_ROOTFS}/lib64") add_toolchain_linker_flag("-L${CROSS_ROOTFS}/usr/lib64") @@ -167,6 +254,13 @@ elseif(TARGET_ARCH_NAME STREQUAL "arm64") endif() elseif(TARGET_ARCH_NAME STREQUAL "x86") add_toolchain_linker_flag(-m32) + + if(TIZEN) + add_toolchain_linker_flag("-B${CROSS_ROOTFS}/usr/lib/gcc/${TIZEN_TOOLCHAIN}") + add_toolchain_linker_flag("-L${CROSS_ROOTFS}/lib") + add_toolchain_linker_flag("-L${CROSS_ROOTFS}/usr/lib") + add_toolchain_linker_flag("-L${CROSS_ROOTFS}/usr/lib/gcc/${TIZEN_TOOLCHAIN}") + endif() elseif(ILLUMOS) add_toolchain_linker_flag("-L${CROSS_ROOTFS}/lib/amd64") add_toolchain_linker_flag("-L${CROSS_ROOTFS}/usr/amd64/lib") @@ -174,7 +268,7 @@ endif() # Specify compile options -if((TARGET_ARCH_NAME MATCHES "^(arm|armel|arm64|s390x)$" AND NOT "$ENV{__DistroRid}" MATCHES "android.*") OR ILLUMOS) +if((TARGET_ARCH_NAME MATCHES "^(arm|arm64|armel|armv6|ppc64le|riscv64|s390x)$" AND NOT ANDROID AND NOT FREEBSD) OR ILLUMOS OR HAIKU) set(CMAKE_C_COMPILER_TARGET ${TOOLCHAIN}) set(CMAKE_CXX_COMPILER_TARGET ${TOOLCHAIN}) set(CMAKE_ASM_COMPILER_TARGET ${TOOLCHAIN}) @@ -201,8 +295,8 @@ elseif(TARGET_ARCH_NAME STREQUAL "x86") add_compile_options(-Wno-error=unused-command-line-argument) endif() -if(DEFINED TIZEN_TOOLCHAIN) - if(TARGET_ARCH_NAME MATCHES "^(armel|arm64)$") +if(TIZEN) + if(TARGET_ARCH_NAME MATCHES "^(arm|armel|arm64|x86)$") add_compile_options(-Wno-deprecated-declarations) # compile-time option add_compile_options(-D__extern_always_inline=inline) # compile-time option endif() diff --git a/eng/common/cross/x86/sources.list.focal b/eng/common/cross/x86/sources.list.focal new file mode 100644 index 00000000..99d57313 --- /dev/null +++ b/eng/common/cross/x86/sources.list.focal @@ -0,0 +1,11 @@ +deb http://archive.ubuntu.com/ubuntu/ focal main restricted universe +deb-src http://archive.ubuntu.com/ubuntu/ focal main restricted universe + +deb http://archive.ubuntu.com/ubuntu/ focal-updates main restricted universe +deb-src http://archive.ubuntu.com/ubuntu/ focal-updates main restricted universe + +deb http://archive.ubuntu.com/ubuntu/ focal-backports main restricted +deb-src http://archive.ubuntu.com/ubuntu/ focal-backports main restricted + +deb http://archive.ubuntu.com/ubuntu/ focal-security main restricted universe multiverse +deb-src http://archive.ubuntu.com/ubuntu/ focal-security main restricted universe multiverse diff --git a/eng/common/cross/x86/sources.list.jammy b/eng/common/cross/x86/sources.list.jammy new file mode 100644 index 00000000..af1c1fea --- /dev/null +++ b/eng/common/cross/x86/sources.list.jammy @@ -0,0 +1,11 @@ +deb http://archive.ubuntu.com/ubuntu/ jammy main restricted universe +deb-src http://archive.ubuntu.com/ubuntu/ jammy main restricted universe + +deb http://archive.ubuntu.com/ubuntu/ jammy-updates main restricted universe +deb-src http://archive.ubuntu.com/ubuntu/ jammy-updates main restricted universe + +deb http://archive.ubuntu.com/ubuntu/ jammy-backports main restricted +deb-src http://archive.ubuntu.com/ubuntu/ jammy-backports main restricted + +deb http://archive.ubuntu.com/ubuntu/ jammy-security main restricted universe multiverse +deb-src http://archive.ubuntu.com/ubuntu/ jammy-security main restricted universe multiverse diff --git a/eng/common/cross/x86/tizen-build-rootfs.sh b/eng/common/cross/x86/tizen-build-rootfs.sh new file mode 100644 index 00000000..f5f955dc --- /dev/null +++ b/eng/common/cross/x86/tizen-build-rootfs.sh @@ -0,0 +1,35 @@ +#!/usr/bin/env bash +set -e + +__X86_CrossDir=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd ) +__TIZEN_CROSSDIR="$__X86_CrossDir/tizen" + +if [[ -z "$ROOTFS_DIR" ]]; then + echo "ROOTFS_DIR is not defined." + exit 1; +fi + +TIZEN_TMP_DIR=$ROOTFS_DIR/tizen_tmp +mkdir -p $TIZEN_TMP_DIR + +# Download files +echo ">>Start downloading files" +VERBOSE=1 $__X86_CrossDir/tizen-fetch.sh $TIZEN_TMP_DIR +echo "<>Start constructing Tizen rootfs" +TIZEN_RPM_FILES=`ls $TIZEN_TMP_DIR/*.rpm` +cd $ROOTFS_DIR +for f in $TIZEN_RPM_FILES; do + rpm2cpio $f | cpio -idm --quiet +done +echo "<>Start configuring Tizen rootfs" +ln -sfn asm-x86 ./usr/include/asm +patch -p1 < $__TIZEN_CROSSDIR/tizen.patch +echo "</dev/null; then + VERBOSE=0 +fi + +Log() +{ + if [ $VERBOSE -ge $1 ]; then + echo ${@:2} + fi +} + +Inform() +{ + Log 1 -e "\x1B[0;34m$@\x1B[m" +} + +Debug() +{ + Log 2 -e "\x1B[0;32m$@\x1B[m" +} + +Error() +{ + >&2 Log 0 -e "\x1B[0;31m$@\x1B[m" +} + +Fetch() +{ + URL=$1 + FILE=$2 + PROGRESS=$3 + if [ $VERBOSE -ge 1 ] && [ $PROGRESS ]; then + CURL_OPT="--progress-bar" + else + CURL_OPT="--silent" + fi + curl $CURL_OPT $URL > $FILE +} + +hash curl 2> /dev/null || { Error "Require 'curl' Aborting."; exit 1; } +hash xmllint 2> /dev/null || { Error "Require 'xmllint' Aborting."; exit 1; } +hash sha256sum 2> /dev/null || { Error "Require 'sha256sum' Aborting."; exit 1; } + +TMPDIR=$1 +if [ ! -d $TMPDIR ]; then + TMPDIR=./tizen_tmp + Debug "Create temporary directory : $TMPDIR" + mkdir -p $TMPDIR +fi + +TIZEN_URL=http://download.tizen.org/snapshots/tizen +BUILD_XML=build.xml +REPOMD_XML=repomd.xml +PRIMARY_XML=primary.xml +TARGET_URL="http://__not_initialized" + +Xpath_get() +{ + XPATH_RESULT='' + XPATH=$1 + XML_FILE=$2 + RESULT=$(xmllint --xpath $XPATH $XML_FILE) + if [[ -z ${RESULT// } ]]; then + Error "Can not find target from $XML_FILE" + Debug "Xpath = $XPATH" + exit 1 + fi + XPATH_RESULT=$RESULT +} + +fetch_tizen_pkgs_init() +{ + TARGET=$1 + PROFILE=$2 + Debug "Initialize TARGET=$TARGET, PROFILE=$PROFILE" + + TMP_PKG_DIR=$TMPDIR/tizen_${PROFILE}_pkgs + if [ -d $TMP_PKG_DIR ]; then rm -rf $TMP_PKG_DIR; fi + mkdir -p $TMP_PKG_DIR + + PKG_URL=$TIZEN_URL/$PROFILE/latest + + BUILD_XML_URL=$PKG_URL/$BUILD_XML + TMP_BUILD=$TMP_PKG_DIR/$BUILD_XML + TMP_REPOMD=$TMP_PKG_DIR/$REPOMD_XML + TMP_PRIMARY=$TMP_PKG_DIR/$PRIMARY_XML + TMP_PRIMARYGZ=${TMP_PRIMARY}.gz + + Fetch $BUILD_XML_URL $TMP_BUILD + + Debug "fetch $BUILD_XML_URL to $TMP_BUILD" + + TARGET_XPATH="//build/buildtargets/buildtarget[@name=\"$TARGET\"]/repo[@type=\"binary\"]/text()" + Xpath_get $TARGET_XPATH $TMP_BUILD + TARGET_PATH=$XPATH_RESULT + TARGET_URL=$PKG_URL/$TARGET_PATH + + REPOMD_URL=$TARGET_URL/repodata/repomd.xml + PRIMARY_XPATH='string(//*[local-name()="data"][@type="primary"]/*[local-name()="location"]/@href)' + + Fetch $REPOMD_URL $TMP_REPOMD + + Debug "fetch $REPOMD_URL to $TMP_REPOMD" + + Xpath_get $PRIMARY_XPATH $TMP_REPOMD + PRIMARY_XML_PATH=$XPATH_RESULT + PRIMARY_URL=$TARGET_URL/$PRIMARY_XML_PATH + + Fetch $PRIMARY_URL $TMP_PRIMARYGZ + + Debug "fetch $PRIMARY_URL to $TMP_PRIMARYGZ" + + gunzip $TMP_PRIMARYGZ + + Debug "unzip $TMP_PRIMARYGZ to $TMP_PRIMARY" +} + +fetch_tizen_pkgs() +{ + ARCH=$1 + PACKAGE_XPATH_TPL='string(//*[local-name()="metadata"]/*[local-name()="package"][*[local-name()="name"][text()="_PKG_"]][*[local-name()="arch"][text()="_ARCH_"]]/*[local-name()="location"]/@href)' + + PACKAGE_CHECKSUM_XPATH_TPL='string(//*[local-name()="metadata"]/*[local-name()="package"][*[local-name()="name"][text()="_PKG_"]][*[local-name()="arch"][text()="_ARCH_"]]/*[local-name()="checksum"]/text())' + + for pkg in ${@:2} + do + Inform "Fetching... $pkg" + XPATH=${PACKAGE_XPATH_TPL/_PKG_/$pkg} + XPATH=${XPATH/_ARCH_/$ARCH} + Xpath_get $XPATH $TMP_PRIMARY + PKG_PATH=$XPATH_RESULT + + XPATH=${PACKAGE_CHECKSUM_XPATH_TPL/_PKG_/$pkg} + XPATH=${XPATH/_ARCH_/$ARCH} + Xpath_get $XPATH $TMP_PRIMARY + CHECKSUM=$XPATH_RESULT + + PKG_URL=$TARGET_URL/$PKG_PATH + PKG_FILE=$(basename $PKG_PATH) + PKG_PATH=$TMPDIR/$PKG_FILE + + Debug "Download $PKG_URL to $PKG_PATH" + Fetch $PKG_URL $PKG_PATH true + + echo "$CHECKSUM $PKG_PATH" | sha256sum -c - > /dev/null + if [ $? -ne 0 ]; then + Error "Fail to fetch $PKG_URL to $PKG_PATH" + Debug "Checksum = $CHECKSUM" + exit 1 + fi + done +} + +Inform "Initialize i686 base" +fetch_tizen_pkgs_init standard base +Inform "fetch common packages" +fetch_tizen_pkgs i686 gcc gcc-devel-static glibc glibc-devel libicu libicu-devel libatomic linux-glibc-devel keyutils keyutils-devel libkeyutils +Inform "fetch coreclr packages" +fetch_tizen_pkgs i686 lldb lldb-devel libgcc libstdc++ libstdc++-devel libunwind libunwind-devel lttng-ust-devel lttng-ust userspace-rcu-devel userspace-rcu +Inform "fetch corefx packages" +fetch_tizen_pkgs i686 libcom_err libcom_err-devel zlib zlib-devel libopenssl11 libopenssl1.1-devel krb5 krb5-devel + +Inform "Initialize standard unified" +fetch_tizen_pkgs_init standard unified +Inform "fetch corefx packages" +fetch_tizen_pkgs i686 gssdp gssdp-devel tizen-release + diff --git a/eng/common/cross/x86/tizen/tizen.patch b/eng/common/cross/x86/tizen/tizen.patch new file mode 100644 index 00000000..f4fe8838 --- /dev/null +++ b/eng/common/cross/x86/tizen/tizen.patch @@ -0,0 +1,9 @@ +diff -u -r a/usr/lib/libc.so b/usr/lib/libc.so +--- a/usr/lib/libc.so 2016-12-30 23:00:08.284951863 +0900 ++++ b/usr/lib/libc.so 2016-12-30 23:00:32.140951815 +0900 +@@ -2,4 +2,4 @@ + Use the shared library, but some functions are only in + the static library, so try that secondarily. */ + OUTPUT_FORMAT(elf32-i386) +-GROUP ( /lib/libc.so.6 /usr/lib/libc_nonshared.a AS_NEEDED ( /lib/ld-linux.so.2 ) ) ++GROUP ( libc.so.6 libc_nonshared.a AS_NEEDED ( ld-linux.so.2 ) ) diff --git a/eng/common/darc-init.sh b/eng/common/darc-init.sh index 39abdbec..84c1d0cc 100755 --- a/eng/common/darc-init.sh +++ b/eng/common/darc-init.sh @@ -53,7 +53,7 @@ fi function InstallDarcCli { local darc_cli_package_name="microsoft.dotnet.darc" - InitializeDotNetCli + InitializeDotNetCli true local dotnet_root=$_InitializeDotNetCli if [ -z "$toolpath" ]; then diff --git a/eng/common/dotnet-install.sh b/eng/common/dotnet-install.sh index fdfeea66..abd045a3 100755 --- a/eng/common/dotnet-install.sh +++ b/eng/common/dotnet-install.sh @@ -52,16 +52,19 @@ done # Use uname to determine what the CPU is, see https://en.wikipedia.org/wiki/Uname#Examples cpuname=$(uname -m) case $cpuname in - aarch64) + arm64|aarch64) buildarch=arm64 ;; + loongarch64) + buildarch=loongarch64 + ;; amd64|x86_64) buildarch=x64 ;; armv*l) buildarch=arm ;; - i686) + i[3-6]86) buildarch=x86 ;; *) diff --git a/eng/common/generate-locproject.ps1 b/eng/common/generate-locproject.ps1 index 25e97ac0..bab18543 100644 --- a/eng/common/generate-locproject.ps1 +++ b/eng/common/generate-locproject.ps1 @@ -10,9 +10,7 @@ Param( Set-StrictMode -Version 2.0 $ErrorActionPreference = "Stop" -. $PSScriptRoot\tools.ps1 - -Import-Module -Name (Join-Path $PSScriptRoot 'native\CommonLibrary.psm1') +. $PSScriptRoot\pipeline-logging-functions.ps1 $exclusionsFilePath = "$SourcesDirectory\eng\Localize\LocExclusions.json" $exclusions = @{ Exclusions = @() } @@ -28,13 +26,15 @@ $jsonFiles = @() $jsonTemplateFiles = Get-ChildItem -Recurse -Path "$SourcesDirectory" | Where-Object { $_.FullName -Match "\.template\.config\\localize\\.+\.en\.json" } # .NET templating pattern $jsonTemplateFiles | ForEach-Object { $null = $_.Name -Match "(.+)\.[\w-]+\.json" # matches '[filename].[langcode].json - + $destinationFile = "$($_.Directory.FullName)\$($Matches.1).json" $jsonFiles += Copy-Item "$($_.FullName)" -Destination $destinationFile -PassThru } $jsonWinformsTemplateFiles = Get-ChildItem -Recurse -Path "$SourcesDirectory" | Where-Object { $_.FullName -Match "en\\strings\.json" } # current winforms pattern +$wxlFiles = Get-ChildItem -Recurse -Path "$SourcesDirectory" | Where-Object { $_.FullName -Match "\\.+\.wxl" -And -Not( $_.Directory.Name -Match "\d{4}" ) } # localized files live in four digit lang ID directories; this excludes them + $xlfFiles = @() $allXlfFiles = Get-ChildItem -Recurse -Path "$SourcesDirectory\*\*.xlf" @@ -46,7 +46,7 @@ if ($allXlfFiles) { } $langXlfFiles | ForEach-Object { $null = $_.Name -Match "(.+)\.[\w-]+\.xlf" # matches '[filename].[langcode].xlf - + $destinationFile = "$($_.Directory.FullName)\$($Matches.1).xlf" $xlfFiles += Copy-Item "$($_.FullName)" -Destination $destinationFile -PassThru } @@ -59,10 +59,10 @@ $locJson = @{ LanguageSet = $LanguageSet LocItems = @( $locFiles | ForEach-Object { - $outputPath = "$(($_.DirectoryName | Resolve-Path -Relative) + "\")" + $outputPath = "$(($_.DirectoryName | Resolve-Path -Relative) + "\")" $continue = $true foreach ($exclusion in $exclusions.Exclusions) { - if ($outputPath.Contains($exclusion)) + if ($_.FullName.Contains($exclusion)) { $continue = $false } @@ -79,8 +79,7 @@ $locJson = @{ CopyOption = "LangIDOnPath" OutputPath = "$($_.Directory.Parent.FullName | Resolve-Path -Relative)\" } - } - else { + } else { return @{ SourceFile = $sourceFile CopyOption = "LangIDOnName" @@ -90,6 +89,32 @@ $locJson = @{ } } ) + }, + @{ + CloneLanguageSet = "WiX_CloneLanguages" + LssFiles = @( "wxl_loc.lss" ) + LocItems = @( + $wxlFiles | ForEach-Object { + $outputPath = "$($_.Directory.FullName | Resolve-Path -Relative)\" + $continue = $true + foreach ($exclusion in $exclusions.Exclusions) { + if ($_.FullName.Contains($exclusion)) + { + $continue = $false + } + } + $sourceFile = ($_.FullName | Resolve-Path -Relative) + if ($continue) + { + return @{ + SourceFile = $sourceFile + CopyOption = "LangIDOnPath" + OutputPath = $outputPath + Languages = "cs-CZ;de-DE;es-ES;fr-FR;it-IT;ja-JP;ko-KR;pl-PL;pt-BR;ru-RU;tr-TR;zh-CN;zh-TW" + } + } + } + ) } ) } @@ -108,10 +133,10 @@ else { if ((Get-FileHash "$SourcesDirectory\eng\Localize\LocProject-generated.json").Hash -ne (Get-FileHash "$SourcesDirectory\eng\Localize\LocProject.json").Hash) { Write-PipelineTelemetryError -Category "OneLocBuild" -Message "Existing LocProject.json differs from generated LocProject.json. Download LocProject-generated.json and compare them." - + exit 1 } else { Write-Host "Generated LocProject.json and current LocProject.json are identical." } -} \ No newline at end of file +} diff --git a/eng/common/generate-sbom-prep.ps1 b/eng/common/generate-sbom-prep.ps1 new file mode 100644 index 00000000..3e5c1c74 --- /dev/null +++ b/eng/common/generate-sbom-prep.ps1 @@ -0,0 +1,21 @@ +Param( + [Parameter(Mandatory=$true)][string] $ManifestDirPath # Manifest directory where sbom will be placed +) + +. $PSScriptRoot\pipeline-logging-functions.ps1 + +Write-Host "Creating dir $ManifestDirPath" +# create directory for sbom manifest to be placed +if (!(Test-Path -path $ManifestDirPath)) +{ + New-Item -ItemType Directory -path $ManifestDirPath + Write-Host "Successfully created directory $ManifestDirPath" +} +else{ + Write-PipelineTelemetryError -category 'Build' "Unable to create sbom folder." +} + +Write-Host "Updating artifact name" +$artifact_name = "${env:SYSTEM_STAGENAME}_${env:AGENT_JOBNAME}_SBOM" -replace '["/:<>\\|?@*"() ]', '_' +Write-Host "Artifact name $artifact_name" +Write-Host "##vso[task.setvariable variable=ARTIFACT_NAME]$artifact_name" diff --git a/eng/common/generate-sbom-prep.sh b/eng/common/generate-sbom-prep.sh new file mode 100644 index 00000000..d5c76dc8 --- /dev/null +++ b/eng/common/generate-sbom-prep.sh @@ -0,0 +1,34 @@ +#!/usr/bin/env bash + +source="${BASH_SOURCE[0]}" + +# resolve $SOURCE until the file is no longer a symlink +while [[ -h $source ]]; do + scriptroot="$( cd -P "$( dirname "$source" )" && pwd )" + source="$(readlink "$source")" + + # if $source was a relative symlink, we need to resolve it relative to the path where the + # symlink file was located + [[ $source != /* ]] && source="$scriptroot/$source" +done +scriptroot="$( cd -P "$( dirname "$source" )" && pwd )" +. $scriptroot/pipeline-logging-functions.sh + +manifest_dir=$1 + +if [ ! -d "$manifest_dir" ] ; then + mkdir -p "$manifest_dir" + echo "Sbom directory created." $manifest_dir +else + Write-PipelineTelemetryError -category 'Build' "Unable to create sbom folder." +fi + +artifact_name=$SYSTEM_STAGENAME"_"$AGENT_JOBNAME"_SBOM" +echo "Artifact name before : "$artifact_name +# replace all special characters with _, some builds use special characters like : in Agent.Jobname, that is not a permissible name while uploading artifacts. +safe_artifact_name="${artifact_name//["/:<>\\|?@*$" ]/_}" +echo "Artifact name after : "$safe_artifact_name +export ARTIFACT_NAME=$safe_artifact_name +echo "##vso[task.setvariable variable=ARTIFACT_NAME]$safe_artifact_name" + +exit 0 diff --git a/eng/common/init-tools-native.ps1 b/eng/common/init-tools-native.ps1 index db830c00..fbc67eff 100644 --- a/eng/common/init-tools-native.ps1 +++ b/eng/common/init-tools-native.ps1 @@ -31,6 +31,10 @@ Wait time between retry attempts in seconds .PARAMETER GlobalJsonFile File path to global.json file +.PARAMETER PathPromotion +Optional switch to enable either promote native tools specified in the global.json to the path (in Azure Pipelines) +or break the build if a native tool is not found on the path (on a local dev machine) + .NOTES #> [CmdletBinding(PositionalBinding=$false)] @@ -41,7 +45,8 @@ Param ( [switch] $Force = $False, [int] $DownloadRetries = 5, [int] $RetryWaitTimeInSeconds = 30, - [string] $GlobalJsonFile + [string] $GlobalJsonFile, + [switch] $PathPromotion ) if (!$GlobalJsonFile) { @@ -77,53 +82,102 @@ try { ConvertFrom-Json | Select-Object -Expand 'native-tools' -ErrorAction SilentlyContinue if ($NativeTools) { - $NativeTools.PSObject.Properties | ForEach-Object { - $ToolName = $_.Name - $ToolVersion = $_.Value - $LocalInstallerArguments = @{ ToolName = "$ToolName" } - $LocalInstallerArguments += @{ InstallPath = "$InstallBin" } - $LocalInstallerArguments += @{ BaseUri = "$BaseUri" } - $LocalInstallerArguments += @{ CommonLibraryDirectory = "$EngCommonBaseDir" } - $LocalInstallerArguments += @{ Version = "$ToolVersion" } - - if ($Verbose) { - $LocalInstallerArguments += @{ Verbose = $True } - } - if (Get-Variable 'Force' -ErrorAction 'SilentlyContinue') { - if($Force) { - $LocalInstallerArguments += @{ Force = $True } - } - } - if ($Clean) { - $LocalInstallerArguments += @{ Clean = $True } - } - - Write-Verbose "Installing $ToolName version $ToolVersion" - Write-Verbose "Executing '$InstallerPath $($LocalInstallerArguments.Keys.ForEach({"-$_ '$($LocalInstallerArguments.$_)'"}) -join ' ')'" - & $InstallerPath @LocalInstallerArguments - if ($LASTEXITCODE -Ne "0") { - $errMsg = "$ToolName installation failed" - if ((Get-Variable 'DoNotAbortNativeToolsInstallationOnFailure' -ErrorAction 'SilentlyContinue') -and $DoNotAbortNativeToolsInstallationOnFailure) { - $showNativeToolsWarning = $true - if ((Get-Variable 'DoNotDisplayNativeToolsInstallationWarnings' -ErrorAction 'SilentlyContinue') -and $DoNotDisplayNativeToolsInstallationWarnings) { - $showNativeToolsWarning = $false + if ($PathPromotion -eq $True) { + if ($env:SYSTEM_TEAMPROJECT) { # check to see if we're in an Azure pipelines build + $NativeTools.PSObject.Properties | ForEach-Object { + $ToolName = $_.Name + $ToolVersion = $_.Value + $InstalledTools = @{} + + if ((Get-Command "$ToolName" -ErrorAction SilentlyContinue) -eq $null) { + if ($ToolVersion -eq "latest") { + $ToolVersion = "" + } + $ArcadeToolsDirectory = "C:\arcade-tools" + if (-not (Test-Path $ArcadeToolsDirectory)) { + Write-Error "Arcade tools directory '$ArcadeToolsDirectory' was not found; artifacts were not properly installed." + exit 1 } - if ($showNativeToolsWarning) { - Write-Warning $errMsg + $ToolDirectories = (Get-ChildItem -Path "$ArcadeToolsDirectory" -Filter "$ToolName-$ToolVersion*" | Sort-Object -Descending) + if ($ToolDirectories -eq $null) { + Write-Error "Unable to find directory for $ToolName $ToolVersion; please make sure the tool is installed on this image." + exit 1 } - $toolInstallationFailure = $true - } else { - # We cannot change this to Write-PipelineTelemetryError because of https://github.com/dotnet/arcade/issues/4482 - Write-Host $errMsg - exit 1 + $ToolDirectory = $ToolDirectories[0] + $BinPathFile = "$($ToolDirectory.FullName)\binpath.txt" + if (-not (Test-Path -Path "$BinPathFile")) { + Write-Error "Unable to find binpath.txt in '$($ToolDirectory.FullName)' ($ToolName $ToolVersion); artifact is either installed incorrectly or is not a bootstrappable tool." + exit 1 + } + $BinPath = Get-Content "$BinPathFile" + $ToolPath = Convert-Path -Path $BinPath + Write-Host "Adding $ToolName to the path ($ToolPath)..." + Write-Host "##vso[task.prependpath]$ToolPath" + $env:PATH = "$ToolPath;$env:PATH" + $InstalledTools += @{ $ToolName = $ToolDirectory.FullName } + } + } + return $InstalledTools + } else { + $NativeTools.PSObject.Properties | ForEach-Object { + $ToolName = $_.Name + $ToolVersion = $_.Value + + if ((Get-Command "$ToolName" -ErrorAction SilentlyContinue) -eq $null) { + Write-PipelineTelemetryError -Category 'NativeToolsBootstrap' -Message "$ToolName not found on path. Please install $ToolName $ToolVersion before proceeding." + } } + exit 0 + } + } else { + $NativeTools.PSObject.Properties | ForEach-Object { + $ToolName = $_.Name + $ToolVersion = $_.Value + $LocalInstallerArguments = @{ ToolName = "$ToolName" } + $LocalInstallerArguments += @{ InstallPath = "$InstallBin" } + $LocalInstallerArguments += @{ BaseUri = "$BaseUri" } + $LocalInstallerArguments += @{ CommonLibraryDirectory = "$EngCommonBaseDir" } + $LocalInstallerArguments += @{ Version = "$ToolVersion" } + + if ($Verbose) { + $LocalInstallerArguments += @{ Verbose = $True } + } + if (Get-Variable 'Force' -ErrorAction 'SilentlyContinue') { + if($Force) { + $LocalInstallerArguments += @{ Force = $True } + } + } + if ($Clean) { + $LocalInstallerArguments += @{ Clean = $True } + } + + Write-Verbose "Installing $ToolName version $ToolVersion" + Write-Verbose "Executing '$InstallerPath $($LocalInstallerArguments.Keys.ForEach({"-$_ '$($LocalInstallerArguments.$_)'"}) -join ' ')'" + & $InstallerPath @LocalInstallerArguments + if ($LASTEXITCODE -Ne "0") { + $errMsg = "$ToolName installation failed" + if ((Get-Variable 'DoNotAbortNativeToolsInstallationOnFailure' -ErrorAction 'SilentlyContinue') -and $DoNotAbortNativeToolsInstallationOnFailure) { + $showNativeToolsWarning = $true + if ((Get-Variable 'DoNotDisplayNativeToolsInstallationWarnings' -ErrorAction 'SilentlyContinue') -and $DoNotDisplayNativeToolsInstallationWarnings) { + $showNativeToolsWarning = $false + } + if ($showNativeToolsWarning) { + Write-Warning $errMsg + } + $toolInstallationFailure = $true + } else { + # We cannot change this to Write-PipelineTelemetryError because of https://github.com/dotnet/arcade/issues/4482 + Write-Host $errMsg + exit 1 + } + } + } + + if ((Get-Variable 'toolInstallationFailure' -ErrorAction 'SilentlyContinue') -and $toolInstallationFailure) { + # We cannot change this to Write-PipelineTelemetryError because of https://github.com/dotnet/arcade/issues/4482 + Write-Host 'Native tools bootstrap failed' + exit 1 } - } - - if ((Get-Variable 'toolInstallationFailure' -ErrorAction 'SilentlyContinue') -and $toolInstallationFailure) { - # We cannot change this to Write-PipelineTelemetryError because of https://github.com/dotnet/arcade/issues/4482 - Write-Host 'Native tools bootstrap failed' - exit 1 } } else { @@ -139,7 +193,7 @@ try { Write-Host "##vso[task.prependpath]$(Convert-Path -Path $InstallBin)" return $InstallBin } - else { + elseif (-not ($PathPromotion)) { Write-PipelineTelemetryError -Category 'NativeToolsBootstrap' -Message 'Native tools install directory does not exist, installation failed' exit 1 } diff --git a/eng/common/internal/NuGet.config b/eng/common/internal/NuGet.config new file mode 100644 index 00000000..19d3d311 --- /dev/null +++ b/eng/common/internal/NuGet.config @@ -0,0 +1,7 @@ + + + + + + + diff --git a/eng/common/internal/Tools.csproj b/eng/common/internal/Tools.csproj index beb9c464..7f5ce6d6 100644 --- a/eng/common/internal/Tools.csproj +++ b/eng/common/internal/Tools.csproj @@ -8,6 +8,9 @@ + + + diff --git a/eng/common/msbuild.ps1 b/eng/common/msbuild.ps1 index eea19cd8..f041e5dd 100644 --- a/eng/common/msbuild.ps1 +++ b/eng/common/msbuild.ps1 @@ -6,6 +6,7 @@ Param( [switch] $ci, [switch] $prepareMachine, [switch] $excludePrereleaseVS, + [string] $msbuildEngine = $null, [Parameter(ValueFromRemainingArguments=$true)][String[]]$extraArgs ) diff --git a/eng/common/native/CommonLibrary.psm1 b/eng/common/native/CommonLibrary.psm1 index adf707c8..ca38268c 100644 --- a/eng/common/native/CommonLibrary.psm1 +++ b/eng/common/native/CommonLibrary.psm1 @@ -276,7 +276,8 @@ function Get-MachineArchitecture { } if (($ProcessorArchitecture -Eq "AMD64") -Or ($ProcessorArchitecture -Eq "IA64") -Or - ($ProcessorArchitecture -Eq "ARM64")) { + ($ProcessorArchitecture -Eq "ARM64") -Or + ($ProcessorArchitecture -Eq "LOONGARCH64")) { return "x64" } return "x86" diff --git a/eng/common/native/init-compiler.sh b/eng/common/native/init-compiler.sh new file mode 100644 index 00000000..41a26d80 --- /dev/null +++ b/eng/common/native/init-compiler.sh @@ -0,0 +1,144 @@ +#!/usr/bin/env bash +# +# This file detects the C/C++ compiler and exports it to the CC/CXX environment variables +# +# NOTE: some scripts source this file and rely on stdout being empty, make sure to not output anything here! + +if [[ "$#" -lt 3 ]]; then + echo "Usage..." + echo "init-compiler.sh