diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index 7dee198a..91601a35 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -3,25 +3,25 @@ - + https://github.com/dotnet/arcade - dcc1a4e5315b4f956d228f46999e8135701d8d4f + e9cdc025014eb2030075303211fdb5af254e278c - + https://github.com/dotnet/arcade - dcc1a4e5315b4f956d228f46999e8135701d8d4f + e9cdc025014eb2030075303211fdb5af254e278c - + https://github.com/dotnet/arcade - dcc1a4e5315b4f956d228f46999e8135701d8d4f + e9cdc025014eb2030075303211fdb5af254e278c - + https://github.com/dotnet/arcade - dcc1a4e5315b4f956d228f46999e8135701d8d4f + e9cdc025014eb2030075303211fdb5af254e278c - + https://github.com/dotnet/arcade - dcc1a4e5315b4f956d228f46999e8135701d8d4f + e9cdc025014eb2030075303211fdb5af254e278c https://github.com/dotnet/arcade-services diff --git a/eng/common/generate-graph-files.ps1 b/eng/common/generate-graph-files.ps1 index bc7ad852..0728b1a8 100644 --- a/eng/common/generate-graph-files.ps1 +++ b/eng/common/generate-graph-files.ps1 @@ -83,4 +83,4 @@ catch { ExitWithExitCode 1 } finally { Pop-Location -} +} \ No newline at end of file diff --git a/eng/common/generate-locproject.ps1 b/eng/common/generate-locproject.ps1 new file mode 100644 index 00000000..7225ddc6 --- /dev/null +++ b/eng/common/generate-locproject.ps1 @@ -0,0 +1,101 @@ +Param( + [Parameter(Mandatory=$true)][string] $SourcesDirectory, # Directory where source files live; if using a Localize directory it should live in here + [string] $LanguageSet = 'VS_Main_Languages', # Language set to be used in the LocProject.json + [switch] $UseCheckedInLocProjectJson, # When set, generates a LocProject.json and compares it to one that already exists in the repo; otherwise just generates one + [switch] $CreateNeutralXlfs # Creates neutral xlf files. Only set to false when running locally +) + +# Generates LocProject.json files for the OneLocBuild task. OneLocBuildTask is described here: +# https://ceapex.visualstudio.com/CEINTL/_wiki/wikis/CEINTL.wiki/107/Localization-with-OneLocBuild-Task + +Set-StrictMode -Version 2.0 +$ErrorActionPreference = "Stop" +. $PSScriptRoot\tools.ps1 + +Import-Module -Name (Join-Path $PSScriptRoot 'native\CommonLibrary.psm1') + +$exclusionsFilePath = "$SourcesDirectory\Localize\LocExclusions.json" +$exclusions = @{ Exclusions = @() } +if (Test-Path -Path $exclusionsFilePath) +{ + $exclusions = Get-Content "$exclusionsFilePath" | ConvertFrom-Json +} + +Push-Location "$SourcesDirectory" # push location for Resolve-Path -Relative to work + +# Template files +$jsonFiles = @() +$jsonFiles += Get-ChildItem -Recurse -Path "$SourcesDirectory" | Where-Object { $_.FullName -Match "\.template\.config\\localize\\en\..+\.json" } # .NET templating pattern +$jsonFiles += Get-ChildItem -Recurse -Path "$SourcesDirectory" | Where-Object { $_.FullName -Match "en\\strings\.json" } # current winforms pattern + +$xlfFiles = @() + +$allXlfFiles = Get-ChildItem -Recurse -Path "$SourcesDirectory\*\*.xlf" +$langXlfFiles = @() +if ($allXlfFiles) { + $null = $allXlfFiles[0].FullName -Match "\.([\w-]+)\.xlf" # matches '[langcode].xlf' + $firstLangCode = $Matches.1 + $langXlfFiles = Get-ChildItem -Recurse -Path "$SourcesDirectory\*\*.$firstLangCode.xlf" +} +$langXlfFiles | ForEach-Object { + $null = $_.Name -Match "(.+)\.[\w-]+\.xlf" # matches '[filename].[langcode].xlf + + $destinationFile = "$($_.Directory.FullName)\$($Matches.1).xlf" + $xlfFiles += Copy-Item "$($_.FullName)" -Destination $destinationFile -PassThru +} + +$locFiles = $jsonFiles + $xlfFiles + +$locJson = @{ + Projects = @( + @{ + LanguageSet = $LanguageSet + LocItems = @( + $locFiles | ForEach-Object { + $outputPath = "$(($_.DirectoryName | Resolve-Path -Relative) + "\")" + $continue = $true + foreach ($exclusion in $exclusions.Exclusions) { + if ($outputPath.Contains($exclusion)) + { + $continue = $false + } + } + $sourceFile = ($_.FullName | Resolve-Path -Relative) + if (!$CreateNeutralXlfs -and $_.Extension -eq '.xlf') { + Remove-Item -Path $sourceFile + } + if ($continue) + { + return @{ + SourceFile = $sourceFile + CopyOption = "LangIDOnName" + OutputPath = $outputPath + } + } + } + ) + } + ) +} + +$json = ConvertTo-Json $locJson -Depth 5 +Write-Host "LocProject.json generated:`n`n$json`n`n" +Pop-Location + +if (!$UseCheckedInLocProjectJson) { + New-Item "$SourcesDirectory\Localize\LocProject.json" -Force # Need this to make sure the Localize directory is created + Set-Content "$SourcesDirectory\Localize\LocProject.json" $json +} +else { + New-Item "$SourcesDirectory\Localize\LocProject-generated.json" -Force # Need this to make sure the Localize directory is created + Set-Content "$SourcesDirectory\Localize\LocProject-generated.json" $json + + if ((Get-FileHash "$SourcesDirectory\Localize\LocProject-generated.json").Hash -ne (Get-FileHash "$SourcesDirectory\Localize\LocProject.json").Hash) { + Write-PipelineTelemetryError -Category "OneLocBuild" -Message "Existing LocProject.json differs from generated LocProject.json. Download LocProject-generated.json and compare them." + + exit 1 + } + else { + Write-Host "Generated LocProject.json and current LocProject.json are identical." + } +} \ No newline at end of file diff --git a/eng/common/performance/performance-setup.ps1 b/eng/common/performance/performance-setup.ps1 index 9a64b07e..b6324039 100644 --- a/eng/common/performance/performance-setup.ps1 +++ b/eng/common/performance/performance-setup.ps1 @@ -83,7 +83,7 @@ if ($RunFromPerformanceRepo) { robocopy $SourceDirectory $PerformanceDirectory /E /XD $PayloadDirectory $SourceDirectory\artifacts $SourceDirectory\.git } else { - git clone --branch master --depth 1 --quiet https://github.com/dotnet/performance $PerformanceDirectory + git clone --branch main --depth 1 --quiet https://github.com/dotnet/performance $PerformanceDirectory } if($MonoDotnet -ne "") diff --git a/eng/common/performance/performance-setup.sh b/eng/common/performance/performance-setup.sh index 33b60b50..200cf71f 100755 --- a/eng/common/performance/performance-setup.sh +++ b/eng/common/performance/performance-setup.sh @@ -17,6 +17,7 @@ kind="micro" llvm=false monointerpreter=false monoaot=false +monoaot_path= run_categories="Libraries Runtime" csproj="src\benchmarks\micro\MicroBenchmarks.csproj" configurations="CompliationMode=$compilation_mode RunKind=$kind" @@ -107,7 +108,8 @@ while (($# > 0)); do ;; --monoaot) monoaot=true - shift 1 + monoaot_path=$2 + shift 2 ;; --monodotnet) mono_dotnet=$2 @@ -230,6 +232,11 @@ if [[ "$mono_dotnet" != "" ]] && [[ "$monointerpreter" == "true" ]]; then extra_benchmark_dotnet_arguments="$extra_benchmark_dotnet_arguments --category-exclusion-filter NoInterpreter NoMono" fi +if [[ "$monoaot" == "true" ]]; then + configurations="$configurations LLVM=$llvm MonoInterpreter=$monointerpreter MonoAOT=$monoaot" + extra_benchmark_dotnet_arguments="$extra_benchmark_dotnet_arguments --category-exclusion-filter NoAOT" +fi + common_setup_arguments="--channel master --queue $queue --build-number $build_number --build-configs $configurations --architecture $architecture" setup_arguments="--repository https://github.com/$repository --branch $branch --get-perf-hash --commit-sha $commit_sha $common_setup_arguments" @@ -239,7 +246,7 @@ if [[ "$run_from_perf_repo" = true ]]; then performance_directory=$workitem_directory setup_arguments="--perf-hash $commit_sha $common_setup_arguments" else - git clone --branch master --depth 1 --quiet https://github.com/dotnet/performance $performance_directory + git clone --branch main --depth 1 --quiet https://github.com/dotnet/performance $performance_directory docs_directory=$performance_directory/docs mv $docs_directory $workitem_directory @@ -252,12 +259,18 @@ if [[ "$wasm_runtime_loc" != "" ]]; then extra_benchmark_dotnet_arguments="$extra_benchmark_dotnet_arguments --wasmMainJS \$HELIX_CORRELATION_PAYLOAD/dotnet-wasm/runtime-test.js --wasmEngine /home/helixbot/.jsvu/v8 --customRuntimePack \$HELIX_CORRELATION_PAYLOAD/dotnet-wasm" fi -if [[ "$mono_dotnet" != "" ]]; then +if [[ "$mono_dotnet" != "" ]] && [[ "$monoaot" == "false" ]]; then using_mono=true mono_dotnet_path=$payload_directory/dotnet-mono mv $mono_dotnet $mono_dotnet_path fi +if [[ "$monoaot" == "true" ]]; then + monoaot_dotnet_path=$payload_directory/monoaot + mv $monoaot_path $monoaot_dotnet_path + extra_benchmark_dotnet_arguments="$extra_benchmark_dotnet_arguments --runtimes monoaotllvm --aotcompilerpath \$HELIX_CORRELATION_PAYLOAD/monoaot/sgen/mini/mono-sgen --customruntimepack \$HELIX_CORRELATION_PAYLOAD/monoaot/pack --aotcompilermode llvm" +fi + if [[ "$use_core_run" = true ]]; then new_core_root=$payload_directory/Core_Root mv $core_root_directory $new_core_root diff --git a/eng/common/templates/job/onelocbuild.yml b/eng/common/templates/job/onelocbuild.yml new file mode 100644 index 00000000..928a70cd --- /dev/null +++ b/eng/common/templates/job/onelocbuild.yml @@ -0,0 +1,78 @@ +parameters: + # Optional: dependencies of the job + dependsOn: '' + + # Optional: A defined YAML pool - https://docs.microsoft.com/en-us/azure/devops/pipelines/yaml-schema?view=vsts&tabs=schema#pool + pool: + vmImage: vs2017-win2016 + + CeapexPat: $(dn-bot-ceapex-package-r) # PAT for the loc AzDO instance https://dev.azure.com/ceapex + GithubPat: $(BotAccount-dotnet-bot-repo-PAT) + + SourcesDirectory: $(Build.SourcesDirectory) + CreatePr: true + UseCheckedInLocProjectJson: false + LanguageSet: VS_Main_Languages + LclSource: lclFilesInRepo + LclPackageId: '' + RepoType: gitHub + +jobs: +- job: OneLocBuild + + dependsOn: ${{ parameters.dependsOn }} + + displayName: OneLocBuild + + pool: ${{ parameters.pool }} + + variables: + - group: OneLocBuildVariables # Contains the CeapexPat and GithubPat + - name: _GenerateLocProjectArguments + value: -SourcesDirectory ${{ parameters.SourcesDirectory }} + -LanguageSet "${{ parameters.LanguageSet }}" + -CreateNeutralXlfs + - ${{ if eq(parameters.UseCheckedInLocProjectJson, 'true') }}: + - name: _GenerateLocProjectArguments + value: ${{ variables._GenerateLocProjectArguments }} -UseCheckedInLocProjectJson + + + steps: + - task: Powershell@2 + inputs: + filePath: $(Build.SourcesDirectory)/eng/common/generate-locproject.ps1 + arguments: $(_GenerateLocProjectArguments) + displayName: Generate LocProject.json + + - task: OneLocBuild@2 + displayName: OneLocBuild + env: + SYSTEM_ACCESSTOKEN: $(System.AccessToken) + inputs: + locProj: Localize/LocProject.json + outDir: $(Build.ArtifactStagingDirectory) + lclSource: ${{ parameters.LclSource }} + lclPackageId: ${{ parameters.LclPackageId }} + isCreatePrSelected: ${{ parameters.CreatePr }} + packageSourceAuth: patAuth + patVariable: ${{ parameters.CeapexPat }} + ${{ if eq(parameters.RepoType, 'gitHub') }}: + repoType: ${{ parameters.RepoType }} + gitHubPatVariable: "${{ parameters.GithubPat }}" + condition: always() + + - task: PublishBuildArtifacts@1 + displayName: Publish Localization Files + inputs: + PathtoPublish: '$(Build.ArtifactStagingDirectory)/loc' + PublishLocation: Container + ArtifactName: Loc + condition: always() + + - task: PublishBuildArtifacts@1 + displayName: Publish LocProject.json + inputs: + PathtoPublish: '$(Build.SourcesDirectory)/Localize/' + PublishLocation: Container + ArtifactName: Loc + condition: always() \ No newline at end of file diff --git a/eng/common/templates/steps/perf-send-to-helix.yml b/eng/common/templates/steps/perf-send-to-helix.yml index 3427b311..0fb786fa 100644 --- a/eng/common/templates/steps/perf-send-to-helix.yml +++ b/eng/common/templates/steps/perf-send-to-helix.yml @@ -29,7 +29,7 @@ steps: sendParams: $(Build.SourcesDirectory)/eng/common/performance/${{ parameters.ProjectFile }} /restore /t:Test /bl:$(Build.SourcesDirectory)/artifacts/log/$(_BuildConfig)/SendToHelix.binlog displayName: ${{ parameters.DisplayNamePrefix }} condition: ${{ parameters.condition }} - continueOnError: ${{ parameters.continueOnError }} + shouldContinueOnError: ${{ parameters.continueOnError }} environment: BuildConfig: $(_BuildConfig) HelixSource: ${{ parameters.HelixSource }} diff --git a/eng/common/templates/steps/source-build.yml b/eng/common/templates/steps/source-build.yml index 8e336b7d..65ee5992 100644 --- a/eng/common/templates/steps/source-build.yml +++ b/eng/common/templates/steps/source-build.yml @@ -36,7 +36,7 @@ steps: ${{ coalesce(parameters.platform.buildScript, './build.sh') }} --ci \ --configuration $buildConfig \ - --restore --build --pack --publish \ + --restore --build --pack --publish -bl \ $officialBuildArgs \ $targetRidArgs \ /p:SourceBuildNonPortable=${{ parameters.platform.nonPortable }} \ diff --git a/global.json b/global.json index 1f94c98e..c4ba8140 100644 --- a/global.json +++ b/global.json @@ -1,6 +1,6 @@ { "tools": { - "dotnet": "6.0.100-preview.1.21103.13", + "dotnet": "6.0.100-preview.2.21155.3", "runtimes": { "dotnet": [ "3.1.2" @@ -11,6 +11,6 @@ } }, "msbuild-sdks": { - "Microsoft.DotNet.Arcade.Sdk": "6.0.0-beta.21160.1" + "Microsoft.DotNet.Arcade.Sdk": "6.0.0-beta.21181.6" } }