diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 3abeb529..68665251 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -115,7 +115,7 @@ stages: steps: - checkout: self clean: true - - script: eng/common/cibuild.sh + - script: eng/common/build.sh --restore --build --pack --publish --ci --configuration $(_BuildConfig) --prepareMachine displayName: Build @@ -146,7 +146,7 @@ stages: steps: - checkout: self clean: true - - script: eng/common/cibuild.sh + - script: eng/common/build.sh --restore --build --pack --publish --ci --configuration $(_BuildConfig) --prepareMachine displayName: Build diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index 7dee198a..a2821e0b 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -3,25 +3,25 @@ - + https://github.com/dotnet/arcade - dcc1a4e5315b4f956d228f46999e8135701d8d4f + 46ad27d3cc557f2b84ce30b2c4e27438526dc91d - + https://github.com/dotnet/arcade - dcc1a4e5315b4f956d228f46999e8135701d8d4f + 46ad27d3cc557f2b84ce30b2c4e27438526dc91d - + https://github.com/dotnet/arcade - dcc1a4e5315b4f956d228f46999e8135701d8d4f + 46ad27d3cc557f2b84ce30b2c4e27438526dc91d - + https://github.com/dotnet/arcade - dcc1a4e5315b4f956d228f46999e8135701d8d4f + 46ad27d3cc557f2b84ce30b2c4e27438526dc91d - + https://github.com/dotnet/arcade - dcc1a4e5315b4f956d228f46999e8135701d8d4f + 46ad27d3cc557f2b84ce30b2c4e27438526dc91d https://github.com/dotnet/arcade-services diff --git a/eng/common/cross/build-android-rootfs.sh b/eng/common/cross/build-android-rootfs.sh index 42516bbe..c29c8267 100755 --- a/eng/common/cross/build-android-rootfs.sh +++ b/eng/common/cross/build-android-rootfs.sh @@ -106,6 +106,7 @@ __AndroidPackages+=" libandroid-glob" __AndroidPackages+=" liblzma" __AndroidPackages+=" krb5" __AndroidPackages+=" openssl" +__AndroidPackages+=" openldap" for path in $(wget -qO- http://termux.net/dists/stable/main/binary-$__AndroidArch/Packages |\ grep -A15 "Package: \(${__AndroidPackages// /\\|}\)" | grep -v "static\|tool" | grep Filename); do diff --git a/eng/common/cross/build-rootfs.sh b/eng/common/cross/build-rootfs.sh index b2662244..81e641a5 100755 --- a/eng/common/cross/build-rootfs.sh +++ b/eng/common/cross/build-rootfs.sh @@ -55,11 +55,13 @@ __UbuntuPackages+=" libcurl4-openssl-dev" __UbuntuPackages+=" libkrb5-dev" __UbuntuPackages+=" libssl-dev" __UbuntuPackages+=" zlib1g-dev" +__UbuntuPackages+=" libldap2-dev" __AlpinePackages+=" curl-dev" __AlpinePackages+=" krb5-dev" __AlpinePackages+=" openssl-dev" __AlpinePackages+=" zlib-dev" +__AlpinePackages+=" openldap-dev" __FreeBSDBase="12.1-RELEASE" __FreeBSDPkg="1.12.0" @@ -68,11 +70,13 @@ __FreeBSDPackages+=" icu" __FreeBSDPackages+=" libinotify" __FreeBSDPackages+=" lttng-ust" __FreeBSDPackages+=" krb5" +__FreeBSDPackages+=" libslapi-2.4" __IllumosPackages="icu-64.2nb2" __IllumosPackages+=" mit-krb5-1.16.2nb4" __IllumosPackages+=" openssl-1.1.1e" __IllumosPackages+=" zlib-1.2.11" +__IllumosPackages+=" openldap-client-2.4.49" __UseMirror=0 diff --git a/eng/common/generate-graph-files.ps1 b/eng/common/generate-graph-files.ps1 index bc7ad852..0728b1a8 100644 --- a/eng/common/generate-graph-files.ps1 +++ b/eng/common/generate-graph-files.ps1 @@ -83,4 +83,4 @@ catch { ExitWithExitCode 1 } finally { Pop-Location -} +} \ No newline at end of file diff --git a/eng/common/generate-locproject.ps1 b/eng/common/generate-locproject.ps1 new file mode 100644 index 00000000..24c00b5b --- /dev/null +++ b/eng/common/generate-locproject.ps1 @@ -0,0 +1,110 @@ +Param( + [Parameter(Mandatory=$true)][string] $SourcesDirectory, # Directory where source files live; if using a Localize directory it should live in here + [string] $LanguageSet = 'VS_Main_Languages', # Language set to be used in the LocProject.json + [switch] $UseCheckedInLocProjectJson, # When set, generates a LocProject.json and compares it to one that already exists in the repo; otherwise just generates one + [switch] $CreateNeutralXlfs # Creates neutral xlf files. Only set to false when running locally +) + +# Generates LocProject.json files for the OneLocBuild task. OneLocBuildTask is described here: +# https://ceapex.visualstudio.com/CEINTL/_wiki/wikis/CEINTL.wiki/107/Localization-with-OneLocBuild-Task + +Set-StrictMode -Version 2.0 +$ErrorActionPreference = "Stop" +. $PSScriptRoot\tools.ps1 + +Import-Module -Name (Join-Path $PSScriptRoot 'native\CommonLibrary.psm1') + +$exclusionsFilePath = "$SourcesDirectory\Localize\LocExclusions.json" +$exclusions = @{ Exclusions = @() } +if (Test-Path -Path $exclusionsFilePath) +{ + $exclusions = Get-Content "$exclusionsFilePath" | ConvertFrom-Json +} + +Push-Location "$SourcesDirectory" # push location for Resolve-Path -Relative to work + +# Template files +$jsonFiles = @() +$jsonFiles += Get-ChildItem -Recurse -Path "$SourcesDirectory" | Where-Object { $_.FullName -Match "\.template\.config\\localize\\en\..+\.json" } # .NET templating pattern +$jsonFiles += Get-ChildItem -Recurse -Path "$SourcesDirectory" | Where-Object { $_.FullName -Match "en\\strings\.json" } # current winforms pattern + +$xlfFiles = @() + +$allXlfFiles = Get-ChildItem -Recurse -Path "$SourcesDirectory\*\*.xlf" +$langXlfFiles = @() +if ($allXlfFiles) { + $null = $allXlfFiles[0].FullName -Match "\.([\w-]+)\.xlf" # matches '[langcode].xlf' + $firstLangCode = $Matches.1 + $langXlfFiles = Get-ChildItem -Recurse -Path "$SourcesDirectory\*\*.$firstLangCode.xlf" +} +$langXlfFiles | ForEach-Object { + $null = $_.Name -Match "(.+)\.[\w-]+\.xlf" # matches '[filename].[langcode].xlf + + $destinationFile = "$($_.Directory.FullName)\$($Matches.1).xlf" + $xlfFiles += Copy-Item "$($_.FullName)" -Destination $destinationFile -PassThru +} + +$locFiles = $jsonFiles + $xlfFiles + +$locJson = @{ + Projects = @( + @{ + LanguageSet = $LanguageSet + LocItems = @( + $locFiles | ForEach-Object { + $outputPath = "$(($_.DirectoryName | Resolve-Path -Relative) + "\")" + $continue = $true + foreach ($exclusion in $exclusions.Exclusions) { + if ($outputPath.Contains($exclusion)) + { + $continue = $false + } + } + $sourceFile = ($_.FullName | Resolve-Path -Relative) + if (!$CreateNeutralXlfs -and $_.Extension -eq '.xlf') { + Remove-Item -Path $sourceFile + } + if ($continue) + { + if ($_.Directory.Name -eq 'en' -and $_.Extension -eq '.json') { + return @{ + SourceFile = $sourceFile + CopyOption = "LangIDOnPath" + OutputPath = "$($_.Directory.Parent.FullName | Resolve-Path -Relative)\" + } + } + else { + return @{ + SourceFile = $sourceFile + CopyOption = "LangIDOnName" + OutputPath = $outputPath + } + } + } + } + ) + } + ) +} + +$json = ConvertTo-Json $locJson -Depth 5 +Write-Host "LocProject.json generated:`n`n$json`n`n" +Pop-Location + +if (!$UseCheckedInLocProjectJson) { + New-Item "$SourcesDirectory\Localize\LocProject.json" -Force # Need this to make sure the Localize directory is created + Set-Content "$SourcesDirectory\Localize\LocProject.json" $json +} +else { + New-Item "$SourcesDirectory\Localize\LocProject-generated.json" -Force # Need this to make sure the Localize directory is created + Set-Content "$SourcesDirectory\Localize\LocProject-generated.json" $json + + if ((Get-FileHash "$SourcesDirectory\Localize\LocProject-generated.json").Hash -ne (Get-FileHash "$SourcesDirectory\Localize\LocProject.json").Hash) { + Write-PipelineTelemetryError -Category "OneLocBuild" -Message "Existing LocProject.json differs from generated LocProject.json. Download LocProject-generated.json and compare them." + + exit 1 + } + else { + Write-Host "Generated LocProject.json and current LocProject.json are identical." + } +} \ No newline at end of file diff --git a/eng/common/performance/blazor_perf.proj b/eng/common/performance/blazor_perf.proj deleted file mode 100644 index 3b25359c..00000000 --- a/eng/common/performance/blazor_perf.proj +++ /dev/null @@ -1,30 +0,0 @@ - - - python3 - $(HelixPreCommands);chmod +x $HELIX_WORKITEM_PAYLOAD/SOD/SizeOnDisk - - - - - %(Identity) - - - - - %HELIX_CORRELATION_PAYLOAD%\performance\src\scenarios\ - $(ScenarioDirectory)blazor\ - - - $HELIX_CORRELATION_PAYLOAD/performance/src/scenarios/ - $(ScenarioDirectory)blazor/ - - - - - $(WorkItemDirectory) - cd $(BlazorDirectory);$(Python) pre.py publish --msbuild %27/p:_TrimmerDumpDependencies=true%27 --msbuild-static AdditionalMonoLinkerOptions=%27"%24(AdditionalMonoLinkerOptions) --dump-dependencies"%27 --binlog %27./traces/blazor_publish.binlog%27 - $(Python) test.py sod --scenario-name "%(Identity)" - $(Python) post.py - - - \ No newline at end of file diff --git a/eng/common/performance/crossgen_perf.proj b/eng/common/performance/crossgen_perf.proj deleted file mode 100644 index eb8bdd9c..00000000 --- a/eng/common/performance/crossgen_perf.proj +++ /dev/null @@ -1,110 +0,0 @@ - - - - - %(Identity) - - - - - - py -3 - $(HelixPreCommands) - %HELIX_CORRELATION_PAYLOAD%\Core_Root - %HELIX_CORRELATION_PAYLOAD%\performance\src\scenarios\ - $(ScenarioDirectory)crossgen\ - $(ScenarioDirectory)crossgen2\ - - - python3 - $(HelixPreCommands);chmod +x $HELIX_WORKITEM_PAYLOAD/startup/Startup;chmod +x $HELIX_WORKITEM_PAYLOAD/startup/perfcollect;sudo apt update;chmod +x $HELIX_WORKITEM_PAYLOAD/SOD/SizeOnDisk - $HELIX_CORRELATION_PAYLOAD/Core_Root - $HELIX_CORRELATION_PAYLOAD/performance/src/scenarios/ - $(ScenarioDirectory)crossgen/ - $(ScenarioDirectory)crossgen2/ - - - - - - - - - - - - - - - - - - - - - - - $(WorkItemDirectory) - $(Python) $(CrossgenDirectory)test.py crossgen --core-root $(CoreRoot) --test-name %(Identity) - - - - - - $(WorkItemDirectory) - $(Python) $(Crossgen2Directory)test.py crossgen2 --core-root $(CoreRoot) --single %(Identity) - - - - - - $(WorkItemDirectory) - $(Python) $(Crossgen2Directory)test.py crossgen2 --core-root $(CoreRoot) --single %(Identity) --singlethreaded True - - - - - - $(WorkItemDirectory) - $(Python) $(CrossgenDirectory)pre.py crossgen --core-root $(CoreRoot) --single %(Identity) - $(Python) $(CrossgenDirectory)test.py sod --scenario-name "Crossgen %(Identity) Size" --dirs ./crossgen.out/ - $(Python) $(CrossgenDirectory)post.py - - - - - - $(WorkItemDirectory) - $(Python) $(Crossgen2Directory)pre.py crossgen2 --core-root $(CoreRoot) --single %(Identity) - $(Python) $(Crossgen2Directory)test.py sod --scenario-name "Crossgen2 %(Identity) Size" --dirs ./crossgen.out/ - $(Python) $(Crossgen2Directory)post.py - - - - - - - 4:00 - - - - 4:00 - - - 4:00 - - - $(WorkItemDirectory) - $(Python) $(Crossgen2Directory)test.py crossgen2 --core-root $(CoreRoot) --composite $(Crossgen2Directory)framework-r2r.dll.rsp - 1:00 - - - 4:00 - - - 4:00 - - - \ No newline at end of file diff --git a/eng/common/performance/microbenchmarks.proj b/eng/common/performance/microbenchmarks.proj deleted file mode 100644 index 318ca5f1..00000000 --- a/eng/common/performance/microbenchmarks.proj +++ /dev/null @@ -1,144 +0,0 @@ - - - - %HELIX_CORRELATION_PAYLOAD%\performance\scripts\benchmarks_ci.py --csproj %HELIX_CORRELATION_PAYLOAD%\performance\$(TargetCsproj) - --dotnet-versions %DOTNET_VERSION% --cli-source-info args --cli-branch %PERFLAB_BRANCH% --cli-commit-sha %PERFLAB_HASH% --cli-repository https://github.com/%PERFLAB_REPO% --cli-source-timestamp %PERFLAB_BUILDTIMESTAMP% - py -3 - %HELIX_CORRELATION_PAYLOAD%\Core_Root\CoreRun.exe - %HELIX_CORRELATION_PAYLOAD%\Baseline_Core_Root\CoreRun.exe - - $(HelixPreCommands);call %HELIX_CORRELATION_PAYLOAD%\performance\tools\machine-setup.cmd;set PYTHONPATH=%HELIX_WORKITEM_PAYLOAD%\scripts%3B%HELIX_WORKITEM_PAYLOAD% - %HELIX_CORRELATION_PAYLOAD%\artifacts\BenchmarkDotNet.Artifacts - %HELIX_CORRELATION_PAYLOAD%\artifacts\BenchmarkDotNet.Artifacts_Baseline - %HELIX_CORRELATION_PAYLOAD%\performance\src\tools\ResultsComparer\ResultsComparer.csproj - %HELIX_CORRELATION_PAYLOAD%\performance\tools\dotnet\$(Architecture)\dotnet.exe - %25%25 - %HELIX_WORKITEM_ROOT%\testResults.xml - - - - $HELIX_CORRELATION_PAYLOAD - $(BaseDirectory)/performance - - - - $HELIX_WORKITEM_PAYLOAD - $(BaseDirectory) - - - - $(PerformanceDirectory)/scripts/benchmarks_ci.py --csproj $(PerformanceDirectory)/$(TargetCsproj) - --dotnet-versions $DOTNET_VERSION --cli-source-info args --cli-branch $PERFLAB_BRANCH --cli-commit-sha $PERFLAB_HASH --cli-repository https://github.com/$PERFLAB_REPO --cli-source-timestamp $PERFLAB_BUILDTIMESTAMP - python3 - $(BaseDirectory)/Core_Root/corerun - $(BaseDirectory)/Baseline_Core_Root/corerun - $(HelixPreCommands);chmod +x $(PerformanceDirectory)/tools/machine-setup.sh;. $(PerformanceDirectory)/tools/machine-setup.sh - $(BaseDirectory)/artifacts/BenchmarkDotNet.Artifacts - $(BaseDirectory)/artifacts/BenchmarkDotNet.Artifacts_Baseline - $(PerformanceDirectory)/src/tools/ResultsComparer/ResultsComparer.csproj - $(PerformanceDirectory)/tools/dotnet/$(Architecture)/dotnet - %25 - $HELIX_WORKITEM_ROOT/testResults.xml - - - - $(CliArguments) --wasm - - - - --corerun %HELIX_CORRELATION_PAYLOAD%\dotnet-mono\shared\Microsoft.NETCore.App\6.0.0\corerun.exe - - - --corerun $(BaseDirectory)/dotnet-mono/shared/Microsoft.NETCore.App/6.0.0/corerun - - - - --corerun $(CoreRun) - - - - --corerun $(BaselineCoreRun) - - - - $(Python) $(WorkItemCommand) --incremental no --architecture $(Architecture) -f $(_Framework) $(PerfLabArguments) - - - - $(WorkItemCommand) $(CliArguments) - - - - 2:30 - 0:15 - - - - - %(Identity) - - - - - 30 - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - false - - - - - - $(WorkItemDirectory) - $(WorkItemCommand) --bdn-artifacts $(BaselineArtifactsDirectory) --bdn-arguments="--anyCategories $(BDNCategories) $(ExtraBenchmarkDotNetArguments) $(BaselineCoreRunArgument) --partition-count $(PartitionCount) --partition-index %(HelixWorkItem.Index)" - $(WorkItemCommand) --bdn-artifacts $(ArtifactsDirectory) --bdn-arguments="--anyCategories $(BDNCategories) $(ExtraBenchmarkDotNetArguments) $(CoreRunArgument) --partition-count $(PartitionCount) --partition-index %(HelixWorkItem.Index)" - $(DotnetExe) run -f $(_Framework) -p $(ResultsComparer) --base $(BaselineArtifactsDirectory) --diff $(ArtifactsDirectory) --threshold 2$(Percent) --xml $(XMLResults);$(FinalCommand) - $(WorkItemTimeout) - - - - - - $(WorkItemDirectory) - $(WorkItemCommand) --bdn-artifacts $(BaselineArtifactsDirectory) --bdn-arguments="--anyCategories $(BDNCategories) $(ExtraBenchmarkDotNetArguments) $(BaselineCoreRunArgument)" - $(WorkItemCommand) --bdn-artifacts $(ArtifactsDirectory) --bdn-arguments="--anyCategories $(BDNCategories) $(ExtraBenchmarkDotNetArguments) $(CoreRunArgument)" - $(DotnetExe) run -f $(_Framework) -p $(ResultsComparer) --base $(BaselineArtifactsDirectory) --diff $(ArtifactsDirectory) --threshold 2$(Percent) --xml $(XMLResults) - 4:00 - - - diff --git a/eng/common/performance/performance-setup.ps1 b/eng/common/performance/performance-setup.ps1 deleted file mode 100644 index 9a64b07e..00000000 --- a/eng/common/performance/performance-setup.ps1 +++ /dev/null @@ -1,139 +0,0 @@ -Param( - [string] $SourceDirectory=$env:BUILD_SOURCESDIRECTORY, - [string] $CoreRootDirectory, - [string] $BaselineCoreRootDirectory, - [string] $Architecture="x64", - [string] $Framework="net5.0", - [string] $CompilationMode="Tiered", - [string] $Repository=$env:BUILD_REPOSITORY_NAME, - [string] $Branch=$env:BUILD_SOURCEBRANCH, - [string] $CommitSha=$env:BUILD_SOURCEVERSION, - [string] $BuildNumber=$env:BUILD_BUILDNUMBER, - [string] $RunCategories="Libraries Runtime", - [string] $Csproj="src\benchmarks\micro\MicroBenchmarks.csproj", - [string] $Kind="micro", - [switch] $LLVM, - [switch] $MonoInterpreter, - [switch] $MonoAOT, - [switch] $Internal, - [switch] $Compare, - [string] $MonoDotnet="", - [string] $Configurations="CompilationMode=$CompilationMode RunKind=$Kind", - [string] $LogicalMachine="" -) - -$RunFromPerformanceRepo = ($Repository -eq "dotnet/performance") -or ($Repository -eq "dotnet-performance") -$UseCoreRun = ($CoreRootDirectory -ne [string]::Empty) -$UseBaselineCoreRun = ($BaselineCoreRootDirectory -ne [string]::Empty) - -$PayloadDirectory = (Join-Path $SourceDirectory "Payload") -$PerformanceDirectory = (Join-Path $PayloadDirectory "performance") -$WorkItemDirectory = (Join-Path $SourceDirectory "workitem") -$ExtraBenchmarkDotNetArguments = "--iterationCount 1 --warmupCount 0 --invocationCount 1 --unrollFactor 1 --strategy ColdStart --stopOnFirstError true" -$Creator = $env:BUILD_DEFINITIONNAME -$PerfLabArguments = "" -$HelixSourcePrefix = "pr" - -$Queue = "" - -if ($Internal) { - switch ($LogicalMachine) { - "perftiger" { $Queue = "Windows.10.Amd64.19H1.Tiger.Perf" } - "perfowl" { $Queue = "Windows.10.Amd64.20H2.Owl.Perf" } - "perfsurf" { $Queue = "Windows.10.Arm64.Perf.Surf" } - Default { $Queue = "Windows.10.Amd64.19H1.Tiger.Perf" } - } - $PerfLabArguments = "--upload-to-perflab-container" - $ExtraBenchmarkDotNetArguments = "" - $Creator = "" - $HelixSourcePrefix = "official" -} -else { - $Queue = "Windows.10.Amd64.ClientRS4.DevEx.15.8.Open" -} - -if($MonoInterpreter) -{ - $ExtraBenchmarkDotNetArguments = "--category-exclusion-filter NoInterpreter" -} - -if($MonoDotnet -ne "") -{ - $Configurations += " LLVM=$LLVM MonoInterpreter=$MonoInterpreter MonoAOT=$MonoAOT" - if($ExtraBenchmarkDotNetArguments -eq "") - { - #FIX ME: We need to block these tests as they don't run on mono for now - $ExtraBenchmarkDotNetArguments = "--exclusion-filter *Perf_Image* *Perf_NamedPipeStream*" - } - else - { - #FIX ME: We need to block these tests as they don't run on mono for now - $ExtraBenchmarkDotNetArguments += " --exclusion-filter *Perf_Image* *Perf_NamedPipeStream*" - } -} - -# FIX ME: This is a workaround until we get this from the actual pipeline -$CommonSetupArguments="--channel master --queue $Queue --build-number $BuildNumber --build-configs $Configurations --architecture $Architecture" -$SetupArguments = "--repository https://github.com/$Repository --branch $Branch --get-perf-hash --commit-sha $CommitSha $CommonSetupArguments" - - -if ($RunFromPerformanceRepo) { - $SetupArguments = "--perf-hash $CommitSha $CommonSetupArguments" - - robocopy $SourceDirectory $PerformanceDirectory /E /XD $PayloadDirectory $SourceDirectory\artifacts $SourceDirectory\.git -} -else { - git clone --branch master --depth 1 --quiet https://github.com/dotnet/performance $PerformanceDirectory -} - -if($MonoDotnet -ne "") -{ - $UsingMono = "true" - $MonoDotnetPath = (Join-Path $PayloadDirectory "dotnet-mono") - Move-Item -Path $MonoDotnet -Destination $MonoDotnetPath -} - -if ($UseCoreRun) { - $NewCoreRoot = (Join-Path $PayloadDirectory "Core_Root") - Move-Item -Path $CoreRootDirectory -Destination $NewCoreRoot -} -if ($UseBaselineCoreRun) { - $NewBaselineCoreRoot = (Join-Path $PayloadDirectory "Baseline_Core_Root") - Move-Item -Path $BaselineCoreRootDirectory -Destination $NewBaselineCoreRoot -} - -$DocsDir = (Join-Path $PerformanceDirectory "docs") -robocopy $DocsDir $WorkItemDirectory - -# Set variables that we will need to have in future steps -$ci = $true - -. "$PSScriptRoot\..\pipeline-logging-functions.ps1" - -# Directories -Write-PipelineSetVariable -Name 'PayloadDirectory' -Value "$PayloadDirectory" -IsMultiJobVariable $false -Write-PipelineSetVariable -Name 'PerformanceDirectory' -Value "$PerformanceDirectory" -IsMultiJobVariable $false -Write-PipelineSetVariable -Name 'WorkItemDirectory' -Value "$WorkItemDirectory" -IsMultiJobVariable $false - -# Script Arguments -Write-PipelineSetVariable -Name 'Python' -Value "py -3" -IsMultiJobVariable $false -Write-PipelineSetVariable -Name 'ExtraBenchmarkDotNetArguments' -Value "$ExtraBenchmarkDotNetArguments" -IsMultiJobVariable $false -Write-PipelineSetVariable -Name 'SetupArguments' -Value "$SetupArguments" -IsMultiJobVariable $false -Write-PipelineSetVariable -Name 'PerfLabArguments' -Value "$PerfLabArguments" -IsMultiJobVariable $false -Write-PipelineSetVariable -Name 'BDNCategories' -Value "$RunCategories" -IsMultiJobVariable $false -Write-PipelineSetVariable -Name 'TargetCsproj' -Value "$Csproj" -IsMultiJobVariable $false -Write-PipelineSetVariable -Name 'Kind' -Value "$Kind" -IsMultiJobVariable $false -Write-PipelineSetVariable -Name 'Architecture' -Value "$Architecture" -IsMultiJobVariable $false -Write-PipelineSetVariable -Name 'UseCoreRun' -Value "$UseCoreRun" -IsMultiJobVariable $false -Write-PipelineSetVariable -Name 'UseBaselineCoreRun' -Value "$UseBaselineCoreRun" -IsMultiJobVariable $false -Write-PipelineSetVariable -Name 'RunFromPerfRepo' -Value "$RunFromPerformanceRepo" -IsMultiJobVariable $false -Write-PipelineSetVariable -Name 'Compare' -Value "$Compare" -IsMultiJobVariable $false -Write-PipelineSetVariable -Name 'MonoDotnet' -Value "$UsingMono" -IsMultiJobVariable $false - -# Helix Arguments -Write-PipelineSetVariable -Name 'Creator' -Value "$Creator" -IsMultiJobVariable $false -Write-PipelineSetVariable -Name 'Queue' -Value "$Queue" -IsMultiJobVariable $false -Write-PipelineSetVariable -Name 'HelixSourcePrefix' -Value "$HelixSourcePrefix" -IsMultiJobVariable $false -Write-PipelineSetVariable -Name '_BuildConfig' -Value "$Architecture.$Kind.$Framework" -IsMultiJobVariable $false - -exit 0 \ No newline at end of file diff --git a/eng/common/performance/performance-setup.sh b/eng/common/performance/performance-setup.sh deleted file mode 100755 index 33b60b50..00000000 --- a/eng/common/performance/performance-setup.sh +++ /dev/null @@ -1,297 +0,0 @@ -#!/usr/bin/env bash - -source_directory=$BUILD_SOURCESDIRECTORY -core_root_directory= -baseline_core_root_directory= -architecture=x64 -framework=net5.0 -compilation_mode=tiered -repository=$BUILD_REPOSITORY_NAME -branch=$BUILD_SOURCEBRANCH -commit_sha=$BUILD_SOURCEVERSION -build_number=$BUILD_BUILDNUMBER -internal=false -compare=false -mono_dotnet= -kind="micro" -llvm=false -monointerpreter=false -monoaot=false -run_categories="Libraries Runtime" -csproj="src\benchmarks\micro\MicroBenchmarks.csproj" -configurations="CompliationMode=$compilation_mode RunKind=$kind" -run_from_perf_repo=false -use_core_run=true -use_baseline_core_run=true -using_mono=false -wasm_runtime_loc= -using_wasm=false -use_latest_dotnet=false -logical_machine= - -while (($# > 0)); do - lowerI="$(echo $1 | tr "[:upper:]" "[:lower:]")" - case $lowerI in - --sourcedirectory) - source_directory=$2 - shift 2 - ;; - --corerootdirectory) - core_root_directory=$2 - shift 2 - ;; - --baselinecorerootdirectory) - baseline_core_root_directory=$2 - shift 2 - ;; - --architecture) - architecture=$2 - shift 2 - ;; - --framework) - framework=$2 - shift 2 - ;; - --compilationmode) - compilation_mode=$2 - shift 2 - ;; - --logicalmachine) - logical_machine=$2 - shift 2 - ;; - --repository) - repository=$2 - shift 2 - ;; - --branch) - branch=$2 - shift 2 - ;; - --commitsha) - commit_sha=$2 - shift 2 - ;; - --buildnumber) - build_number=$2 - shift 2 - ;; - --kind) - kind=$2 - configurations="CompilationMode=$compilation_mode RunKind=$kind" - shift 2 - ;; - --runcategories) - run_categories=$2 - shift 2 - ;; - --csproj) - csproj=$2 - shift 2 - ;; - --internal) - internal=true - shift 1 - ;; - --alpine) - alpine=true - shift 1 - ;; - --llvm) - llvm=true - shift 1 - ;; - --monointerpreter) - monointerpreter=true - shift 1 - ;; - --monoaot) - monoaot=true - shift 1 - ;; - --monodotnet) - mono_dotnet=$2 - shift 2 - ;; - --wasm) - wasm_runtime_loc=$2 - shift 2 - ;; - --compare) - compare=true - shift 1 - ;; - --configurations) - configurations=$2 - shift 2 - ;; - --latestdotnet) - use_latest_dotnet=true - shift 1 - ;; - *) - echo "Common settings:" - echo " --corerootdirectory Directory where Core_Root exists, if running perf testing with --corerun" - echo " --architecture Architecture of the testing being run" - echo " --configurations List of key=value pairs that will be passed to perf testing infrastructure." - echo " ex: --configurations \"CompilationMode=Tiered OptimzationLevel=PGO\"" - echo " --help Print help and exit" - echo "" - echo "Advanced settings:" - echo " --framework The framework to run, if not running in master" - echo " --compliationmode The compilation mode if not passing --configurations" - echo " --sourcedirectory The directory of the sources. Defaults to env:BUILD_SOURCESDIRECTORY" - echo " --repository The name of the repository in the / format. Defaults to env:BUILD_REPOSITORY_NAME" - echo " --branch The name of the branch. Defaults to env:BUILD_SOURCEBRANCH" - echo " --commitsha The commit sha1 to run against. Defaults to env:BUILD_SOURCEVERSION" - echo " --buildnumber The build number currently running. Defaults to env:BUILD_BUILDNUMBER" - echo " --csproj The relative path to the benchmark csproj whose tests should be run. Defaults to src\benchmarks\micro\MicroBenchmarks.csproj" - echo " --kind Related to csproj. The kind of benchmarks that should be run. Defaults to micro" - echo " --runcategories Related to csproj. Categories of benchmarks to run. Defaults to \"coreclr corefx\"" - echo " --internal If the benchmarks are running as an official job." - echo " --monodotnet Pass the path to the mono dotnet for mono performance testing." - echo " --wasm Path to the unpacked wasm runtime pack." - echo " --latestdotnet --dotnet-versions will not be specified. --dotnet-versions defaults to LKG version in global.json " - echo " --alpine Set for runs on Alpine" - echo "" - exit 0 - ;; - esac -done - -if [ "$repository" == "dotnet/performance" ] || [ "$repository" == "dotnet-performance" ]; then - run_from_perf_repo=true -fi - -if [ -z "$configurations" ]; then - configurations="CompilationMode=$compilation_mode" -fi - -if [ -z "$core_root_directory" ]; then - use_core_run=false -fi - -if [ -z "$baseline_core_root_directory" ]; then - use_baseline_core_run=false -fi - -payload_directory=$source_directory/Payload -performance_directory=$payload_directory/performance -workitem_directory=$source_directory/workitem -extra_benchmark_dotnet_arguments="--iterationCount 1 --warmupCount 0 --invocationCount 1 --unrollFactor 1 --strategy ColdStart --stopOnFirstError true" -perflab_arguments= -queue=Ubuntu.1804.Amd64.Open -creator=$BUILD_DEFINITIONNAME -helix_source_prefix="pr" - -if [[ "$internal" == true ]]; then - perflab_arguments="--upload-to-perflab-container" - helix_source_prefix="official" - creator= - extra_benchmark_dotnet_arguments= - - if [[ "$architecture" = "arm64" ]]; then - queue=Ubuntu.1804.Arm64.Perf - else - if [[ "$logical_machine" = "perfowl" ]]; then - queue=Ubuntu.1804.Amd64.Owl.Perf - else - queue=Ubuntu.1804.Amd64.Tiger.Perf - fi - fi - - if [[ "$alpine" = "true" ]]; then - queue=alpine.amd64.tiger.perf - fi -else - if [[ "$architecture" = "arm64" ]]; then - queue=ubuntu.1804.armarch.open - else - queue=Ubuntu.1804.Amd64.Open - fi - - if [[ "$alpine" = "true" ]]; then - queue=alpine.amd64.tiger.perf - fi -fi - -if [[ "$mono_dotnet" != "" ]] && [[ "$monointerpreter" == "false" ]]; then - configurations="$configurations LLVM=$llvm MonoInterpreter=$monointerpreter MonoAOT=$monoaot" - extra_benchmark_dotnet_arguments="$extra_benchmark_dotnet_arguments --category-exclusion-filter NoMono" -fi - -if [[ "$wasm_runtime_loc" != "" ]]; then - configurations="CompilationMode=wasm RunKind=$kind" - extra_benchmark_dotnet_arguments="$extra_benchmark_dotnet_arguments --category-exclusion-filter NoInterpreter NoWASM NoMono" -fi - -if [[ "$mono_dotnet" != "" ]] && [[ "$monointerpreter" == "true" ]]; then - configurations="$configurations LLVM=$llvm MonoInterpreter=$monointerpreter MonoAOT=$monoaot" - extra_benchmark_dotnet_arguments="$extra_benchmark_dotnet_arguments --category-exclusion-filter NoInterpreter NoMono" -fi - -common_setup_arguments="--channel master --queue $queue --build-number $build_number --build-configs $configurations --architecture $architecture" -setup_arguments="--repository https://github.com/$repository --branch $branch --get-perf-hash --commit-sha $commit_sha $common_setup_arguments" - -if [[ "$run_from_perf_repo" = true ]]; then - payload_directory= - workitem_directory=$source_directory - performance_directory=$workitem_directory - setup_arguments="--perf-hash $commit_sha $common_setup_arguments" -else - git clone --branch master --depth 1 --quiet https://github.com/dotnet/performance $performance_directory - - docs_directory=$performance_directory/docs - mv $docs_directory $workitem_directory -fi - -if [[ "$wasm_runtime_loc" != "" ]]; then - using_wasm=true - wasm_dotnet_path=$payload_directory/dotnet-wasm - mv $wasm_runtime_loc $wasm_dotnet_path - extra_benchmark_dotnet_arguments="$extra_benchmark_dotnet_arguments --wasmMainJS \$HELIX_CORRELATION_PAYLOAD/dotnet-wasm/runtime-test.js --wasmEngine /home/helixbot/.jsvu/v8 --customRuntimePack \$HELIX_CORRELATION_PAYLOAD/dotnet-wasm" -fi - -if [[ "$mono_dotnet" != "" ]]; then - using_mono=true - mono_dotnet_path=$payload_directory/dotnet-mono - mv $mono_dotnet $mono_dotnet_path -fi - -if [[ "$use_core_run" = true ]]; then - new_core_root=$payload_directory/Core_Root - mv $core_root_directory $new_core_root -fi - -if [[ "$use_baseline_core_run" = true ]]; then - new_baseline_core_root=$payload_directory/Baseline_Core_Root - mv $baseline_core_root_directory $new_baseline_core_root -fi - -ci=true - -_script_dir=$(pwd)/eng/common -. "$_script_dir/pipeline-logging-functions.sh" - -# Make sure all of our variables are available for future steps -Write-PipelineSetVariable -name "UseCoreRun" -value "$use_core_run" -is_multi_job_variable false -Write-PipelineSetVariable -name "UseBaselineCoreRun" -value "$use_baseline_core_run" -is_multi_job_variable false -Write-PipelineSetVariable -name "Architecture" -value "$architecture" -is_multi_job_variable false -Write-PipelineSetVariable -name "PayloadDirectory" -value "$payload_directory" -is_multi_job_variable false -Write-PipelineSetVariable -name "PerformanceDirectory" -value "$performance_directory" -is_multi_job_variable false -Write-PipelineSetVariable -name "WorkItemDirectory" -value "$workitem_directory" -is_multi_job_variable false -Write-PipelineSetVariable -name "Queue" -value "$queue" -is_multi_job_variable false -Write-PipelineSetVariable -name "SetupArguments" -value "$setup_arguments" -is_multi_job_variable false -Write-PipelineSetVariable -name "Python" -value "python3" -is_multi_job_variable false -Write-PipelineSetVariable -name "PerfLabArguments" -value "$perflab_arguments" -is_multi_job_variable false -Write-PipelineSetVariable -name "ExtraBenchmarkDotNetArguments" -value "$extra_benchmark_dotnet_arguments" -is_multi_job_variable false -Write-PipelineSetVariable -name "BDNCategories" -value "$run_categories" -is_multi_job_variable false -Write-PipelineSetVariable -name "TargetCsproj" -value "$csproj" -is_multi_job_variable false -Write-PipelineSetVariable -name "RunFromPerfRepo" -value "$run_from_perf_repo" -is_multi_job_variable false -Write-PipelineSetVariable -name "Creator" -value "$creator" -is_multi_job_variable false -Write-PipelineSetVariable -name "HelixSourcePrefix" -value "$helix_source_prefix" -is_multi_job_variable false -Write-PipelineSetVariable -name "Kind" -value "$kind" -is_multi_job_variable false -Write-PipelineSetVariable -name "_BuildConfig" -value "$architecture.$kind.$framework" -is_multi_job_variable false -Write-PipelineSetVariable -name "Compare" -value "$compare" -is_multi_job_variable false -Write-PipelineSetVariable -name "MonoDotnet" -value "$using_mono" -is_multi_job_variable false -Write-PipelineSetVariable -name "WasmDotnet" -value "$using_wasm" -is_multi_job_variable false diff --git a/eng/common/post-build/symbols-validation.ps1 b/eng/common/post-build/symbols-validation.ps1 index 99bf28cd..e6d5d2fd 100644 --- a/eng/common/post-build/symbols-validation.ps1 +++ b/eng/common/post-build/symbols-validation.ps1 @@ -1,9 +1,10 @@ param( - [Parameter(Mandatory=$true)][string] $InputPath, # Full path to directory where NuGet packages to be checked are stored - [Parameter(Mandatory=$true)][string] $ExtractPath, # Full path to directory where the packages will be extracted during validation - [Parameter(Mandatory=$true)][string] $DotnetSymbolVersion, # Version of dotnet symbol to use - [Parameter(Mandatory=$false)][switch] $ContinueOnError, # If we should keep checking symbols after an error - [Parameter(Mandatory=$false)][switch] $Clean # Clean extracted symbols directory after checking symbols + [Parameter(Mandatory = $true)][string] $InputPath, # Full path to directory where NuGet packages to be checked are stored + [Parameter(Mandatory = $true)][string] $ExtractPath, # Full path to directory where the packages will be extracted during validation + [Parameter(Mandatory = $true)][string] $DotnetSymbolVersion, # Version of dotnet symbol to use + [Parameter(Mandatory = $false)][switch] $CheckForWindowsPdbs, # If we should check for the existence of windows pdbs in addition to portable PDBs + [Parameter(Mandatory = $false)][switch] $ContinueOnError, # If we should keep checking symbols after an error + [Parameter(Mandatory = $false)][switch] $Clean # Clean extracted symbols directory after checking symbols ) # Maximum number of jobs to run in parallel @@ -19,9 +20,15 @@ $SecondsBetweenLoadChecks = 10 Set-Variable -Name "ERROR_BADEXTRACT" -Option Constant -Value -1 Set-Variable -Name "ERROR_FILEDOESNOTEXIST" -Option Constant -Value -2 +$WindowsPdbVerificationParam = "" +if ($CheckForWindowsPdbs) { + $WindowsPdbVerificationParam = "--windows-pdbs" +} + $CountMissingSymbols = { param( - [string] $PackagePath # Path to a NuGet package + [string] $PackagePath, # Path to a NuGet package + [string] $WindowsPdbVerificationParam # If we should check for the existence of windows pdbs in addition to portable PDBs ) . $using:PSScriptRoot\..\tools.ps1 @@ -34,7 +41,7 @@ $CountMissingSymbols = { if (!(Test-Path $PackagePath)) { Write-PipelineTaskError "Input file does not exist: $PackagePath" return [pscustomobject]@{ - result = $using:ERROR_FILEDOESNOTEXIST + result = $using:ERROR_FILEDOESNOTEXIST packagePath = $PackagePath } } @@ -57,24 +64,25 @@ $CountMissingSymbols = { Write-Host "Something went wrong extracting $PackagePath" Write-Host $_ return [pscustomobject]@{ - result = $using:ERROR_BADEXTRACT + result = $using:ERROR_BADEXTRACT packagePath = $PackagePath } } Get-ChildItem -Recurse $ExtractPath | - Where-Object {$RelevantExtensions -contains $_.Extension} | - ForEach-Object { - $FileName = $_.FullName - if ($FileName -Match '\\ref\\') { - Write-Host "`t Ignoring reference assembly file " $FileName - return - } + Where-Object { $RelevantExtensions -contains $_.Extension } | + ForEach-Object { + $FileName = $_.FullName + if ($FileName -Match '\\ref\\') { + Write-Host "`t Ignoring reference assembly file " $FileName + return + } - $FirstMatchingSymbolDescriptionOrDefault = { + $FirstMatchingSymbolDescriptionOrDefault = { param( - [string] $FullPath, # Full path to the module that has to be checked - [string] $TargetServerParam, # Parameter to pass to `Symbol Tool` indicating the server to lookup for symbols + [string] $FullPath, # Full path to the module that has to be checked + [string] $TargetServerParam, # Parameter to pass to `Symbol Tool` indicating the server to lookup for symbols + [string] $WindowsPdbVerificationParam, # Parameter to pass to potential check for windows-pdbs. [string] $SymbolsPath ) @@ -99,7 +107,7 @@ $CountMissingSymbols = { # DWARF file for a .dylib $DylibDwarf = $SymbolPath.Replace($Extension, '.dylib.dwarf') - + $dotnetSymbolExe = "$env:USERPROFILE\.dotnet\tools" $dotnetSymbolExe = Resolve-Path "$dotnetSymbolExe\dotnet-symbol.exe" @@ -107,7 +115,7 @@ $CountMissingSymbols = { while ($totalRetries -lt $using:MaxRetry) { # Save the output and get diagnostic output - $output = & $dotnetSymbolExe --symbols --modules --windows-pdbs $TargetServerParam $FullPath -o $SymbolsPath --diagnostics | Out-String + $output = & $dotnetSymbolExe --symbols --modules $WindowsPdbVerificationParam $TargetServerParam $FullPath -o $SymbolsPath --diagnostics | Out-String if (Test-Path $PdbPath) { return 'PDB' @@ -136,30 +144,30 @@ $CountMissingSymbols = { return $null } - $SymbolsOnMSDL = & $FirstMatchingSymbolDescriptionOrDefault $FileName '--microsoft-symbol-server' $SymbolsPath - $SymbolsOnSymWeb = & $FirstMatchingSymbolDescriptionOrDefault $FileName '--internal-server' $SymbolsPath + $SymbolsOnMSDL = & $FirstMatchingSymbolDescriptionOrDefault $FileName '--microsoft-symbol-server' $SymbolsPath $WindowsPdbVerificationParam + $SymbolsOnSymWeb = & $FirstMatchingSymbolDescriptionOrDefault $FileName '--internal-server' $SymbolsPath $WindowsPdbVerificationParam - Write-Host -NoNewLine "`t Checking file " $FileName "... " + Write-Host -NoNewLine "`t Checking file " $FileName "... " - if ($SymbolsOnMSDL -ne $null -and $SymbolsOnSymWeb -ne $null) { - Write-Host "Symbols found on MSDL ($SymbolsOnMSDL) and SymWeb ($SymbolsOnSymWeb)" + if ($SymbolsOnMSDL -ne $null -and $SymbolsOnSymWeb -ne $null) { + Write-Host "Symbols found on MSDL ($SymbolsOnMSDL) and SymWeb ($SymbolsOnSymWeb)" + } + else { + $MissingSymbols++ + + if ($SymbolsOnMSDL -eq $null -and $SymbolsOnSymWeb -eq $null) { + Write-Host 'No symbols found on MSDL or SymWeb!' } else { - $MissingSymbols++ - - if ($SymbolsOnMSDL -eq $null -and $SymbolsOnSymWeb -eq $null) { - Write-Host 'No symbols found on MSDL or SymWeb!' + if ($SymbolsOnMSDL -eq $null) { + Write-Host 'No symbols found on MSDL!' } else { - if ($SymbolsOnMSDL -eq $null) { - Write-Host 'No symbols found on MSDL!' - } - else { - Write-Host 'No symbols found on SymWeb!' - } + Write-Host 'No symbols found on SymWeb!' } } } + } if ($using:Clean) { Remove-Item $ExtractPath -Recurse -Force @@ -168,16 +176,16 @@ $CountMissingSymbols = { Pop-Location return [pscustomobject]@{ - result = $MissingSymbols - packagePath = $PackagePath - } + result = $MissingSymbols + packagePath = $PackagePath + } } function CheckJobResult( - $result, - $packagePath, - [ref]$DupedSymbols, - [ref]$TotalFailures) { + $result, + $packagePath, + [ref]$DupedSymbols, + [ref]$TotalFailures) { if ($result -eq $ERROR_BADEXTRACT) { Write-PipelineTelemetryError -Category 'CheckSymbols' -Message "$packagePath has duplicated symbol files" $DupedSymbols.Value++ @@ -222,7 +230,7 @@ function CheckSymbolsAvailable { return } - Start-Job -ScriptBlock $CountMissingSymbols -ArgumentList $FullName | Out-Null + Start-Job -ScriptBlock $CountMissingSymbols -ArgumentList @($FullName,$WindowsPdbVerificationParam) | Out-Null $NumJobs = @(Get-Job -State 'Running').Count diff --git a/eng/common/templates/job/onelocbuild.yml b/eng/common/templates/job/onelocbuild.yml new file mode 100644 index 00000000..b27d6faf --- /dev/null +++ b/eng/common/templates/job/onelocbuild.yml @@ -0,0 +1,83 @@ +parameters: + # Optional: dependencies of the job + dependsOn: '' + + # Optional: A defined YAML pool - https://docs.microsoft.com/en-us/azure/devops/pipelines/yaml-schema?view=vsts&tabs=schema#pool + pool: + vmImage: vs2017-win2016 + + CeapexPat: $(dn-bot-ceapex-package-r) # PAT for the loc AzDO instance https://dev.azure.com/ceapex + GithubPat: $(BotAccount-dotnet-bot-repo-PAT) + + SourcesDirectory: $(Build.SourcesDirectory) + CreatePr: true + AutoCompletePr: false + UseCheckedInLocProjectJson: false + LanguageSet: VS_Main_Languages + LclSource: lclFilesInRepo + LclPackageId: '' + RepoType: gitHub + condition: '' + +jobs: +- job: OneLocBuild + + dependsOn: ${{ parameters.dependsOn }} + + displayName: OneLocBuild + + pool: ${{ parameters.pool }} + + variables: + - group: OneLocBuildVariables # Contains the CeapexPat and GithubPat + - name: _GenerateLocProjectArguments + value: -SourcesDirectory ${{ parameters.SourcesDirectory }} + -LanguageSet "${{ parameters.LanguageSet }}" + -CreateNeutralXlfs + - ${{ if eq(parameters.UseCheckedInLocProjectJson, 'true') }}: + - name: _GenerateLocProjectArguments + value: ${{ variables._GenerateLocProjectArguments }} -UseCheckedInLocProjectJson + + + steps: + - task: Powershell@2 + inputs: + filePath: $(Build.SourcesDirectory)/eng/common/generate-locproject.ps1 + arguments: $(_GenerateLocProjectArguments) + displayName: Generate LocProject.json + condition: ${{ parameters.condition }} + + - task: OneLocBuild@2 + displayName: OneLocBuild + env: + SYSTEM_ACCESSTOKEN: $(System.AccessToken) + inputs: + locProj: Localize/LocProject.json + outDir: $(Build.ArtifactStagingDirectory) + lclSource: ${{ parameters.LclSource }} + lclPackageId: ${{ parameters.LclPackageId }} + isCreatePrSelected: ${{ parameters.CreatePr }} + ${{ if eq(parameters.CreatePr, true) }}: + isAutoCompletePrSelected: ${{ parameters.AutoCompletePr }} + packageSourceAuth: patAuth + patVariable: ${{ parameters.CeapexPat }} + ${{ if eq(parameters.RepoType, 'gitHub') }}: + repoType: ${{ parameters.RepoType }} + gitHubPatVariable: "${{ parameters.GithubPat }}" + condition: ${{ parameters.condition }} + + - task: PublishBuildArtifacts@1 + displayName: Publish Localization Files + inputs: + PathtoPublish: '$(Build.ArtifactStagingDirectory)/loc' + PublishLocation: Container + ArtifactName: Loc + condition: ${{ parameters.condition }} + + - task: PublishBuildArtifacts@1 + displayName: Publish LocProject.json + inputs: + PathtoPublish: '$(Build.SourcesDirectory)/Localize/' + PublishLocation: Container + ArtifactName: Loc + condition: ${{ parameters.condition }} \ No newline at end of file diff --git a/eng/common/templates/job/performance.yml b/eng/common/templates/job/performance.yml deleted file mode 100644 index f877fd7a..00000000 --- a/eng/common/templates/job/performance.yml +++ /dev/null @@ -1,95 +0,0 @@ -parameters: - steps: [] # optional -- any additional steps that need to happen before pulling down the performance repo and sending the performance benchmarks to helix (ie building your repo) - variables: [] # optional -- list of additional variables to send to the template - jobName: '' # required -- job name - displayName: '' # optional -- display name for the job. Will use jobName if not passed - pool: '' # required -- name of the Build pool - container: '' # required -- name of the container - osGroup: '' # required -- operating system for the job - extraSetupParameters: '' # optional -- extra arguments to pass to the setup script - frameworks: ['netcoreapp3.0'] # optional -- list of frameworks to run against - continueOnError: 'false' # optional -- determines whether to continue the build if the step errors - dependsOn: '' # optional -- dependencies of the job - timeoutInMinutes: 320 # optional -- timeout for the job - enableTelemetry: false # optional -- enable for telemetry - -jobs: -- template: ../jobs/jobs.yml - parameters: - dependsOn: ${{ parameters.dependsOn }} - enableTelemetry: ${{ parameters.enableTelemetry }} - enablePublishBuildArtifacts: true - continueOnError: ${{ parameters.continueOnError }} - - jobs: - - job: '${{ parameters.jobName }}' - - ${{ if ne(parameters.displayName, '') }}: - displayName: '${{ parameters.displayName }}' - ${{ if eq(parameters.displayName, '') }}: - displayName: '${{ parameters.jobName }}' - - timeoutInMinutes: ${{ parameters.timeoutInMinutes }} - - variables: - - - ${{ each variable in parameters.variables }}: - - ${{ if ne(variable.name, '') }}: - - name: ${{ variable.name }} - value: ${{ variable.value }} - - ${{ if ne(variable.group, '') }}: - - group: ${{ variable.group }} - - - IsInternal: '' - - HelixApiAccessToken: '' - - HelixPreCommand: '' - - - ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: - - ${{ if eq( parameters.osGroup, 'Windows_NT') }}: - - HelixPreCommand: 'set "PERFLAB_UPLOAD_TOKEN=$(PerfCommandUploadToken)"' - - IsInternal: -Internal - - ${{ if ne(parameters.osGroup, 'Windows_NT') }}: - - HelixPreCommand: 'export PERFLAB_UPLOAD_TOKEN="$(PerfCommandUploadTokenLinux)"' - - IsInternal: --internal - - - group: DotNet-HelixApi-Access - - group: dotnet-benchview - - workspace: - clean: all - pool: - ${{ parameters.pool }} - container: ${{ parameters.container }} - strategy: - matrix: - ${{ each framework in parameters.frameworks }}: - ${{ framework }}: - _Framework: ${{ framework }} - steps: - - checkout: self - clean: true - # Run all of the steps to setup repo - - ${{ each step in parameters.steps }}: - - ${{ step }} - - powershell: $(Build.SourcesDirectory)\eng\common\performance\performance-setup.ps1 $(IsInternal) -Framework $(_Framework) ${{ parameters.extraSetupParameters }} - displayName: Performance Setup (Windows) - condition: and(succeeded(), eq(variables['Agent.Os'], 'Windows_NT')) - continueOnError: ${{ parameters.continueOnError }} - - script: $(Build.SourcesDirectory)/eng/common/performance/performance-setup.sh $(IsInternal) --framework $(_Framework) ${{ parameters.extraSetupParameters }} - displayName: Performance Setup (Unix) - condition: and(succeeded(), ne(variables['Agent.Os'], 'Windows_NT')) - continueOnError: ${{ parameters.continueOnError }} - - script: $(Python) $(PerformanceDirectory)/scripts/ci_setup.py $(SetupArguments) - displayName: Run ci setup script - # Run perf testing in helix - - template: /eng/common/templates/steps/perf-send-to-helix.yml - parameters: - HelixSource: '$(HelixSourcePrefix)/$(Build.Repository.Name)/$(Build.SourceBranch)' # sources must start with pr/, official/, prodcon/, or agent/ - HelixType: 'test/performance/$(Kind)/$(_Framework)/$(Architecture)' - HelixAccessToken: $(HelixApiAccessToken) - HelixTargetQueues: $(Queue) - HelixPreCommands: $(HelixPreCommand) - Creator: $(Creator) - WorkItemTimeout: 4:00 # 4 hours - WorkItemDirectory: '$(WorkItemDirectory)' # WorkItemDirectory can not be empty, so we send it some docs to keep it happy - CorrelationPayloadDirectory: '$(PayloadDirectory)' # it gets checked out to a folder with shorter path than WorkItemDirectory so we can avoid file name too long exceptions \ No newline at end of file diff --git a/eng/common/templates/job/source-index-stage1.yml b/eng/common/templates/job/source-index-stage1.yml index c002a2b1..a649d2b5 100644 --- a/eng/common/templates/job/source-index-stage1.yml +++ b/eng/common/templates/job/source-index-stage1.yml @@ -1,6 +1,6 @@ parameters: runAsPublic: false - sourceIndexPackageVersion: 1.0.1-20210225.1 + sourceIndexPackageVersion: 1.0.1-20210421.1 sourceIndexPackageSource: https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools/nuget/v3/index.json sourceIndexBuildCommand: powershell -NoLogo -NoProfile -ExecutionPolicy Bypass -Command "eng/common/build.ps1 -restore -build -binarylog -ci" preSteps: [] diff --git a/eng/common/templates/steps/perf-send-to-helix.yml b/eng/common/templates/steps/perf-send-to-helix.yml deleted file mode 100644 index 3427b311..00000000 --- a/eng/common/templates/steps/perf-send-to-helix.yml +++ /dev/null @@ -1,50 +0,0 @@ -# Please remember to update the documentation if you make changes to these parameters! -parameters: - ProjectFile: '' # required -- project file that specifies the helix workitems - HelixSource: 'pr/default' # required -- sources must start with pr/, official/, prodcon/, or agent/ - HelixType: 'tests/default/' # required -- Helix telemetry which identifies what type of data this is; should include "test" for clarity and must end in '/' - HelixBuild: $(Build.BuildNumber) # required -- the build number Helix will use to identify this -- automatically set to the AzDO build number - HelixTargetQueues: '' # required -- semicolon delimited list of Helix queues to test on; see https://helix.dot.net/ for a list of queues - HelixAccessToken: '' # required -- access token to make Helix API requests; should be provided by the appropriate variable group - HelixPreCommands: '' # optional -- commands to run before Helix work item execution - HelixPostCommands: '' # optional -- commands to run after Helix work item execution - WorkItemDirectory: '' # optional -- a payload directory to zip up and send to Helix; requires WorkItemCommand; incompatible with XUnitProjects - CorrelationPayloadDirectory: '' # optional -- a directory to zip up and send to Helix as a correlation payload - IncludeDotNetCli: false # optional -- true will download a version of the .NET CLI onto the Helix machine as a correlation payload; requires DotNetCliPackageType and DotNetCliVersion - DotNetCliPackageType: '' # optional -- either 'sdk', 'runtime' or 'aspnetcore-runtime'; determines whether the sdk or runtime will be sent to Helix; see https://raw.githubusercontent.com/dotnet/core/main/release-notes/releases.json - DotNetCliVersion: '' # optional -- version of the CLI to send to Helix; based on this: https://raw.githubusercontent.com/dotnet/core/main/release-notes/releases.json - EnableXUnitReporter: false # optional -- true enables XUnit result reporting to Mission Control - WaitForWorkItemCompletion: true # optional -- true will make the task wait until work items have been completed and fail the build if work items fail. False is "fire and forget." - Creator: '' # optional -- if the build is external, use this to specify who is sending the job - DisplayNamePrefix: 'Send job to Helix' # optional -- rename the beginning of the displayName of the steps in AzDO - condition: succeeded() # optional -- condition for step to execute; defaults to succeeded() - continueOnError: false # optional -- determines whether to continue the build if the step errors; defaults to false - osGroup: '' # required -- operating system for the job - - -steps: -- template: /eng/pipelines/common/templates/runtimes/send-to-helix-inner-step.yml - parameters: - osGroup: ${{ parameters.osGroup }} - sendParams: $(Build.SourcesDirectory)/eng/common/performance/${{ parameters.ProjectFile }} /restore /t:Test /bl:$(Build.SourcesDirectory)/artifacts/log/$(_BuildConfig)/SendToHelix.binlog - displayName: ${{ parameters.DisplayNamePrefix }} - condition: ${{ parameters.condition }} - continueOnError: ${{ parameters.continueOnError }} - environment: - BuildConfig: $(_BuildConfig) - HelixSource: ${{ parameters.HelixSource }} - HelixType: ${{ parameters.HelixType }} - HelixBuild: ${{ parameters.HelixBuild }} - HelixTargetQueues: ${{ parameters.HelixTargetQueues }} - HelixAccessToken: ${{ parameters.HelixAccessToken }} - HelixPreCommands: ${{ parameters.HelixPreCommands }} - HelixPostCommands: ${{ parameters.HelixPostCommands }} - WorkItemDirectory: ${{ parameters.WorkItemDirectory }} - CorrelationPayloadDirectory: ${{ parameters.CorrelationPayloadDirectory }} - IncludeDotNetCli: ${{ parameters.IncludeDotNetCli }} - DotNetCliPackageType: ${{ parameters.DotNetCliPackageType }} - DotNetCliVersion: ${{ parameters.DotNetCliVersion }} - EnableXUnitReporter: ${{ parameters.EnableXUnitReporter }} - WaitForWorkItemCompletion: ${{ parameters.WaitForWorkItemCompletion }} - Creator: ${{ parameters.Creator }} - SYSTEM_ACCESSTOKEN: $(System.AccessToken) diff --git a/eng/common/templates/steps/source-build.yml b/eng/common/templates/steps/source-build.yml index 8e336b7d..65ee5992 100644 --- a/eng/common/templates/steps/source-build.yml +++ b/eng/common/templates/steps/source-build.yml @@ -36,7 +36,7 @@ steps: ${{ coalesce(parameters.platform.buildScript, './build.sh') }} --ci \ --configuration $buildConfig \ - --restore --build --pack --publish \ + --restore --build --pack --publish -bl \ $officialBuildArgs \ $targetRidArgs \ /p:SourceBuildNonPortable=${{ parameters.platform.nonPortable }} \ diff --git a/global.json b/global.json index 1f94c98e..3c093d2e 100644 --- a/global.json +++ b/global.json @@ -1,16 +1,16 @@ { "tools": { - "dotnet": "6.0.100-preview.1.21103.13", + "dotnet": "6.0.100-preview.3.21202.5", "runtimes": { "dotnet": [ - "3.1.2" + "3.1.14" ], "aspnetcore": [ - "3.1.4" + "3.1.14" ] } }, "msbuild-sdks": { - "Microsoft.DotNet.Arcade.Sdk": "6.0.0-beta.21160.1" + "Microsoft.DotNet.Arcade.Sdk": "6.0.0-beta.21230.2" } } diff --git a/src/Microsoft.Tye.Core/ApplicationFactory.cs b/src/Microsoft.Tye.Core/ApplicationFactory.cs index cd1f59ff..35c1ee37 100644 --- a/src/Microsoft.Tye.Core/ApplicationFactory.cs +++ b/src/Microsoft.Tye.Core/ApplicationFactory.cs @@ -519,7 +519,7 @@ namespace Microsoft.Tye var projectEvaluationTargets = Path.Combine(Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location)!, "ProjectEvaluation.targets"); var msbuildEvaluationResult = await ProcessUtil.RunAsync( "dotnet", - $"build " + + $"build --no-restore " + $"\"{projectPath}\" " + // CustomAfterMicrosoftCommonTargets is imported by non-crosstargeting (single TFM) projects @$"/p:CustomAfterMicrosoftCommonTargets=""{projectEvaluationTargets}"" " + diff --git a/src/Microsoft.Tye.Core/Microsoft.Tye.Core.csproj b/src/Microsoft.Tye.Core/Microsoft.Tye.Core.csproj index 32da156e..b35ca1b9 100644 --- a/src/Microsoft.Tye.Core/Microsoft.Tye.Core.csproj +++ b/src/Microsoft.Tye.Core/Microsoft.Tye.Core.csproj @@ -9,13 +9,7 @@ - - - - + diff --git a/src/Microsoft.Tye.Core/MsBuild/EscapingUtilities.cs b/src/Microsoft.Tye.Core/MsBuild/EscapingUtilities.cs new file mode 100644 index 00000000..e21308e8 --- /dev/null +++ b/src/Microsoft.Tye.Core/MsBuild/EscapingUtilities.cs @@ -0,0 +1,358 @@ +// Copyright (c) Microsoft. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using System; +using System.Collections.Generic; +using System.Globalization; +using System.Text; + +#pragma warning disable CS8618, CS8625, CS8601, CS8600, CS8604, CS0162, CS8603, CS0168 + +namespace Microsoft.Build.Shared +{ + /// + /// This class implements static methods to assist with unescaping of %XX codes + /// in the MSBuild file format. + /// + /// + /// PERF: since we escape and unescape relatively frequently, it may be worth caching + /// the last N strings that were (un)escaped + /// + static internal class EscapingUtilities + { + /// + /// Optional cache of escaped strings for use when needing to escape in performance-critical scenarios with significant + /// expected string reuse. + /// + private static Dictionary s_unescapedToEscapedStrings = new Dictionary(StringComparer.Ordinal); + + private static bool TryDecodeHexDigit(char character, out int value) + { + if (character >= '0' && character <= '9') + { + value = character - '0'; + return true; + } + if (character >= 'A' && character <= 'F') + { + value = character - 'A' + 10; + return true; + } + if (character >= 'a' && character <= 'f') + { + value = character - 'a' + 10; + return true; + } + value = default; + return false; + } + + /// + /// Replaces all instances of %XX in the input string with the character represented + /// by the hexadecimal number XX. + /// + /// The string to unescape. + /// If the string should be trimmed before being unescaped. + /// unescaped string + internal static string UnescapeAll(string escapedString, bool trim = false) + { + // If the string doesn't contain anything, then by definition it doesn't + // need unescaping. + if (String.IsNullOrEmpty(escapedString)) + { + return escapedString; + } + + // If there are no percent signs, just return the original string immediately. + // Don't even instantiate the StringBuilder. + int indexOfPercent = escapedString.IndexOf('%'); + if (indexOfPercent == -1) + { + return trim ? escapedString.Trim() : escapedString; + } + + // This is where we're going to build up the final string to return to the caller. + StringBuilder unescapedString = StringBuilderCache.Acquire(escapedString.Length); + + int currentPosition = 0; + int escapedStringLength = escapedString.Length; + if (trim) + { + while (currentPosition < escapedString.Length && Char.IsWhiteSpace(escapedString[currentPosition])) + { + currentPosition++; + } + if (currentPosition == escapedString.Length) + { + return String.Empty; + } + while (Char.IsWhiteSpace(escapedString[escapedStringLength - 1])) + { + escapedStringLength--; + } + } + + // Loop until there are no more percent signs in the input string. + while (indexOfPercent != -1) + { + // There must be two hex characters following the percent sign + // for us to even consider doing anything with this. + if ( + (indexOfPercent <= (escapedStringLength - 3)) && + TryDecodeHexDigit(escapedString[indexOfPercent + 1], out int digit1) && + TryDecodeHexDigit(escapedString[indexOfPercent + 2], out int digit2) + ) + { + // First copy all the characters up to the current percent sign into + // the destination. + unescapedString.Append(escapedString, currentPosition, indexOfPercent - currentPosition); + + // Convert the %XX to an actual real character. + char unescapedCharacter = (char)((digit1 << 4) + digit2); + + // if the unescaped character is not on the exception list, append it + unescapedString.Append(unescapedCharacter); + + // Advance the current pointer to reflect the fact that the destination string + // is up to date with everything up to and including this escape code we just found. + currentPosition = indexOfPercent + 3; + } + + // Find the next percent sign. + indexOfPercent = escapedString.IndexOf('%', indexOfPercent + 1); + } + + // Okay, there are no more percent signs in the input string, so just copy the remaining + // characters into the destination. + unescapedString.Append(escapedString, currentPosition, escapedStringLength - currentPosition); + + return StringBuilderCache.GetStringAndRelease(unescapedString); + } + + + /// + /// Adds instances of %XX in the input string where the char to be escaped appears + /// XX is the hex value of the ASCII code for the char. Interns and caches the result. + /// + /// + /// NOTE: Only recommended for use in scenarios where there's expected to be significant + /// repetition of the escaped string. Cache currently grows unbounded. + /// + internal static string EscapeWithCaching(string unescapedString) + { + return EscapeWithOptionalCaching(unescapedString, cache: true); + } + + /// + /// Adds instances of %XX in the input string where the char to be escaped appears + /// XX is the hex value of the ASCII code for the char. + /// + /// The string to escape. + /// escaped string + internal static string Escape(string unescapedString) + { + return EscapeWithOptionalCaching(unescapedString, cache: false); + } + + /// + /// Adds instances of %XX in the input string where the char to be escaped appears + /// XX is the hex value of the ASCII code for the char. Caches if requested. + /// + /// The string to escape. + /// + /// True if the cache should be checked, and if the resultant string + /// should be cached. + /// + private static string EscapeWithOptionalCaching(string unescapedString, bool cache) + { + // If there are no special chars, just return the original string immediately. + // Don't even instantiate the StringBuilder. + if (String.IsNullOrEmpty(unescapedString) || !ContainsReservedCharacters(unescapedString)) + { + return unescapedString; + } + + // next, if we're caching, check to see if it's already there. + if (cache) + { + lock (s_unescapedToEscapedStrings) + { + string cachedEscapedString; + if (s_unescapedToEscapedStrings.TryGetValue(unescapedString, out cachedEscapedString)) + { + return cachedEscapedString; + } + } + } + + // This is where we're going to build up the final string to return to the caller. + StringBuilder escapedStringBuilder = StringBuilderCache.Acquire(unescapedString.Length * 2); + + AppendEscapedString(escapedStringBuilder, unescapedString); + + if (!cache) + { + return StringBuilderCache.GetStringAndRelease(escapedStringBuilder); + } + + string escapedString = escapedStringBuilder.ToString(); + StringBuilderCache.Release(escapedStringBuilder); + + lock (s_unescapedToEscapedStrings) + { + s_unescapedToEscapedStrings[unescapedString] = escapedString; + } + + return escapedString; + } + + /// + /// Before trying to actually escape the string, it can be useful to call this method to determine + /// if escaping is necessary at all. This can save lots of calls to copy around item metadata + /// that is really the same whether escaped or not. + /// + /// + /// + private static bool ContainsReservedCharacters + ( + string unescapedString + ) + { + return -1 != unescapedString.IndexOfAny(s_charsToEscape); + } + + /// + /// Determines whether the string contains the escaped form of '*' or '?'. + /// + /// + /// + internal static bool ContainsEscapedWildcards(string escapedString) + { + if (escapedString.Length < 3) + { + return false; + } + // Look for the first %. We know that it has to be followed by at least two more characters so we subtract 2 + // from the length to search. + int index = escapedString.IndexOf('%', 0, escapedString.Length - 2); + while (index != -1) + { + if (escapedString[index + 1] == '2' && (escapedString[index + 2] == 'a' || escapedString[index + 2] == 'A')) + { + // %2a or %2A + return true; + } + if (escapedString[index + 1] == '3' && (escapedString[index + 2] == 'f' || escapedString[index + 2] == 'F')) + { + // %3f or %3F + return true; + } + // Continue searching for % starting at (index + 1). We know that it has to be followed by at least two + // more characters so we subtract 2 from the length of the substring to search. + index = escapedString.IndexOf('%', index + 1, escapedString.Length - (index + 1) - 2); + } + return false; + } + + /// + /// Convert the given integer into its hexadecimal representation. + /// + /// The number to convert, which must be non-negative and less than 16 + /// The character which is the hexadecimal representation of . + private static char HexDigitChar(int x) + { + return (char)(x + (x < 10 ? '0' : ('a' - 10))); + } + + /// + /// Append the escaped version of the given character to a . + /// + /// The to which to append. + /// The character to escape. + private static void AppendEscapedChar(StringBuilder sb, char ch) + { + // Append the escaped version which is a percent sign followed by two hexadecimal digits + sb.Append('%'); + sb.Append(HexDigitChar(ch / 0x10)); + sb.Append(HexDigitChar(ch & 0x0F)); + } + + /// + /// Append the escaped version of the given string to a . + /// + /// The to which to append. + /// The unescaped string. + private static void AppendEscapedString(StringBuilder sb, string unescapedString) + { + // Replace each unescaped special character with an escape sequence one + for (int idx = 0; ;) + { + int nextIdx = unescapedString.IndexOfAny(s_charsToEscape, idx); + if (nextIdx == -1) + { + sb.Append(unescapedString, idx, unescapedString.Length - idx); + break; + } + + sb.Append(unescapedString, idx, nextIdx - idx); + AppendEscapedChar(sb, unescapedString[nextIdx]); + idx = nextIdx + 1; + } + } + + /// + /// Special characters that need escaping. + /// It's VERY important that the percent character is the FIRST on the list - since it's both a character + /// we escape and use in escape sequences, we can unintentionally escape other escape sequences if we + /// don't process it first. Of course we'll have a similar problem if we ever decide to escape hex digits + /// (that would require rewriting the algorithm) but since it seems unlikely that we ever do, this should + /// be good enough to avoid complicating the algorithm at this point. + /// + private static readonly char[] s_charsToEscape = { '%', '*', '?', '@', '$', '(', ')', ';', '\'' }; + } + internal static class StringBuilderCache + { + // The value 360 was chosen in discussion with performance experts as a compromise between using + // as little memory (per thread) as possible and still covering a large part of short-lived + // StringBuilder creations on the startup path of VS designers. + private const int MAX_BUILDER_SIZE = 360; + + [ThreadStatic] + private static StringBuilder t_cachedInstance; + + public static StringBuilder Acquire(int capacity = 16 /*StringBuilder.DefaultCapacity*/) + { + if (capacity <= MAX_BUILDER_SIZE) + { + StringBuilder sb = StringBuilderCache.t_cachedInstance; + if (sb != null) + { + // Avoid StringBuilder block fragmentation by getting a new StringBuilder + // when the requested size is larger than the current capacity + if (capacity <= sb.Capacity) + { + StringBuilderCache.t_cachedInstance = null; + sb.Length = 0; // Equivalent of sb.Clear() that works on .Net 3.5 + return sb; + } + } + } + return new StringBuilder(capacity); + } + + public static void Release(StringBuilder sb) + { + if (sb.Capacity <= MAX_BUILDER_SIZE) + { + StringBuilderCache.t_cachedInstance = sb; + } + } + + public static string GetStringAndRelease(StringBuilder sb) + { + string result = sb.ToString(); + Release(sb); + return result; + } + } +} diff --git a/src/Microsoft.Tye.Core/MsBuild/FileUtilities.cs b/src/Microsoft.Tye.Core/MsBuild/FileUtilities.cs new file mode 100644 index 00000000..9676c3ce --- /dev/null +++ b/src/Microsoft.Tye.Core/MsBuild/FileUtilities.cs @@ -0,0 +1,458 @@ +// Copyright (c) Microsoft. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using System; +#if !CLR2COMPATIBILITY +using System.Collections.Concurrent; +#else +using Microsoft.Build.Shared.Concurrent; +#endif +using System.Collections.Generic; +using System.Diagnostics; +using System.Diagnostics.CodeAnalysis; +using System.IO; +using System.Globalization; +using System.Linq; +using System.Reflection; +using System.Runtime.CompilerServices; +using System.Text; +using System.Threading; +using System.Runtime.InteropServices; + +#pragma warning disable CS8618, CS8625, CS8601, CS8600, CS8604, CS0162, CS8603, CS0168 + +namespace Microsoft.Build.Shared +{ + /// + /// This class contains utility methods for file IO. + /// PERF\COVERAGE NOTE: Try to keep classes in 'shared' as granular as possible. All the methods in + /// each class get pulled into the resulting assembly. + /// + internal static partial class FileUtilities + { + // A list of possible test runners. If the program running has one of these substrings in the name, we assume + // this is a test harness. + + // This flag, when set, indicates that we are running tests. Initially assume it's true. It also implies that + // the currentExecutableOverride is set to a path (that is non-null). Assume this is not initialized when we + // have the impossible combination of runningTests = false and currentExecutableOverride = null. + + // This is the fake current executable we use in case we are running tests. + + /// + /// The directory where MSBuild stores cache information used during the build. + /// + internal static string cacheDirectory = null; + + /// + /// FOR UNIT TESTS ONLY + /// Clear out the static variable used for the cache directory so that tests that + /// modify it can validate their modifications. + /// + internal static void ClearCacheDirectoryPath() + { + cacheDirectory = null; + } + + internal static readonly StringComparison PathComparison = GetIsFileSystemCaseSensitive() ? StringComparison.Ordinal : StringComparison.OrdinalIgnoreCase; + + /// + /// Determines whether the file system is case sensitive. + /// Copied from https://github.com/dotnet/runtime/blob/73ba11f3015216b39cb866d9fb7d3d25e93489f2/src/libraries/Common/src/System/IO/PathInternal.CaseSensitivity.cs#L41-L59 + /// + public static bool GetIsFileSystemCaseSensitive() + { + try + { + string pathWithUpperCase = Path.Combine(Path.GetTempPath(), "CASESENSITIVETEST" + Guid.NewGuid().ToString("N")); + using (new FileStream(pathWithUpperCase, FileMode.CreateNew, FileAccess.ReadWrite, FileShare.None, 0x1000, FileOptions.DeleteOnClose)) + { + string lowerCased = pathWithUpperCase.ToLowerInvariant(); + return !File.Exists(lowerCased); + } + } + catch (Exception exc) + { + // In case something goes terribly wrong, we don't want to fail just because + // of a casing test, so we assume case-insensitive-but-preserving. + Debug.Fail("Casing test failed: " + exc); + return false; + } + } + + /// + /// Copied from https://github.com/dotnet/corefx/blob/056715ff70e14712419d82d51c8c50c54b9ea795/src/Common/src/System/IO/PathInternal.Windows.cs#L61 + /// MSBuild should support the union of invalid path chars across the supported OSes, so builds can have the same behaviour crossplatform: https://github.com/Microsoft/msbuild/issues/781#issuecomment-243942514 + /// + internal static readonly char[] InvalidPathChars = new char[] + { + '|', '\0', + (char)1, (char)2, (char)3, (char)4, (char)5, (char)6, (char)7, (char)8, (char)9, (char)10, + (char)11, (char)12, (char)13, (char)14, (char)15, (char)16, (char)17, (char)18, (char)19, (char)20, + (char)21, (char)22, (char)23, (char)24, (char)25, (char)26, (char)27, (char)28, (char)29, (char)30, + (char)31 + }; + + /// + /// Copied from https://github.com/dotnet/corefx/blob/387cf98c410bdca8fd195b28cbe53af578698f94/src/System.Runtime.Extensions/src/System/IO/Path.Windows.cs#L18 + /// MSBuild should support the union of invalid path chars across the supported OSes, so builds can have the same behaviour crossplatform: https://github.com/Microsoft/msbuild/issues/781#issuecomment-243942514 + /// + internal static readonly char[] InvalidFileNameChars = new char[] + { + '\"', '<', '>', '|', '\0', + (char)1, (char)2, (char)3, (char)4, (char)5, (char)6, (char)7, (char)8, (char)9, (char)10, + (char)11, (char)12, (char)13, (char)14, (char)15, (char)16, (char)17, (char)18, (char)19, (char)20, + (char)21, (char)22, (char)23, (char)24, (char)25, (char)26, (char)27, (char)28, (char)29, (char)30, + (char)31, ':', '*', '?', '\\', '/' + }; + + internal static readonly char[] Slashes = { '/', '\\' }; + + internal static readonly string DirectorySeparatorString = Path.DirectorySeparatorChar.ToString(); + + private static readonly ConcurrentDictionary FileExistenceCache = new ConcurrentDictionary(StringComparer.OrdinalIgnoreCase); + + + private static string GetFullPath(string path) + { + return Path.GetFullPath(path); + } + + internal static string FixFilePath(string path) + { + return string.IsNullOrEmpty(path) || Path.DirectorySeparatorChar == '\\' ? path : path.Replace('\\', '/');//.Replace("//", "/"); + } + + /// + /// Determines the full path for the given file-spec. + /// ASSUMES INPUT IS STILL ESCAPED + /// + /// The file spec to get the full path of. + /// + /// full path + internal static string GetFullPath(string fileSpec, string currentDirectory) + { + // Sending data out of the engine into the filesystem, so time to unescape. + fileSpec = FixFilePath(EscapingUtilities.UnescapeAll(fileSpec)); + + // Data coming back from the filesystem into the engine, so time to escape it back. + string fullPath = EscapingUtilities.Escape(NormalizePath(Path.Combine(currentDirectory, fileSpec))); + + if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows) && !EndsWithSlash(fullPath)) + { + if (FileUtilitiesRegex.IsDrivePattern(fileSpec) || + FileUtilitiesRegex.IsUncPattern(fullPath)) + { + // append trailing slash if Path.GetFullPath failed to (this happens with drive-specs and UNC shares) + fullPath += Path.DirectorySeparatorChar; + } + } + + return fullPath; + } + + /// + /// Indicates if the given file-spec ends with a slash. + /// + /// The file spec. + /// true, if file-spec has trailing slash + internal static bool EndsWithSlash(string fileSpec) + { + return (fileSpec.Length > 0) + ? IsSlash(fileSpec[fileSpec.Length - 1]) + : false; + } + + /// + /// Indicates if the given character is a slash. + /// + /// + /// true, if slash + internal static bool IsSlash(char c) + { + return (c == Path.DirectorySeparatorChar) || (c == Path.AltDirectorySeparatorChar); + } + + + /// + /// Gets the canonicalized full path of the provided path. + /// Guidance for use: call this on all paths accepted through public entry + /// points that need normalization. After that point, only verify the path + /// is rooted, using ErrorUtilities.VerifyThrowPathRooted. + /// ASSUMES INPUT IS ALREADY UNESCAPED. + /// + internal static string NormalizePath(string path) + { + string fullPath = GetFullPath(path); + return FixFilePath(fullPath); + } + + internal static bool IsSolutionFilterFilename(string filename) + { + return HasExtension(filename, ".slnf"); + } + + private static bool HasExtension(string filename, string extension) + { + if (String.IsNullOrEmpty(filename)) + return false; + + return filename.EndsWith(extension, PathComparison); + } + + /// + /// If on Unix, convert backslashes to slashes for strings that resemble paths. + /// The heuristic is if something resembles paths (contains slashes) check if the + /// first segment exists and is a directory. + /// Use a native shared method to massage file path. If the file is adjusted, + /// that qualifies is as a path. + /// + /// @baseDirectory is just passed to LooksLikeUnixFilePath, to help with the check + /// + internal static string MaybeAdjustFilePath(string value, string baseDirectory = "") + { + var comparisonType = StringComparison.Ordinal; + + // Don't bother with arrays or properties or network paths, or those that + // have no slashes. + if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows) || string.IsNullOrEmpty(value) + || value.StartsWith("$(", comparisonType) || value.StartsWith("@(", comparisonType) + || value.StartsWith("\\\\", comparisonType)) + { + return value; + } + + // For Unix-like systems, we may want to convert backslashes to slashes + Span newValue = ConvertToUnixSlashes(value.ToCharArray()); + + // Find the part of the name we want to check, that is remove quotes, if present + bool shouldAdjust = newValue.IndexOf('/') != -1 && LooksLikeUnixFilePath(RemoveQuotes(newValue), baseDirectory); + return shouldAdjust ? newValue.ToString() : value; + } + + private static Span ConvertToUnixSlashes(Span path) + { + return path.IndexOf('\\') == -1 ? path : CollapseSlashes(path); + } + + /// + /// If on Unix, check if the string looks like a file path. + /// The heuristic is if something resembles paths (contains slashes) check if the + /// first segment exists and is a directory. + /// + /// If @baseDirectory is not null, then look for the first segment exists under + /// that + /// + internal static bool LooksLikeUnixFilePath(string value, string baseDirectory = "") + => LooksLikeUnixFilePath(value.AsSpan(), baseDirectory); + + internal static bool LooksLikeUnixFilePath(ReadOnlySpan value, string baseDirectory = "") + { + if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) + { + return false; + } + + // The first slash will either be at the beginning of the string or after the first directory name + int directoryLength = value.Slice(1).IndexOf('/') + 1; + bool shouldCheckDirectory = directoryLength != 0; + + // Check for actual files or directories under / that get missed by the above logic + bool shouldCheckFileOrDirectory = !shouldCheckDirectory && value.Length > 0 && value[0] == '/'; + ReadOnlySpan directory = value.Slice(0, directoryLength); + + return (shouldCheckDirectory && Directory.Exists(Path.Combine(baseDirectory, directory.ToString()))) + || (shouldCheckFileOrDirectory && Directory.Exists(value.ToString())); + } + + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static Span CollapseSlashes(Span str) + { + int sliceLength = 0; + + // Performs Regex.Replace(str, @"[\\/]+", "/") + for (int i = 0; i < str.Length; i++) + { + bool isCurSlash = IsAnySlash(str[i]); + bool isPrevSlash = i > 0 && IsAnySlash(str[i - 1]); + + if (!isCurSlash || !isPrevSlash) + { + str[sliceLength] = str[i] == '\\' ? '/' : str[i]; + sliceLength++; + } + } + + return str.Slice(0, sliceLength); + } + + internal static bool IsAnySlash(char c) => c == '/' || c == '\\'; + + + private static Span RemoveQuotes(Span path) + { + int endId = path.Length - 1; + char singleQuote = '\''; + char doubleQuote = '\"'; + + bool hasQuotes = path.Length > 2 + && ((path[0] == singleQuote && path[endId] == singleQuote) + || (path[0] == doubleQuote && path[endId] == doubleQuote)); + + return hasQuotes ? path.Slice(1, endId - 1) : path; + } + } + + internal static class FileUtilitiesRegex + { + private static readonly char _backSlash = '\\'; + private static readonly char _forwardSlash = '/'; + + /// + /// Indicates whether the specified string follows the pattern drive pattern (for example "C:", "D:"). + /// + /// Input to check for drive pattern. + /// true if follows the drive pattern, false otherwise. + internal static bool IsDrivePattern(string pattern) + { + // Format must be two characters long: ":" + return pattern.Length == 2 && + StartsWithDrivePattern(pattern); + } + + /// + /// Indicates whether the specified string follows the pattern drive pattern (for example "C:/" or "C:\"). + /// + /// Input to check for drive pattern with slash. + /// true if follows the drive pattern with slash, false otherwise. + internal static bool IsDrivePatternWithSlash(string pattern) + { + return pattern.Length == 3 && + StartsWithDrivePatternWithSlash(pattern); + } + + /// + /// Indicates whether the specified string starts with the drive pattern (for example "C:"). + /// + /// Input to check for drive pattern. + /// true if starts with drive pattern, false otherwise. + internal static bool StartsWithDrivePattern(string pattern) + { + // Format dictates a length of at least 2, + // first character must be a letter, + // second character must be a ":" + return pattern.Length >= 2 && + ((pattern[0] >= 'A' && pattern[0] <= 'Z') || (pattern[0] >= 'a' && pattern[0] <= 'z')) && + pattern[1] == ':'; + } + + /// + /// Indicates whether the specified string starts with the drive pattern (for example "C:/" or "C:\"). + /// + /// Input to check for drive pattern. + /// true if starts with drive pattern with slash, false otherwise. + internal static bool StartsWithDrivePatternWithSlash(string pattern) + { + // Format dictates a length of at least 3, + // first character must be a letter, + // second character must be a ":" + // third character must be a slash. + return pattern.Length >= 3 && + StartsWithDrivePattern(pattern) && + (pattern[2] == _backSlash || pattern[2] == _forwardSlash); + } + + /// + /// Indicates whether the specified file-spec comprises exactly "\\server\share" (with no trailing characters). + /// + /// Input to check for UNC pattern. + /// true if comprises UNC pattern. + internal static bool IsUncPattern(string pattern) + { + //Return value == pattern.length means: + // meets minimum unc requirements + // pattern does not end in a '/' or '\' + // if a subfolder were found the value returned would be length up to that subfolder, therefore no subfolder exists + return StartsWithUncPatternMatchLength(pattern) == pattern.Length; + } + + /// + /// Indicates whether the specified file-spec begins with "\\server\share". + /// + /// Input to check for UNC pattern. + /// true if starts with UNC pattern. + internal static bool StartsWithUncPattern(string pattern) + { + //Any non -1 value returned means there was a match, therefore is begins with the pattern. + return StartsWithUncPatternMatchLength(pattern) != -1; + } + + /// + /// Indicates whether the file-spec begins with a UNC pattern and how long the match is. + /// + /// Input to check for UNC pattern. + /// length of the match, -1 if no match. + internal static int StartsWithUncPatternMatchLength(string pattern) + { + if (!MeetsUncPatternMinimumRequirements(pattern)) + { + return -1; + } + + bool prevCharWasSlash = true; + bool hasShare = false; + + for (int i = 2; i < pattern.Length; i++) + { + //Real UNC paths should only contain backslashes. However, the previous + // regex pattern accepted both so functionality will be retained. + if (pattern[i] == _backSlash || + pattern[i] == _forwardSlash) + { + if (prevCharWasSlash) + { + //We get here in the case of an extra slash. + return -1; + } + else if (hasShare) + { + return i; + } + + hasShare = true; + prevCharWasSlash = true; + } + else + { + prevCharWasSlash = false; + } + } + + if (!hasShare) + { + //no subfolder means no unc pattern. string is something like "\\abc" in this case + return -1; + } + + return pattern.Length; + } + + /// + /// Indicates whether or not the file-spec meets the minimum requirements of a UNC pattern. + /// + /// Input to check for UNC pattern minimum requirements. + /// true if the UNC pattern is a minimum length of 5 and the first two characters are be a slash, false otherwise. +#if !NET35 + [MethodImpl(MethodImplOptions.AggressiveInlining)] +#endif + internal static bool MeetsUncPatternMinimumRequirements(string pattern) + { + return pattern.Length >= 5 && + (pattern[0] == _backSlash || + pattern[0] == _forwardSlash) && + (pattern[1] == _backSlash || + pattern[1] == _forwardSlash); + } + } +} diff --git a/src/Microsoft.Tye.Core/MsBuild/ProjectConfigurationInSolution.cs b/src/Microsoft.Tye.Core/MsBuild/ProjectConfigurationInSolution.cs new file mode 100644 index 00000000..0700c134 --- /dev/null +++ b/src/Microsoft.Tye.Core/MsBuild/ProjectConfigurationInSolution.cs @@ -0,0 +1,62 @@ +// Copyright (c) Microsoft. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using System; + +namespace Microsoft.Build.Construction +{ + /// + /// This class represents an entry for a project configuration in a solution configuration. + /// + public sealed class ProjectConfigurationInSolution + { + /// + /// Constructor + /// + internal ProjectConfigurationInSolution(string configurationName, string platformName, bool includeInBuild) + { + ConfigurationName = configurationName; + PlatformName = RemoveSpaceFromAnyCpuPlatform(platformName); + IncludeInBuild = includeInBuild; + FullName = SolutionConfigurationInSolution.ComputeFullName(ConfigurationName, PlatformName); + } + + /// + /// The configuration part of this configuration - e.g. "Debug", "Release" + /// + public string ConfigurationName { get; } + + /// + /// The platform part of this configuration - e.g. "Any CPU", "Win32" + /// + public string PlatformName { get; } + + /// + /// The full name of this configuration - e.g. "Debug|Any CPU" + /// + public string FullName { get; } + + /// + /// True if this project configuration should be built as part of its parent solution configuration + /// + public bool IncludeInBuild { get; } + + /// + /// This is a hacky method to remove the space in the "Any CPU" platform in project configurations. + /// The problem is that this platform is stored as "AnyCPU" in project files, but the project system + /// reports it as "Any CPU" to the solution configuration manager. Because of that all solution configurations + /// contain the version with a space in it, and when we try and give that name to actual projects, + /// they have no clue what we're talking about. We need to remove the space in project platforms so that + /// the platform name matches the one used in projects. + /// + private static string RemoveSpaceFromAnyCpuPlatform(string platformName) + { + if (string.Equals(platformName, "Any CPU", StringComparison.OrdinalIgnoreCase)) + { + return "AnyCPU"; + } + + return platformName; + } + } +} diff --git a/src/Microsoft.Tye.Core/MsBuild/ProjectInSolution.cs b/src/Microsoft.Tye.Core/MsBuild/ProjectInSolution.cs new file mode 100644 index 00000000..1d3dd412 --- /dev/null +++ b/src/Microsoft.Tye.Core/MsBuild/ProjectInSolution.cs @@ -0,0 +1,557 @@ +// Copyright (c) Microsoft. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using System; +using System.Collections; +using System.Collections.Generic; +using System.IO; +using System.Security; +using System.Text; +using System.Xml; +using Microsoft.Build.Shared; +using System.Collections.ObjectModel; +using System.Linq; + +#pragma warning disable CS8618, CS8625, CS8601, CS8600, CS8604, CS0162, CS8603, CS0168, CS8602 + +namespace Microsoft.Build.Construction +{ + /// + /// An enumeration defining the different types of projects we might find in an SLN. + /// + public enum SolutionProjectType + { + /// + /// Everything else besides the below well-known project types. + /// + Unknown, + /// + /// C#, VB, F#, and VJ# projects + /// + KnownToBeMSBuildFormat, + /// + /// Solution folders appear in the .sln file, but aren't buildable projects. + /// + SolutionFolder, + /// + /// ASP.NET projects + /// + WebProject, + /// + /// Web Deployment (.wdproj) projects + /// + WebDeploymentProject, // MSBuildFormat, but Whidbey-era ones specify ProjectReferences differently + /// + /// Project inside an Enterprise Template project + /// + EtpSubProject, + /// + /// A shared project represents a collection of shared files that is not buildable on its own. + /// + SharedProject + } + + internal struct AspNetCompilerParameters + { + internal string aspNetVirtualPath; // For Venus projects only, Virtual path for web + internal string aspNetPhysicalPath; // For Venus projects only, Physical path for web + internal string aspNetTargetPath; // For Venus projects only, Target for output files + internal string aspNetForce; // For Venus projects only, Force overwrite of target + internal string aspNetUpdateable; // For Venus projects only, compiled web application is updateable + internal string aspNetDebug; // For Venus projects only, generate symbols, etc. + internal string aspNetKeyFile; // For Venus projects only, strong name key file. + internal string aspNetKeyContainer; // For Venus projects only, strong name key container. + internal string aspNetDelaySign; // For Venus projects only, delay sign strong name. + internal string aspNetAPTCA; // For Venus projects only, AllowPartiallyTrustedCallers. + internal string aspNetFixedNames; // For Venus projects only, generate fixed assembly names. + } + + /// + /// This class represents a project (or SLN folder) that is read in from a solution file. + /// + public sealed class ProjectInSolution + { + #region Constants + + /// + /// Characters that need to be cleansed from a project name. + /// + private static readonly char[] s_charsToCleanse = { '%', '$', '@', ';', '.', '(', ')', '\'' }; + + /// + /// Project names that need to be disambiguated when forming a target name + /// + internal static readonly string[] projectNamesToDisambiguate = { "Build", "Rebuild", "Clean", "Publish" }; + + /// + /// Character that will be used to replace 'unclean' ones. + /// + private const char cleanCharacter = '_'; + + #endregion + #region Member data + private string _relativePath; // Relative from .SLN file. For example, "WindowsApplication1\WindowsApplication1.csproj" + private string _absolutePath; // Absolute path to the project file + private readonly List _dependencies; // A list of strings representing the Guids of the dependent projects. + private IReadOnlyList _dependenciesAsReadonly; + private string _uniqueProjectName; // For example, "MySlnFolder\MySubSlnFolder\Windows_Application1" + private string _originalProjectName; // For example, "MySlnFolder\MySubSlnFolder\Windows.Application1" + + /// + /// The project configuration in given solution configuration + /// K: full solution configuration name (cfg + platform) + /// V: project configuration + /// + private readonly Dictionary _projectConfigurations; + private IReadOnlyDictionary _projectConfigurationsReadOnly; + + #endregion + + #region Constructors + + internal ProjectInSolution(SolutionFile solution) + { + ProjectType = SolutionProjectType.Unknown; + ProjectName = null; + _relativePath = null; + ProjectGuid = null; + _dependencies = new List(); + ParentProjectGuid = null; + _uniqueProjectName = null; + ParentSolution = solution; + + // default to .NET Framework 3.5 if this is an old solution that doesn't explicitly say. + TargetFrameworkMoniker = ".NETFramework,Version=v3.5"; + + // This hashtable stores a AspNetCompilerParameters struct for each configuration name supported. + AspNetConfigurations = new Hashtable(StringComparer.OrdinalIgnoreCase); + + _projectConfigurations = new Dictionary(StringComparer.OrdinalIgnoreCase); + } + + #endregion + + #region Properties + + /// + /// This project's name + /// + public string ProjectName { get; internal set; } + + /// + /// The path to this project file, relative to the solution location + /// + public string RelativePath + { + get { return _relativePath; } + internal set + { +#if NETFRAMEWORK && !MONO + // Avoid loading System.Runtime.InteropServices.RuntimeInformation in full-framework + // cases. It caused https://github.com/NuGet/Home/issues/6918. + _relativePath = value; +#else + _relativePath = FileUtilities.MaybeAdjustFilePath(value, ParentSolution.SolutionFileDirectory); +#endif + } + } + + /// + /// Returns the absolute path for this project + /// + public string AbsolutePath + { + get + { + if (_absolutePath == null) + { + _absolutePath = Path.Combine(ParentSolution.SolutionFileDirectory, _relativePath); + + // For web site projects, Visual Studio stores the URL of the site as the relative path so it cannot be normalized. + // Legacy behavior dictates that we must just return the result of Path.Combine() + if (!Uri.TryCreate(_relativePath, UriKind.Absolute, out Uri _)) + { + try + { +#if NETFRAMEWORK && !MONO + _absolutePath = Path.GetFullPath(_absolutePath); +#else + _absolutePath = FileUtilities.NormalizePath(_absolutePath); +#endif + } + catch (Exception) + { + // The call to GetFullPath() can throw if the relative path is some unsupported value or the paths are too long for the current file system + // This falls back to previous behavior of returning a path that may not be correct but at least returns some value + } + } + } + + return _absolutePath; + } + } + + /// + /// The unique guid associated with this project, in "{XXXXXXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX}" form + /// + public string ProjectGuid { get; internal set; } + + /// + /// The guid, in "{XXXXXXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX}" form, of this project's + /// parent project, if any. + /// + public string ParentProjectGuid { get; internal set; } + + /// + /// List of guids, in "{XXXXXXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX}" form, mapping to projects + /// that this project has a build order dependency on, as defined in the solution file. + /// + public IReadOnlyList Dependencies => _dependenciesAsReadonly ?? (_dependenciesAsReadonly = _dependencies.AsReadOnly()); + + /// + /// Configurations for this project, keyed off the configuration's full name, e.g. "Debug|x86" + /// They contain only the project configurations from the solution file that fully matched (configuration and platform) against the solution configurations. + /// + public IReadOnlyDictionary ProjectConfigurations + => + _projectConfigurationsReadOnly + ?? (_projectConfigurationsReadOnly = new ReadOnlyDictionary(_projectConfigurations)); + + /// + /// Extension of the project file, if any + /// + internal string Extension => Path.GetExtension(_relativePath); + + /// + /// This project's type. + /// + public SolutionProjectType ProjectType { get; set; } + + /// + /// Only applies to websites -- for other project types, references are + /// either specified as Dependencies above, or as ProjectReferences in the + /// project file, which the solution doesn't have insight into. + /// + internal List ProjectReferences { get; } = new List(); + + internal SolutionFile ParentSolution { get; set; } + + // Key is configuration name, value is [struct] AspNetCompilerParameters + internal Hashtable AspNetConfigurations { get; set; } + + internal string TargetFrameworkMoniker { get; set; } + + #endregion + + #region Methods + + private bool _checkedIfCanBeMSBuildProjectFile; + private bool _canBeMSBuildProjectFile; + private string _canBeMSBuildProjectFileErrorMessage; + + /// + /// Add the guid of a referenced project to our dependencies list. + /// + internal void AddDependency(string referencedProjectGuid) + { + _dependencies.Add(referencedProjectGuid); + _dependenciesAsReadonly = null; + } + + /// + /// Set the requested project configuration. + /// + internal void SetProjectConfiguration(string configurationName, ProjectConfigurationInSolution configuration) + { + _projectConfigurations[configurationName] = configuration; + _projectConfigurationsReadOnly = null; + } + + /// + /// Looks at the project file node and determines (roughly) if the project file is in the MSBuild format. + /// The results are cached in case this method is called multiple times. + /// + /// Detailed error message in case we encounter critical problems reading the file + /// + internal bool CanBeMSBuildProjectFile(out string errorMessage) + { + if (_checkedIfCanBeMSBuildProjectFile) + { + errorMessage = _canBeMSBuildProjectFileErrorMessage; + return _canBeMSBuildProjectFile; + } + + _checkedIfCanBeMSBuildProjectFile = true; + _canBeMSBuildProjectFile = false; + errorMessage = null; + + try + { + // Read project thru a XmlReader with proper setting to avoid DTD processing + var xrSettings = new XmlReaderSettings { DtdProcessing = DtdProcessing.Ignore }; + var projectDocument = new XmlDocument(); + + using (XmlReader xmlReader = XmlReader.Create(AbsolutePath, xrSettings)) + { + // Load the project file and get the first node + projectDocument.Load(xmlReader); + } + + XmlElement mainProjectElement = null; + + // The XML parser will guarantee that we only have one real root element, + // but we need to find it amongst the other types of XmlNode at the root. + foreach (XmlNode childNode in projectDocument.ChildNodes) + { + if (childNode.NodeType == XmlNodeType.Element) + { + mainProjectElement = (XmlElement)childNode; + break; + } + } + + if (mainProjectElement?.LocalName == "Project") + { + // MSBuild supports project files with an empty (supported in Visual Studio 2017) or the default MSBuild + // namespace. + bool emptyNamespace = string.IsNullOrEmpty(mainProjectElement.NamespaceURI); + bool defaultNamespace = String.Equals(mainProjectElement.NamespaceURI, + "http://schemas.microsoft.com/developer/msbuild/2003", + StringComparison.OrdinalIgnoreCase); + bool projectElementInvalid = ElementContainsInvalidNamespaceDefitions(mainProjectElement); + + // If the MSBuild namespace is declared, it is very likely an MSBuild project that should be built. + if (defaultNamespace) + { + _canBeMSBuildProjectFile = true; + return _canBeMSBuildProjectFile; + } + + // This is a bit of a special case, but an rptproj file will contain a Project with no schema that is + // not an MSBuild file. It will however have ToolsVersion="2.0" which is not supported with an empty + // schema. This is not a great solution, but it should cover the customer reported issue. See: + // https://github.com/Microsoft/msbuild/issues/2064 + if (emptyNamespace && !projectElementInvalid && mainProjectElement.GetAttribute("ToolsVersion") != "2.0") + { + _canBeMSBuildProjectFile = true; + return _canBeMSBuildProjectFile; + } + } + } + // catch all sorts of exceptions - if we encounter any problems here, we just assume the project file is not + // in the MSBuild format + + // handle errors in path resolution + catch (SecurityException e) + { + _canBeMSBuildProjectFileErrorMessage = e.Message; + } + // handle errors in path resolution + catch (NotSupportedException e) + { + _canBeMSBuildProjectFileErrorMessage = e.Message; + } + // handle errors in loading project file + catch (IOException e) + { + _canBeMSBuildProjectFileErrorMessage = e.Message; + } + // handle errors in loading project file + catch (UnauthorizedAccessException e) + { + _canBeMSBuildProjectFileErrorMessage = e.Message; + } + // handle XML parsing errors (when reading project file) + // this is not critical, since the project file doesn't have to be in XML formal + catch (XmlException) + { + } + + errorMessage = _canBeMSBuildProjectFileErrorMessage; + + return _canBeMSBuildProjectFile; + } + + /// + /// Find the unique name for this project, e.g. SolutionFolder\SubSolutionFolder\Project_Name + /// + internal string GetUniqueProjectName() + { + if (_uniqueProjectName == null) + { + // EtpSubProject and Venus projects have names that are already unique. No need to prepend the SLN folder. + if ((ProjectType == SolutionProjectType.WebProject) || (ProjectType == SolutionProjectType.EtpSubProject)) + { + _uniqueProjectName = CleanseProjectName(ProjectName); + } + else + { + // This is "normal" project, which in this context means anything non-Venus and non-EtpSubProject. + + // If this project has a parent SLN folder, first get the full unique name for the SLN folder, + // and tack on trailing backslash. + string uniqueName = String.Empty; + + if (ParentProjectGuid != null) + { + if (!ParentSolution.ProjectsByGuid.TryGetValue(ParentProjectGuid, out ProjectInSolution proj)) + { + if (proj == null) + { + throw new Exception(); + } + } + + uniqueName = proj.GetUniqueProjectName() + "\\"; + } + + // Now tack on our own project name, and cache it in the ProjectInSolution object for future quick access. + _uniqueProjectName = CleanseProjectName(uniqueName + ProjectName); + } + } + + return _uniqueProjectName; + } + + /// + /// Gets the original project name with the parent project as it is declared in the solution file, e.g. SolutionFolder\SubSolutionFolder\Project.Name + /// + internal string GetOriginalProjectName() + { + if (_originalProjectName == null) + { + // EtpSubProject and Venus projects have names that are already unique. No need to prepend the SLN folder. + if ((ProjectType == SolutionProjectType.WebProject) || (ProjectType == SolutionProjectType.EtpSubProject)) + { + _originalProjectName = ProjectName; + } + else + { + // This is "normal" project, which in this context means anything non-Venus and non-EtpSubProject. + + // If this project has a parent SLN folder, first get the full project name for the SLN folder, + // and tack on trailing backslash. + string projectName = String.Empty; + + if (ParentProjectGuid != null) + { + if (!ParentSolution.ProjectsByGuid.TryGetValue(ParentProjectGuid, out ProjectInSolution parent)) + { + if (parent == null) + { + throw new Exception(); + } + } + + projectName = parent.GetOriginalProjectName() + "\\"; + } + + // Now tack on our own project name, and cache it in the ProjectInSolution object for future quick access. + _originalProjectName = projectName + ProjectName; + } + } + + return _originalProjectName; + } + + internal string GetProjectGuidWithoutCurlyBrackets() + { + if (string.IsNullOrEmpty(ProjectGuid)) + { + return null; + } + + return ProjectGuid.Trim(new char[] { '{', '}' }); + } + + /// + /// Changes the unique name of the project. + /// + internal void UpdateUniqueProjectName(string newUniqueName) + { + //ErrorUtilities.VerifyThrowArgumentLength(newUniqueName, nameof(newUniqueName)); + + _uniqueProjectName = newUniqueName; + } + + /// + /// Cleanse the project name, by replacing characters like '@', '$' with '_' + /// + /// The name to be cleansed + /// string + private static string CleanseProjectName(string projectName) + { + //ErrorUtilities.VerifyThrow(projectName != null, "Null strings not allowed."); + + // If there are no special chars, just return the original string immediately. + // Don't even instantiate the StringBuilder. + int indexOfChar = projectName.IndexOfAny(s_charsToCleanse); + if (indexOfChar == -1) + { + return projectName; + } + + // This is where we're going to work on the final string to return to the caller. + var cleanProjectName = new StringBuilder(projectName); + + // Replace each unclean character with a clean one + foreach (char uncleanChar in s_charsToCleanse) + { + cleanProjectName.Replace(uncleanChar, cleanCharacter); + } + + return cleanProjectName.ToString(); + } + + /// + /// If the unique project name provided collides with one of the standard Solution project + /// entry point targets (Build, Rebuild, Clean, Publish), then disambiguate it by prepending the string "Solution:" + /// + /// The unique name for the project + /// string + internal static string DisambiguateProjectTargetName(string uniqueProjectName) + { + // Test our unique project name against those names that collide with Solution + // entry point targets + foreach (string projectName in projectNamesToDisambiguate) + { + if (String.Equals(uniqueProjectName, projectName, StringComparison.OrdinalIgnoreCase)) + { + // Prepend "Solution:" so that the collision is resolved, but the + // log of the solution project still looks reasonable. + return "Solution:" + uniqueProjectName; + } + } + + return uniqueProjectName; + } + + /// + /// Check a Project element for known invalid namespace definitions. + /// + /// Project XML Element + /// True if the element contains known invalid namespace definitions + private static bool ElementContainsInvalidNamespaceDefitions(XmlElement mainProjectElement) + { + if (mainProjectElement.HasAttributes) + { + // Data warehouse projects (.dwproj) will contain a Project element but are invalid MSBuild. Check attributes + // on Project for signs that this is a .dwproj file. If there are, it's not a valid MSBuild file. + return mainProjectElement.Attributes.OfType().Any(a => + a.Name.Equals("xmlns:dwd", StringComparison.OrdinalIgnoreCase) || + a.Name.StartsWith("xmlns:dd", StringComparison.OrdinalIgnoreCase)); + } + + return false; + } + + #endregion + + #region Constants + + internal const int DependencyLevelUnknown = -1; + internal const int DependencyLevelBeingDetermined = -2; + + #endregion + } +} diff --git a/src/Microsoft.Tye.Core/MsBuild/SolutionConfigurationInSolution.cs b/src/Microsoft.Tye.Core/MsBuild/SolutionConfigurationInSolution.cs new file mode 100644 index 00000000..d5e869cb --- /dev/null +++ b/src/Microsoft.Tye.Core/MsBuild/SolutionConfigurationInSolution.cs @@ -0,0 +1,58 @@ +// Copyright (c) Microsoft. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +namespace Microsoft.Build.Construction +{ + /// + /// This represents an entry for a solution configuration + /// + public sealed class SolutionConfigurationInSolution + { + /// + /// Default separator between configuration and platform in configuration + /// full names + /// + internal const char ConfigurationPlatformSeparator = '|'; + + internal static readonly char[] ConfigurationPlatformSeparatorArray = new char[] { '|' }; + + /// + /// Constructor + /// + internal SolutionConfigurationInSolution(string configurationName, string platformName) + { + ConfigurationName = configurationName; + PlatformName = platformName; + FullName = ComputeFullName(configurationName, platformName); + } + + /// + /// The configuration part of this configuration - e.g. "Debug", "Release" + /// + public string ConfigurationName { get; } + + /// + /// The platform part of this configuration - e.g. "Any CPU", "Win32" + /// + public string PlatformName { get; } + + /// + /// The full name of this configuration - e.g. "Debug|Any CPU" + /// + public string FullName { get; } + + /// + /// Given a configuration name and a platform name, compute the full name + /// of this configuration + /// + internal static string ComputeFullName(string configurationName, string platformName) + { + // Some configurations don't have the platform part + if (!string.IsNullOrEmpty(platformName)) + { + return $"{configurationName}{ConfigurationPlatformSeparator}{platformName}"; + } + return configurationName; + } + } +} diff --git a/src/Microsoft.Tye.Core/MsBuild/SolutionFile.cs b/src/Microsoft.Tye.Core/MsBuild/SolutionFile.cs new file mode 100644 index 00000000..1986ed76 --- /dev/null +++ b/src/Microsoft.Tye.Core/MsBuild/SolutionFile.cs @@ -0,0 +1,1266 @@ +// Copyright (c) Microsoft. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using System; +using System.Collections.Generic; +using System.Xml; +using System.IO; +using System.Text; +using System.Globalization; +using System.Security; +using System.Text.Json; +using System.Text.RegularExpressions; + +//using ErrorUtilities = Microsoft.Build.Shared.ErrorUtilities; +//using VisualStudioConstants = Microsoft.Build.Shared.VisualStudioConstants; +//using ProjectFileErrorUtilities = Microsoft.Build.Shared.ProjectFileErrorUtilities; +//using BuildEventFileInfo = Microsoft.Build.Shared.BuildEventFileInfo; +//using ResourceUtilities = Microsoft.Build.Shared.ResourceUtilities; +//using ExceptionUtilities = Microsoft.Build.Shared.ExceptionHandling; +using System.Collections.ObjectModel; +using Microsoft.Build.Shared; +using System.Runtime.InteropServices; + +#pragma warning disable CS8618, CS8625, CS8601, CS8600, CS8604, CS0162, CS8603, CS0168 + +namespace Microsoft.Build.Construction +{ + /// + /// This class contains the functionality to parse a solution file and return a corresponding + /// MSBuild project file containing the projects and dependencies defined in the solution. + /// + public sealed class SolutionFile + { + #region Solution specific constants + + // An example of a project line looks like this: + // Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "ClassLibrary1", "ClassLibrary1\ClassLibrary1.csproj", "{05A5AD00-71B5-4612-AF2F-9EA9121C4111}" + private static readonly Lazy s_crackProjectLine = new Lazy( + () => new Regex + ( + "^" // Beginning of line + + "Project\\(\"(?.*)\"\\)" + + "\\s*=\\s*" // Any amount of whitespace plus "=" plus any amount of whitespace + + "\"(?.*)\"" + + "\\s*,\\s*" // Any amount of whitespace plus "," plus any amount of whitespace + + "\"(?.*)\"" + + "\\s*,\\s*" // Any amount of whitespace plus "," plus any amount of whitespace + + "\"(?.*)\"" + + "$", // End-of-line + RegexOptions.Compiled + ) + ); + + // An example of a property line looks like this: + // AspNetCompiler.VirtualPath = "/webprecompile" + // Because website projects now include the target framework moniker as + // one of their properties, may now have '=' in it. + + private static readonly Lazy s_crackPropertyLine = new Lazy( + () => new Regex + ( + "^" // Beginning of line + + "(?[^=]*)" + + "\\s*=\\s*" // Any amount of whitespace plus "=" plus any amount of whitespace + + "(?.*)" + + "$", // End-of-line + RegexOptions.Compiled + ) + ); + + internal const int slnFileMinUpgradableVersion = 7; // Minimum version for MSBuild to give a nice message + internal const int slnFileMinVersion = 9; // Minimum version for MSBuild to actually do anything useful + internal const int slnFileMaxVersion = 12; + + private const string vbProjectGuid = "{F184B08F-C81C-45F6-A57F-5ABD9991F28F}"; + private const string csProjectGuid = "{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}"; + private const string cpsProjectGuid = "{13B669BE-BB05-4DDF-9536-439F39A36129}"; + private const string cpsCsProjectGuid = "{9A19103F-16F7-4668-BE54-9A1E7A4F7556}"; + private const string cpsVbProjectGuid = "{778DAE3C-4631-46EA-AA77-85C1314464D9}"; + private const string cpsFsProjectGuid = "{6EC3EE1D-3C4E-46DD-8F32-0CC8E7565705}"; + private const string vjProjectGuid = "{E6FDF86B-F3D1-11D4-8576-0002A516ECE8}"; + private const string vcProjectGuid = "{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}"; + private const string fsProjectGuid = "{F2A71F9B-5D33-465A-A702-920D77279786}"; + private const string dbProjectGuid = "{C8D11400-126E-41CD-887F-60BD40844F9E}"; + private const string wdProjectGuid = "{2CFEAB61-6A3B-4EB8-B523-560B4BEEF521}"; + private const string synProjectGuid = "{BBD0F5D1-1CC4-42FD-BA4C-A96779C64378}"; + private const string webProjectGuid = "{E24C65DC-7377-472B-9ABA-BC803B73C61A}"; + private const string solutionFolderGuid = "{2150E333-8FDC-42A3-9474-1A3956D46DE8}"; + private const string sharedProjectGuid = "{D954291E-2A0B-460D-934E-DC6B0785DB48}"; + + private const char CommentStartChar = '#'; + #endregion + #region Member data + private string _solutionFile; + private string _solutionFilterFile; + private HashSet _solutionFilter; + private bool _parsingForConversionOnly; + + // The list of projects in this SLN, keyed by the project GUID. + private Dictionary _projects; + + // The list of projects in the SLN, in order of their appearance in the SLN. + private List _projectsInOrder; + + // The list of solution configurations in the solution + private List _solutionConfigurations; + + // cached default configuration name for GetDefaultConfigurationName + private string _defaultConfigurationName; + + // cached default platform name for GetDefaultPlatformName + private string _defaultPlatformName; + + // VisualStudionVersion specified in Dev12+ solutions + private Version _currentVisualStudioVersion; + private int _currentLineNumber; + + #endregion + + #region Constructors + + /// + /// Constructor + /// + internal SolutionFile() + { + } + + #endregion + + #region Properties + + /// + /// This property returns the list of warnings that were generated during solution parsing + /// + internal List SolutionParserWarnings { get; } = new List(); + + /// + /// This property returns the list of comments that were generated during the solution parsing + /// + internal List SolutionParserComments { get; } = new List(); + + /// + /// This property returns the list of error codes for warnings/errors that were generated during solution parsing. + /// + internal List SolutionParserErrorCodes { get; } = new List(); + + /// + /// Returns the actual major version of the parsed solution file + /// + internal int Version { get; private set; } + + /// + /// Returns Visual Studio major version + /// + internal int VisualStudioVersion + { + get + { + if (_currentVisualStudioVersion != null) + { + return _currentVisualStudioVersion.Major; + } + else + { + return Version - 1; + } + } + } + + /// + /// Returns true if the solution contains any web projects + /// + internal bool ContainsWebProjects { get; private set; } + + /// + /// Returns true if the solution contains any .wdproj projects. Used to determine + /// whether we need to load up any projects to examine dependencies. + /// + internal bool ContainsWebDeploymentProjects { get; private set; } + + /// + /// All projects in this solution, in the order they appeared in the solution file + /// + public IReadOnlyList ProjectsInOrder => _projectsInOrder.AsReadOnly(); + + /// + /// The collection of projects in this solution, accessible by their guids as a + /// string in "{XXXXXXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX}" form + /// + public IReadOnlyDictionary ProjectsByGuid => new ReadOnlyDictionary(_projects); + + /// + /// This is the read/write accessor for the solution file which we will parse. This + /// must be set before calling any other methods on this class. + /// + /// + internal string FullPath + { + get => _solutionFile; + + set + { + // Should already be canonicalized to a full path + //ErrorUtilities.VerifyThrowInternalRooted(value); + if (FileUtilities.IsSolutionFilterFilename(value)) + { + ParseSolutionFilter(value); + } + else + { + _solutionFile = value; + _solutionFilter = null; + + SolutionFileDirectory = Path.GetDirectoryName(_solutionFile); + } + } + } + + internal string SolutionFileDirectory + { + get; + // This setter is only used by the unit tests + set; + } + + /// + /// For unit-testing only. + /// + /// + internal StreamReader SolutionReader { get; set; } + + /// + /// The list of all full solution configurations (configuration + platform) in this solution + /// + public IReadOnlyList SolutionConfigurations => _solutionConfigurations.AsReadOnly(); + + #endregion + + #region Methods + + internal bool ProjectShouldBuild(string projectFile) + { + return _solutionFilter?.Contains(FileUtilities.FixFilePath(projectFile)) != false; + } + + /// + /// This method takes a path to a solution file, parses the projects and project dependencies + /// in the solution file, and creates internal data structures representing the projects within + /// the SLN. + /// + public static SolutionFile Parse(string solutionFile) + { + var parser = new SolutionFile { FullPath = solutionFile }; + parser.ParseSolutionFile(); + return parser; + } + + /// + /// Returns "true" if it's a project that's expected to be buildable, or false if it's + /// not (e.g. a solution folder) + /// + /// The project in the solution + /// Whether the project is expected to be buildable + internal static bool IsBuildableProject(ProjectInSolution project) + { + return project.ProjectType != SolutionProjectType.SolutionFolder && project.ProjectConfigurations.Count > 0; + } + + /// + /// Given a solution file, parses the header and returns the major version numbers of the solution file + /// and the visual studio. + /// Throws InvalidProjectFileException if the solution header is invalid, or if the version is less than + /// our minimum version. + /// + internal static void GetSolutionFileAndVisualStudioMajorVersions(string solutionFile, out int solutionVersion, out int visualStudioMajorVersion) + { + //ErrorUtilities.VerifyThrow(!String.IsNullOrEmpty(solutionFile), "null solution file passed to GetSolutionFileMajorVersion!"); + //ErrorUtilities.VerifyThrowInternalRooted(solutionFile); + + const string slnFileHeaderNoVersion = "Microsoft Visual Studio Solution File, Format Version "; + const string slnFileVSVLinePrefix = "VisualStudioVersion"; + FileStream fileStream = null; + StreamReader reader = null; + bool validVersionFound = false; + + solutionVersion = 0; + visualStudioMajorVersion = 0; + + try + { + // Open the file + fileStream = File.OpenRead(solutionFile); + reader = new StreamReader(fileStream, Encoding.GetEncoding(0)); // HIGHCHAR: If solution files have no byte-order marks, then assume ANSI rather than ASCII. + + // Read first 4 lines of the solution file. + // The header is expected to be in line 1 or 2 + // VisualStudioVersion is expected to be in line 3 or 4. + for (int i = 0; i < 4; i++) + { + string line = reader.ReadLine(); + + if (line == null) + { + break; + } + + if (line.Trim().StartsWith(slnFileHeaderNoVersion, StringComparison.Ordinal)) + { + // Found it. Validate the version. + string fileVersionFromHeader = line.Substring(slnFileHeaderNoVersion.Length); + + if (!System.Version.TryParse(fileVersionFromHeader, out Version version)) + { + throw new Exception(); + //ProjectFileErrorUtilities.ThrowInvalidProjectFile + // ( + // "SubCategoryForSolutionParsingErrors", + // new BuildEventFileInfo(solutionFile), + // "SolutionParseVersionMismatchError", + // slnFileMinUpgradableVersion, + // slnFileMaxVersion + // ); + } + + solutionVersion = version.Major; + + // Validate against our min & max + if (solutionVersion < slnFileMinUpgradableVersion) + { + throw new Exception(); + } + + validVersionFound = true; + } + else if (line.Trim().StartsWith(slnFileVSVLinePrefix, StringComparison.Ordinal)) + { + Version visualStudioVersion = ParseVisualStudioVersion(line); + if (visualStudioVersion != null) + { + visualStudioMajorVersion = visualStudioVersion.Major; + } + } + } + } + finally + { + fileStream?.Dispose(); + reader?.Dispose(); + } + + if (validVersionFound) + { + return; + } + + // Didn't find the header in lines 1-4, so the solution file is invalid. + throw new Exception(); + } + + private void ParseSolutionFilter(string solutionFilterFile) + { + _solutionFilterFile = solutionFilterFile; + try + { + _solutionFile = ParseSolutionFromSolutionFilter(solutionFilterFile, out JsonElement solution); + if (!File.Exists(_solutionFile)) + { + throw new Exception(); + } + + SolutionFileDirectory = Path.GetDirectoryName(_solutionFile); + + _solutionFilter = new HashSet(RuntimeInformation.IsOSPlatform(OSPlatform.Linux) ? StringComparer.Ordinal : StringComparer.OrdinalIgnoreCase); + foreach (JsonElement project in solution.GetProperty("projects").EnumerateArray()) + { + _solutionFilter.Add(FileUtilities.FixFilePath(project.GetString())); + } + } + catch (Exception e) when (e is JsonException || e is KeyNotFoundException || e is InvalidOperationException) + { + throw; + //ProjectFileErrorUtilities.VerifyThrowInvalidProjectFile + //( + // false, /* Just throw the exception */ + // "SubCategoryForSolutionParsingErrors", + // new BuildEventFileInfo(solutionFilterFile), + // e, + // "SolutionFilterJsonParsingError", + // solutionFilterFile + //); + } + } + + internal static string ParseSolutionFromSolutionFilter(string solutionFilterFile, out JsonElement solution) + { + try + { + // This is to align MSBuild with what VS permits in loading solution filter files. These are not in them by default but can be added manually. + JsonDocumentOptions options = new JsonDocumentOptions() { AllowTrailingCommas = true, CommentHandling = JsonCommentHandling.Skip }; + JsonDocument text = JsonDocument.Parse(File.ReadAllText(solutionFilterFile), options); + solution = text.RootElement.GetProperty("solution"); + return FileUtilities.GetFullPath(solution.GetProperty("path").GetString(), Path.GetDirectoryName(solutionFilterFile)); + } + catch (Exception e) when (e is JsonException || e is KeyNotFoundException || e is InvalidOperationException) + { + throw; + //ProjectFileErrorUtilities.VerifyThrowInvalidProjectFile + //( + // false, /* Just throw the exception */ + // "SubCategoryForSolutionParsingErrors", + // new BuildEventFileInfo(solutionFilterFile), + // e, + // "SolutionFilterJsonParsingError", + // solutionFilterFile + //); + } + solution = new JsonElement(); + return string.Empty; + } + + + /// + /// Adds a configuration to this solution + /// + internal void AddSolutionConfiguration(string configurationName, string platformName) + { + _solutionConfigurations.Add(new SolutionConfigurationInSolution(configurationName, platformName)); + } + + /// + /// Reads a line from the StreamReader, trimming leading and trailing whitespace. + /// + /// + private string ReadLine() + { + string line = SolutionReader.ReadLine(); + _currentLineNumber++; + + return line?.Trim(); + } + + /// + /// This method takes a path to a solution file, parses the projects and project dependencies + /// in the solution file, and creates internal data structures representing the projects within + /// the SLN. Used for conversion, which means it allows situations that we refuse to actually build. + /// + internal void ParseSolutionFileForConversion() + { + _parsingForConversionOnly = true; + ParseSolutionFile(); + } + + /// + /// This method takes a path to a solution file, parses the projects and project dependencies + /// in the solution file, and creates internal data structures representing the projects within + /// the SLN. + /// + internal void ParseSolutionFile() + { + + FileStream fileStream = null; + SolutionReader = null; + + try + { + // Open the file + fileStream = File.OpenRead(_solutionFile); + SolutionReader = new StreamReader(fileStream, Encoding.GetEncoding(0)); // HIGHCHAR: If solution files have no byte-order marks, then assume ANSI rather than ASCII. + ParseSolution(); + } + catch (Exception e) + { + throw; + } + finally + { + fileStream?.Dispose(); + SolutionReader?.Dispose(); + } + } + + /// + /// Parses the SLN file represented by the StreamReader in this.reader, and populates internal + /// data structures based on the SLN file contents. + /// + internal void ParseSolution() + { + _projects = new Dictionary(StringComparer.OrdinalIgnoreCase); + _projectsInOrder = new List(); + ContainsWebProjects = false; + Version = 0; + _currentLineNumber = 0; + _solutionConfigurations = new List(); + _defaultConfigurationName = null; + _defaultPlatformName = null; + + // the raw list of project configurations in solution configurations, to be processed after it's fully read in. + Dictionary rawProjectConfigurationsEntries = null; + + ParseFileHeader(); + + string str; + while ((str = ReadLine()) != null) + { + if (str.StartsWith("Project(", StringComparison.Ordinal)) + { + ParseProject(str); + } + else if (str.StartsWith("GlobalSection(NestedProjects)", StringComparison.Ordinal)) + { + ParseNestedProjects(); + } + else if (str.StartsWith("GlobalSection(SolutionConfigurationPlatforms)", StringComparison.Ordinal)) + { + ParseSolutionConfigurations(); + } + else if (str.StartsWith("GlobalSection(ProjectConfigurationPlatforms)", StringComparison.Ordinal)) + { + rawProjectConfigurationsEntries = ParseProjectConfigurations(); + } + else if (str.StartsWith("VisualStudioVersion", StringComparison.Ordinal)) + { + _currentVisualStudioVersion = ParseVisualStudioVersion(str); + } + else + { + // No other section types to process at this point, so just ignore the line + // and continue. + } + } + + if (_solutionFilter != null) + { + HashSet projectPaths = new HashSet(_projectsInOrder.Count, RuntimeInformation.IsOSPlatform(OSPlatform.Linux) ? StringComparer.Ordinal : StringComparer.OrdinalIgnoreCase); + foreach (ProjectInSolution project in _projectsInOrder) + { + projectPaths.Add(FileUtilities.FixFilePath(project.RelativePath)); + } + foreach (string project in _solutionFilter) + { + if (!projectPaths.Contains(project)) + { + throw new Exception(); + //ProjectFileErrorUtilities.ThrowInvalidProjectFile + //( + // "SubCategoryForSolutionParsingErrors", + // new BuildEventFileInfo(FileUtilities.GetFullPath(project, Path.GetDirectoryName(_solutionFile))), + // "SolutionFilterFilterContainsProjectNotInSolution", + // _solutionFilterFile, + // project, + // _solutionFile + //); + } + } + } + + if (rawProjectConfigurationsEntries != null) + { + ProcessProjectConfigurationSection(rawProjectConfigurationsEntries); + } + + // Cache the unique name of each project, and check that we don't have any duplicates. + var projectsByUniqueName = new Dictionary(StringComparer.OrdinalIgnoreCase); + var projectsByOriginalName = new HashSet(StringComparer.OrdinalIgnoreCase); + + foreach (ProjectInSolution proj in _projectsInOrder) + { + // Find the unique name for the project. This method also caches the unique name, + // so it doesn't have to be recomputed later. + string uniqueName = proj.GetUniqueProjectName(); + + if (proj.ProjectType == SolutionProjectType.WebProject) + { + // Examine port information and determine if we need to disambiguate similarly-named projects with different ports. + if (Uri.TryCreate(proj.RelativePath, UriKind.Absolute, out Uri uri)) + { + if (!uri.IsDefaultPort) + { + // If there are no other projects with the same name as this one, then we will keep this project's unique name, otherwise + // we will create a new unique name with the port added. + foreach (ProjectInSolution otherProj in _projectsInOrder) + { + if (ReferenceEquals(proj, otherProj)) + { + continue; + } + + if (String.Equals(otherProj.ProjectName, proj.ProjectName, StringComparison.OrdinalIgnoreCase)) + { + uniqueName = $"{uniqueName}:{uri.Port}"; + proj.UpdateUniqueProjectName(uniqueName); + break; + } + } + } + } + } + + // Detect collision caused by unique name's normalization + if (projectsByUniqueName.TryGetValue(uniqueName, out ProjectInSolution project)) + { + // Did normalization occur in the current project? + if (uniqueName != proj.ProjectName) + { + // Generates a new unique name + string tempUniqueName = $"{uniqueName}_{proj.GetProjectGuidWithoutCurlyBrackets()}"; + proj.UpdateUniqueProjectName(tempUniqueName); + uniqueName = tempUniqueName; + } + // Did normalization occur in a previous project? + else if (uniqueName != project.ProjectName) + { + // Generates a new unique name + string tempUniqueName = $"{uniqueName}_{project.GetProjectGuidWithoutCurlyBrackets()}"; + project.UpdateUniqueProjectName(tempUniqueName); + + projectsByUniqueName.Remove(uniqueName); + projectsByUniqueName.Add(tempUniqueName, project); + } + } + + bool uniqueNameExists = projectsByUniqueName.ContainsKey(uniqueName); + + // Add the unique name (if it does not exist) to the hash table + if (!uniqueNameExists) + { + projectsByUniqueName.Add(uniqueName, proj); + } + + bool didntAlreadyExist = !uniqueNameExists && projectsByOriginalName.Add(proj.GetOriginalProjectName()); + + if (!didntAlreadyExist) + { + throw new Exception(); + } + //ProjectFileErrorUtilities.VerifyThrowInvalidProjectFile( + // didntAlreadyExist, + // "SubCategoryForSolutionParsingErrors", + // new BuildEventFileInfo(FullPath), + // "SolutionParseDuplicateProject", + // uniqueNameExists ? uniqueName : proj.ProjectName); + } + } // ParseSolutionFile() + + /// + /// This method searches the first two lines of the solution file opened by the specified + /// StreamReader for the solution file header. An exception is thrown if it is not found. + /// + /// The solution file header looks like this: + /// + /// Microsoft Visual Studio Solution File, Format Version 9.00 + /// + /// + private void ParseFileHeader() + { + //ErrorUtilities.VerifyThrow(SolutionReader != null, "ParseFileHeader(): reader is null!"); + + const string slnFileHeaderNoVersion = "Microsoft Visual Studio Solution File, Format Version "; + + // Read the file header. This can be on either of the first two lines. + for (int i = 1; i <= 2; i++) + { + string str = ReadLine(); + if (str == null) + { + break; + } + + if (str.StartsWith(slnFileHeaderNoVersion, StringComparison.Ordinal)) + { + // Found it. Validate the version. + ValidateSolutionFileVersion(str.Substring(slnFileHeaderNoVersion.Length)); + return; + } + } + + // Didn't find the header on either the first or second line, so the solution file + // is invalid. + throw new Exception(); + //ProjectFileErrorUtilities.VerifyThrowInvalidProjectFile(false, "SubCategoryForSolutionParsingErrors", + // new BuildEventFileInfo(FullPath), "SolutionParseNoHeaderError"); + } + + /// + /// This method parses the Visual Studio version in Dev 12 solution files + /// The version line looks like this: + /// + /// VisualStudioVersion = 12.0.20311.0 VSPRO_PLATFORM + /// + /// If such a line is found, the version is stored in this.currentVisualStudioVersion + /// + private static Version ParseVisualStudioVersion(string str) + { + Version currentVisualStudioVersion = null; + char[] delimiterChars = { ' ', '=' }; + string[] words = str.Split(delimiterChars, StringSplitOptions.RemoveEmptyEntries); + + if (words.Length >= 2) + { + string versionStr = words[1]; + if (!System.Version.TryParse(versionStr, out currentVisualStudioVersion)) + { + currentVisualStudioVersion = null; + } + } + + return currentVisualStudioVersion; + } + /// + /// This method extracts the whole part of the version number from the specified line + /// containing the solution file format header, and throws an exception if the version number + /// is outside of the valid range. + /// + /// The solution file header looks like this: + /// + /// Microsoft Visual Studio Solution File, Format Version 9.00 + /// + /// + /// + private void ValidateSolutionFileVersion(string versionString) + { + if (!System.Version.TryParse(versionString, out Version version)) + { + throw new Exception(); + } + + Version = version.Major; + + // Validate against our min & max + + // If the solution file version is greater than the maximum one we will create a comment rather than warn + // as users such as blend opening a dev10 project cannot do anything about it. + if (Version > slnFileMaxVersion) + { + //SolutionParserComments.Add(ResourceUtilities.FormatResourceStringStripCodeAndKeyword("UnrecognizedSolutionComment", Version)); + } + } + + /// + /// + /// This method processes a "Project" section in the solution file opened by the specified + /// StreamReader, and returns a populated ProjectInSolution instance, if successful. + /// An exception is thrown if the solution file is invalid. + /// + /// The format of the parts of a Project section that we care about is as follows: + /// + /// Project("{Project type GUID}") = "Project name", "Relative path to project file", "{Project GUID}" + /// ProjectSection(ProjectDependencies) = postProject + /// {Parent project unique name} = {Parent project unique name} + /// ... + /// EndProjectSection + /// EndProject + /// + /// + private void ParseProject(string firstLine) + { + //ErrorUtilities.VerifyThrow(!string.IsNullOrEmpty(firstLine), "ParseProject() got a null firstLine!"); + //ErrorUtilities.VerifyThrow(SolutionReader != null, "ParseProject() got a null reader!"); + + var proj = new ProjectInSolution(this); + + // Extract the important information from the first line. + ParseFirstProjectLine(firstLine, proj); + + // Search for project dependencies. Keeping reading lines until we either 1.) reach + // the end of the file, 2.) see "ProjectSection(ProjectDependencies)" at the beginning + // of the line, or 3.) see "EndProject" at the beginning of the line. + string line; + while ((line = ReadLine()) != null) + { + // If we see an "EndProject", well ... that's the end of this project! + if (line == "EndProject") + { + break; + } + else if (line.StartsWith("ProjectSection(ProjectDependencies)", StringComparison.Ordinal)) + { + // We have a ProjectDependencies section. Each subsequent line should identify + // a dependency. + line = ReadLine(); + while ((line?.StartsWith("EndProjectSection", StringComparison.Ordinal) == false)) + { + // This should be a dependency. The GUID identifying the parent project should + // be both the property name and the property value. + Match match = s_crackPropertyLine.Value.Match(line); + if (!match.Success) + { + throw new Exception(); + } + + string referenceGuid = match.Groups["PROPERTYNAME"].Value.Trim(); + proj.AddDependency(referenceGuid); + + line = ReadLine(); + } + } + else if (line.StartsWith("Project(", StringComparison.Ordinal)) + { + // The line with new project is already read and we can't go one line back - we have no choice but to recursively parse spotted project + ParseProject(line); + + // We're not waiting for the EndProject for malformed project, so we carry on + break; + } + } + + // Add the project to the collection + AddProjectToSolution(proj); + // If the project is an etp project then parse the etp project file + // to get the projects contained in it. + + } // ParseProject() + + /// + /// Adds a given project to the project collections of this class + /// + /// proj + private void AddProjectToSolution(ProjectInSolution proj) + { + if (!String.IsNullOrEmpty(proj.ProjectGuid)) + { + _projects[proj.ProjectGuid] = proj; + } + _projectsInOrder.Add(proj); + } + + /// + /// Validate relative path of a project + /// + /// proj + private void ValidateProjectRelativePath(ProjectInSolution proj) + { + // Verify the relative path is not null + if (proj.RelativePath.IndexOfAny(Path.GetInvalidPathChars()) != -1) + { + throw new Exception(); + } + } + + /// + /// Strips a single pair of leading/trailing double-quotes from a string. + /// + private static string TrimQuotes + ( + string property + ) + { + // If the incoming string starts and ends with a double-quote, strip the double-quotes. + if (!string.IsNullOrEmpty(property) && (property[0] == '"') && (property[property.Length - 1] == '"')) + { + return property.Substring(1, property.Length - 2); + } + else + { + return property; + } + } + + /// + /// Parse the first line of a Project section of a solution file. This line should look like: + /// + /// Project("{Project type GUID}") = "Project name", "Relative path to project file", "{Project GUID}" + /// + /// + /// + /// + internal void ParseFirstProjectLine + ( + string firstLine, + ProjectInSolution proj + ) + { + Match match = s_crackProjectLine.Value.Match(firstLine); + if (!match.Success) + { + throw new Exception(); + } + + string projectTypeGuid = match.Groups["PROJECTTYPEGUID"].Value.Trim(); + proj.ProjectName = match.Groups["PROJECTNAME"].Value.Trim(); + proj.RelativePath = match.Groups["RELATIVEPATH"].Value.Trim(); + proj.ProjectGuid = match.Groups["PROJECTGUID"].Value.Trim(); + + // If the project name is empty (as in some bad solutions) set it to some generated generic value. + // This allows us to at least generate reasonable target names etc. instead of crashing. + if (String.IsNullOrEmpty(proj.ProjectName)) + { + proj.ProjectName = "EmptyProjectName." + Guid.NewGuid(); + } + + // Validate project relative path + ValidateProjectRelativePath(proj); + + // Figure out what type of project this is. + if ((String.Equals(projectTypeGuid, vbProjectGuid, StringComparison.OrdinalIgnoreCase)) || + (String.Equals(projectTypeGuid, csProjectGuid, StringComparison.OrdinalIgnoreCase)) || + (String.Equals(projectTypeGuid, cpsProjectGuid, StringComparison.OrdinalIgnoreCase)) || + (String.Equals(projectTypeGuid, cpsCsProjectGuid, StringComparison.OrdinalIgnoreCase)) || + (String.Equals(projectTypeGuid, cpsVbProjectGuid, StringComparison.OrdinalIgnoreCase)) || + (String.Equals(projectTypeGuid, cpsFsProjectGuid, StringComparison.OrdinalIgnoreCase)) || + (String.Equals(projectTypeGuid, fsProjectGuid, StringComparison.OrdinalIgnoreCase)) || + (String.Equals(projectTypeGuid, dbProjectGuid, StringComparison.OrdinalIgnoreCase)) || + (String.Equals(projectTypeGuid, vjProjectGuid, StringComparison.OrdinalIgnoreCase)) || + (String.Equals(projectTypeGuid, synProjectGuid, StringComparison.OrdinalIgnoreCase))) + { + proj.ProjectType = SolutionProjectType.KnownToBeMSBuildFormat; + } + else if (String.Equals(projectTypeGuid, sharedProjectGuid, StringComparison.OrdinalIgnoreCase)) + { + proj.ProjectType = SolutionProjectType.SharedProject; + } + else if (String.Equals(projectTypeGuid, solutionFolderGuid, StringComparison.OrdinalIgnoreCase)) + { + proj.ProjectType = SolutionProjectType.SolutionFolder; + } + // MSBuild format VC projects have the same project type guid as old style VC projects. + // If it's not an old-style VC project, we'll assume it's MSBuild format + else if (String.Equals(projectTypeGuid, vcProjectGuid, StringComparison.OrdinalIgnoreCase)) + { + if (String.Equals(proj.Extension, ".vcproj", StringComparison.OrdinalIgnoreCase)) + { + if (!_parsingForConversionOnly) + { + throw new Exception(); + } + // otherwise, we're parsing this solution file because we want the P2P information during + // conversion, and it's perfectly valid for an unconverted solution file to still contain .vcprojs + } + else + { + proj.ProjectType = SolutionProjectType.KnownToBeMSBuildFormat; + } + } + else if (String.Equals(projectTypeGuid, webProjectGuid, StringComparison.OrdinalIgnoreCase)) + { + proj.ProjectType = SolutionProjectType.WebProject; + ContainsWebProjects = true; + } + else if (String.Equals(projectTypeGuid, wdProjectGuid, StringComparison.OrdinalIgnoreCase)) + { + proj.ProjectType = SolutionProjectType.WebDeploymentProject; + ContainsWebDeploymentProjects = true; + } + else + { + proj.ProjectType = SolutionProjectType.Unknown; + } + } + + /// + /// Read nested projects section. + /// This is required to find a unique name for each project's target + /// + internal void ParseNestedProjects() + { + do + { + string str = ReadLine(); + if ((str == null) || (str == "EndGlobalSection")) + { + break; + } + + // Ignore empty line or comment + if (String.IsNullOrWhiteSpace(str) || str[0] == CommentStartChar) + { + continue; + } + + Match match = s_crackPropertyLine.Value.Match(str); + if (!match.Success) + { + throw new Exception(); + } + + string projectGuid = match.Groups["PROPERTYNAME"].Value.Trim(); + string parentProjectGuid = match.Groups["PROPERTYVALUE"].Value.Trim(); + + if (!_projects.TryGetValue(projectGuid, out ProjectInSolution proj)) + { + if (proj == null) + { + throw new Exception(); + } + } + + proj.ParentProjectGuid = parentProjectGuid; + } while (true); + } + + /// + /// Read solution configuration section. + /// + /// + /// A sample section: + /// + /// GlobalSection(SolutionConfigurationPlatforms) = preSolution + /// Debug|Any CPU = Debug|Any CPU + /// Release|Any CPU = Release|Any CPU + /// EndGlobalSection + /// + internal void ParseSolutionConfigurations() + { + var nameValueSeparators = '='; + var configPlatformSeparators = new[] { SolutionConfigurationInSolution.ConfigurationPlatformSeparator }; + + do + { + string str = ReadLine(); + + if ((str == null) || (str == "EndGlobalSection")) + { + break; + } + + // Ignore empty line or comment + if (String.IsNullOrWhiteSpace(str) || str[0] == CommentStartChar) + { + continue; + } + + string[] configurationNames = str.Split(nameValueSeparators); + + // There should be exactly one '=' character, separating two names. + if (configurationNames.Length != 2) + { + throw new Exception(); + } + + string fullConfigurationName = configurationNames[0].Trim(); + + //Fixing bug 555577: Solution file can have description information, in which case we ignore. + if (String.Equals(fullConfigurationName, "DESCRIPTION", StringComparison.OrdinalIgnoreCase)) + { + continue; + } + + // Both names must be identical + if (fullConfigurationName != configurationNames[1].Trim()) + { + throw new Exception(); + } + + string[] configurationPlatformParts = fullConfigurationName.Split(configPlatformSeparators); + + if (configurationPlatformParts.Length != 2) + { + throw new Exception(); + } + + _solutionConfigurations.Add(new SolutionConfigurationInSolution(configurationPlatformParts[0], configurationPlatformParts[1])); + } while (true); + } + + /// + /// Read project configurations in solution configurations section. + /// + /// + /// A sample (incomplete) section: + /// + /// GlobalSection(ProjectConfigurationPlatforms) = postSolution + /// {6185CC21-BE89-448A-B3C0-D1C27112E595}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + /// {6185CC21-BE89-448A-B3C0-D1C27112E595}.Debug|Any CPU.Build.0 = Debug|Any CPU + /// {6185CC21-BE89-448A-B3C0-D1C27112E595}.Debug|Mixed Platforms.ActiveCfg = Release|Any CPU + /// {6185CC21-BE89-448A-B3C0-D1C27112E595}.Debug|Mixed Platforms.Build.0 = Release|Any CPU + /// {6185CC21-BE89-448A-B3C0-D1C27112E595}.Debug|Win32.ActiveCfg = Debug|Any CPU + /// {A6F99D27-47B9-4EA4-BFC9-25157CBDC281}.Release|Any CPU.ActiveCfg = Release|Win32 + /// {A6F99D27-47B9-4EA4-BFC9-25157CBDC281}.Release|Mixed Platforms.ActiveCfg = Release|Win32 + /// {A6F99D27-47B9-4EA4-BFC9-25157CBDC281}.Release|Mixed Platforms.Build.0 = Release|Win32 + /// {A6F99D27-47B9-4EA4-BFC9-25157CBDC281}.Release|Win32.ActiveCfg = Release|Win32 + /// {A6F99D27-47B9-4EA4-BFC9-25157CBDC281}.Release|Win32.Build.0 = Release|Win32 + /// EndGlobalSection + /// + /// An unprocessed hashtable of entries in this section + internal Dictionary ParseProjectConfigurations() + { + var rawProjectConfigurationsEntries = new Dictionary(StringComparer.OrdinalIgnoreCase); + + do + { + string str = ReadLine(); + + if ((str == null) || (str == "EndGlobalSection")) + { + break; + } + + // Ignore empty line or comment + if (String.IsNullOrWhiteSpace(str) || str[0] == CommentStartChar) + { + continue; + } + + string[] nameValue = str.Split('='); + + // There should be exactly one '=' character, separating the name and value. + if (nameValue.Length != 2) + { + throw new Exception(); + } + + rawProjectConfigurationsEntries[nameValue[0].Trim()] = nameValue[1].Trim(); + } while (true); + + return rawProjectConfigurationsEntries; + } + + /// + /// Read the project configuration information for every project in the solution, using pre-cached + /// solution section data. + /// + /// Cached data from the project configuration section + internal void ProcessProjectConfigurationSection(Dictionary rawProjectConfigurationsEntries) + { + // Instead of parsing the data line by line, we parse it project by project, constructing the + // entry name (e.g. "{A6F99D27-47B9-4EA4-BFC9-25157CBDC281}.Release|Any CPU.ActiveCfg") and retrieving its + // value from the raw data. The reason for this is that the IDE does it this way, and as the result + // the '.' character is allowed in configuration names although it technically separates different + // parts of the entry name string. This could lead to ambiguous results if we tried to parse + // the entry name instead of constructing it and looking it up. Although it's pretty unlikely that + // this would ever be a problem, it's safer to do it the same way VS IDE does it. + foreach (ProjectInSolution project in _projectsInOrder) + { + // Solution folders don't have configurations + if (project.ProjectType != SolutionProjectType.SolutionFolder) + { + foreach (SolutionConfigurationInSolution solutionConfiguration in _solutionConfigurations) + { + // The "ActiveCfg" entry defines the active project configuration in the given solution configuration + // This entry must be present for every possible solution configuration/project combination. + string entryNameActiveConfig = string.Format(CultureInfo.InvariantCulture, "{0}.{1}.ActiveCfg", + project.ProjectGuid, solutionConfiguration.FullName); + + // The "Build.0" entry tells us whether to build the project configuration in the given solution configuration. + // Technically, it specifies a configuration name of its own which seems to be a remnant of an initial, + // more flexible design of solution configurations (as well as the '.0' suffix - no higher values are ever used). + // The configuration name is not used, and the whole entry means "build the project configuration" + // if it's present in the solution file, and "don't build" if it's not. + string entryNameBuild = string.Format(CultureInfo.InvariantCulture, "{0}.{1}.Build.0", + project.ProjectGuid, solutionConfiguration.FullName); + + if (rawProjectConfigurationsEntries.TryGetValue(entryNameActiveConfig, out string configurationPlatform)) + { + string[] configurationPlatformParts = configurationPlatform.Split(SolutionConfigurationInSolution.ConfigurationPlatformSeparatorArray); + + // Project configuration may not necessarily contain the platform part. Some project support only the configuration part. + var projectConfiguration = new ProjectConfigurationInSolution( + configurationPlatformParts[0], + (configurationPlatformParts.Length > 1) ? configurationPlatformParts[1] : string.Empty, + rawProjectConfigurationsEntries.ContainsKey(entryNameBuild) + ); + + project.SetProjectConfiguration(solutionConfiguration.FullName, projectConfiguration); + } + } + } + } + } + + /// + /// Gets the default configuration name for this solution. Usually it's Debug, unless it's not present + /// in which case it's the first configuration name we find. + /// + public string GetDefaultConfigurationName() + { + // Have we done this already? Return the cached name + if (_defaultConfigurationName != null) + { + return _defaultConfigurationName; + } + + _defaultConfigurationName = string.Empty; + + // Pick the Debug configuration as default if present + foreach (SolutionConfigurationInSolution solutionConfiguration in SolutionConfigurations) + { + if (string.Equals(solutionConfiguration.ConfigurationName, "Debug", StringComparison.OrdinalIgnoreCase)) + { + _defaultConfigurationName = solutionConfiguration.ConfigurationName; + break; + } + } + + // Failing that, just pick the first configuration name as default + if ((_defaultConfigurationName.Length == 0) && (SolutionConfigurations.Count > 0)) + { + _defaultConfigurationName = SolutionConfigurations[0].ConfigurationName; + } + + return _defaultConfigurationName; + } + + /// + /// Gets the default platform name for this solution. Usually it's Mixed Platforms, unless it's not present + /// in which case it's the first platform name we find. + /// + public string GetDefaultPlatformName() + { + // Have we done this already? Return the cached name + if (_defaultPlatformName != null) + { + return _defaultPlatformName; + } + + _defaultPlatformName = string.Empty; + + // Pick the Mixed Platforms platform as default if present + foreach (SolutionConfigurationInSolution solutionConfiguration in SolutionConfigurations) + { + if (string.Equals(solutionConfiguration.PlatformName, "Mixed Platforms", StringComparison.OrdinalIgnoreCase)) + { + _defaultPlatformName = solutionConfiguration.PlatformName; + break; + } + // We would like this to be chosen if Mixed platforms does not exist. + else if (string.Equals(solutionConfiguration.PlatformName, "Any CPU", StringComparison.OrdinalIgnoreCase)) + { + _defaultPlatformName = solutionConfiguration.PlatformName; + } + } + + // Failing that, just pick the first platform name as default + if ((_defaultPlatformName.Length == 0) && (SolutionConfigurations.Count > 0)) + { + _defaultPlatformName = SolutionConfigurations[0].PlatformName; + } + + return _defaultPlatformName; + } + + /// + /// This method takes a string representing one of the project's unique names (guid), and + /// returns the corresponding "friendly" name for this project. + /// + /// + /// + internal string GetProjectUniqueNameByGuid(string projectGuid) + { + if (_projects.TryGetValue(projectGuid, out ProjectInSolution proj)) + { + return proj.GetUniqueProjectName(); + } + + return null; + } + + /// + /// This method takes a string representing one of the project's unique names (guid), and + /// returns the corresponding relative path to this project. + /// + /// + /// + internal string GetProjectRelativePathByGuid(string projectGuid) + { + if (_projects.TryGetValue(projectGuid, out ProjectInSolution proj)) + { + return proj.RelativePath; + } + + return null; + } + + #endregion + } // class SolutionParser +} // namespace Microsoft.Build.Construction diff --git a/src/Microsoft.Tye.Core/ProjectReader.cs b/src/Microsoft.Tye.Core/ProjectReader.cs index 553964bf..745308fb 100644 --- a/src/Microsoft.Tye.Core/ProjectReader.cs +++ b/src/Microsoft.Tye.Core/ProjectReader.cs @@ -12,20 +12,15 @@ using System.Runtime.CompilerServices; using System.Runtime.InteropServices; using System.Threading.Tasks; using Microsoft.Build.Construction; -using Microsoft.Build.Locator; using Semver; namespace Microsoft.Tye { public static class ProjectReader { - private static object @lock = new object(); - - private static bool registered; public static IEnumerable EnumerateProjects(FileInfo solutionFile) { - EnsureMSBuildRegistered(null, solutionFile); return EnumerateProjectsCore(solutionFile); } @@ -82,58 +77,6 @@ namespace Microsoft.Tye } } - private static void EnsureMSBuildRegistered(OutputContext? output, FileInfo projectFile) - { - if (!registered) - { - lock (@lock) - { - output?.WriteDebugLine("Locating .NET SDK..."); - - // It says VisualStudio - but on .NET Core, it defaults to just DotNetSdk. - // https://github.com/microsoft/MSBuildLocator/blob/v1.2.6/src/MSBuildLocator/VisualStudioInstanceQueryOptions.cs#L23 - // - // Resolve the SDK from the project directory and fall back to the global SDK. - // We're making the assumption that all of the projects want to use the same - // SDK version. This library is going load a single version of the SDK's - // assemblies into our process, so we can't use support SDKs at once without - // getting really tricky. - // - // The .NET SDK-based discovery uses `dotnet --info` and returns the SDK - // in use for the directory. - // - // https://github.com/microsoft/MSBuildLocator/blob/v1.2.6/src/MSBuildLocator/DotNetSdkLocationHelper.cs#L68 - var instance = MSBuildLocator - .QueryVisualStudioInstances(new VisualStudioInstanceQueryOptions { WorkingDirectory = projectFile.DirectoryName }) - .FirstOrDefault(); - - if (instance == null) - { - instance = MSBuildLocator - .QueryVisualStudioInstances() - .FirstOrDefault(); - } - - if (instance == null) - { - throw new CommandException($"Failed to resolve dotnet in {projectFile.Directory} or the PATH. Make sure the .NET SDK is installed and is on the PATH."); - } - - output?.WriteDebugLine("Found .NET SDK at: " + instance.MSBuildPath); - - try - { - MSBuildLocator.RegisterInstance(instance); - output?.WriteDebugLine("Registered .NET SDK."); - } - finally - { - registered = true; - } - } - } - } - // Do not load MSBuild types before using EnsureMSBuildRegistered. [MethodImpl(MethodImplOptions.NoInlining)] private static void EvaluateProject(OutputContext output, DotnetProjectServiceBuilder project, string metadataFile) diff --git a/src/Microsoft.Tye.Hosting/ProcessRunner.cs b/src/Microsoft.Tye.Hosting/ProcessRunner.cs index 2bec6e9f..57920dad 100644 --- a/src/Microsoft.Tye.Hosting/ProcessRunner.cs +++ b/src/Microsoft.Tye.Hosting/ProcessRunner.cs @@ -145,7 +145,7 @@ namespace Microsoft.Tye.Hosting _logger.LogInformation("Building projects"); - var buildResult = await ProcessUtil.RunAsync("dotnet", $"build \"{projectPath}\" /nologo", throwOnError: false, workingDirectory: application.ContextDirectory); + var buildResult = await ProcessUtil.RunAsync("dotnet", $"build --no-restore \"{projectPath}\" /nologo", throwOnError: false, workingDirectory: application.ContextDirectory); if (buildResult.ExitCode != 0) { diff --git a/src/tye/Properties/launchSettings.json b/src/tye/Properties/launchSettings.json index 91755933..e28cc71a 100644 --- a/src/tye/Properties/launchSettings.json +++ b/src/tye/Properties/launchSettings.json @@ -2,8 +2,8 @@ "profiles": { "tye": { "commandName": "Project", - "commandLineArgs": "build --framework netcoreapp3.1", - "workingDirectory": "..\\..\\samples\\app-with-targetframeworks\\" + "commandLineArgs": "run", + "workingDirectory": "..\\..\\samples\\frontend-backend\\" } } } \ No newline at end of file diff --git a/test/E2ETest/TaskExtensions.cs b/test/E2ETest/TaskExtensions.cs new file mode 100644 index 00000000..80f4bbaa --- /dev/null +++ b/test/E2ETest/TaskExtensions.cs @@ -0,0 +1,151 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Diagnostics; +using System.Diagnostics.CodeAnalysis; +using System.Runtime.CompilerServices; +using System.Threading; +using System.Threading.Tasks; +#pragma warning disable CS8625 // Cannot convert null literal to non-nullable reference type. + +#if AspNetCoreTesting +namespace Microsoft.AspNetCore.Testing +#else +namespace System.Threading.Tasks.Extensions +#endif +{ + +#if AspNetCoreTesting + public +#else + internal +#endif + static class TaskExtensions + { +#if DEBUG + // Shorter duration when running tests with debug. + // Less time waiting for hang unit tests to fail in aspnetcore solution. + private const int DefaultTimeoutDuration = 5 * 1000; +#else + private const int DefaultTimeoutDuration = 30 * 1000; +#endif + + public static TimeSpan DefaultTimeoutTimeSpan { get; } = TimeSpan.FromMilliseconds(DefaultTimeoutDuration); + + public static Task DefaultTimeout(this Task task, int milliseconds = DefaultTimeoutDuration, [CallerFilePath] string filePath = null, [CallerLineNumber] int lineNumber = default) + { + return task.TimeoutAfter(TimeSpan.FromMilliseconds(milliseconds), filePath, lineNumber); + } + + public static Task DefaultTimeout(this Task task, TimeSpan timeout, [CallerFilePath] string filePath = null, [CallerLineNumber] int lineNumber = default) + { + return task.TimeoutAfter(timeout, filePath, lineNumber); + } + + public static Task DefaultTimeout(this ValueTask task, int milliseconds = DefaultTimeoutDuration, [CallerFilePath] string filePath = null, [CallerLineNumber] int lineNumber = default) + { + return task.AsTask().TimeoutAfter(TimeSpan.FromMilliseconds(milliseconds), filePath, lineNumber); + } + + public static Task DefaultTimeout(this ValueTask task, TimeSpan timeout, [CallerFilePath] string filePath = null, [CallerLineNumber] int lineNumber = default) + { + return task.AsTask().TimeoutAfter(timeout, filePath, lineNumber); + } + + public static Task DefaultTimeout(this Task task, int milliseconds = DefaultTimeoutDuration, [CallerFilePath] string filePath = null, [CallerLineNumber] int lineNumber = default) + { + return task.TimeoutAfter(TimeSpan.FromMilliseconds(milliseconds), filePath, lineNumber); + } + + public static Task DefaultTimeout(this Task task, TimeSpan timeout, [CallerFilePath] string filePath = null, [CallerLineNumber] int lineNumber = default) + { + return task.TimeoutAfter(timeout, filePath, lineNumber); + } + + public static Task DefaultTimeout(this ValueTask task, int milliseconds = DefaultTimeoutDuration, [CallerFilePath] string filePath = null, [CallerLineNumber] int lineNumber = default) + { + return task.AsTask().TimeoutAfter(TimeSpan.FromMilliseconds(milliseconds), filePath, lineNumber); + } + + public static Task DefaultTimeout(this ValueTask task, TimeSpan timeout, [CallerFilePath] string filePath = null, [CallerLineNumber] int lineNumber = default) + { + return task.AsTask().TimeoutAfter(timeout, filePath, lineNumber); + } + + + [SuppressMessage("ApiDesign", "RS0026:Do not add multiple public overloads with optional parameters", Justification = "Required to maintain compatibility")] + public static async Task TimeoutAfter(this Task task, TimeSpan timeout, + [CallerFilePath] string filePath = null, + [CallerLineNumber] int lineNumber = default) + { + // Don't create a timer if the task is already completed + // or the debugger is attached + if (task.IsCompleted || Debugger.IsAttached) + { + return await task; + } +#if NET6_0_OR_GREATER + try + { + return await task.WaitAsync(timeout); + } + catch (TimeoutException ex) when (ex.Source == typeof(TaskExtensions).Namespace) + { + throw new TimeoutException(CreateMessage(timeout, filePath, lineNumber)); + } +#else + var cts = new CancellationTokenSource(); + if (task == await Task.WhenAny(task, Task.Delay(timeout, cts.Token))) + { + cts.Cancel(); + return await task; + } + else + { + throw new TimeoutException(CreateMessage(timeout, filePath, lineNumber)); + } +#endif + } + + [SuppressMessage("ApiDesign", "RS0026:Do not add multiple public overloads with optional parameters", Justification = "Required to maintain compatibility")] + public static async Task TimeoutAfter(this Task task, TimeSpan timeout, + [CallerFilePath] string filePath = null, + [CallerLineNumber] int lineNumber = default) + { + // Don't create a timer if the task is already completed + // or the debugger is attached + if (task.IsCompleted || Debugger.IsAttached) + { + await task; + return; + } +#if NET6_0_OR_GREATER + try + { + await task.WaitAsync(timeout); + } + catch (TimeoutException ex) when (ex.Source == typeof(TaskExtensions).Namespace) + { + throw new TimeoutException(CreateMessage(timeout, filePath, lineNumber)); + } +#else + var cts = new CancellationTokenSource(); + if (task == await Task.WhenAny(task, Task.Delay(timeout, cts.Token))) + { + cts.Cancel(); + await task; + } + else + { + throw new TimeoutException(CreateMessage(timeout, filePath, lineNumber)); + } +#endif + } + + private static string CreateMessage(TimeSpan timeout, string filePath, int lineNumber) + => string.IsNullOrEmpty(filePath) + ? $"The operation timed out after reaching the limit of {timeout.TotalMilliseconds}ms." + : $"The operation at {filePath}:{lineNumber} timed out after reaching the limit of {timeout.TotalMilliseconds}ms."; + } +} diff --git a/test/E2ETest/TyeRunTests.cs b/test/E2ETest/TyeRunTests.cs index 19c13e63..95193fbf 100644 --- a/test/E2ETest/TyeRunTests.cs +++ b/test/E2ETest/TyeRunTests.cs @@ -25,7 +25,6 @@ using static Test.Infrastructure.TestHelpers; namespace E2ETest { - public class TyeRunTests { private readonly ITestOutputHelper _output;