Browse Source

Migrate to the .NET 11 SDK, add a net11.0 TFM and update all the dependencies

release/8.0.0-preview.6
Kévin Chalet 4 days ago
parent
commit
04d7bfdb03
  1. 19
      Directory.Build.props
  2. 2
      Directory.Build.targets
  3. 141
      Directory.Packages.props
  4. 34
      WorkloadRollback.json
  5. 2
      eng/Tools.props
  6. 4
      eng/Version.Details.xml
  7. 17
      eng/common/SetupNugetSources.ps1
  8. 17
      eng/common/SetupNugetSources.sh
  9. 2
      eng/common/build.sh
  10. 8
      eng/common/core-templates/job/job.yml
  11. 29
      eng/common/core-templates/job/publish-build-assets.yml
  12. 8
      eng/common/core-templates/job/source-build.yml
  13. 8
      eng/common/core-templates/job/source-index-stage1.yml
  14. 466
      eng/common/core-templates/post-build/post-build.yml
  15. 2
      eng/common/core-templates/steps/generate-sbom.yml
  16. 34
      eng/common/core-templates/steps/install-microbuild-impl.yml
  17. 94
      eng/common/core-templates/steps/install-microbuild.yml
  18. 4
      eng/common/core-templates/steps/publish-logs.yml
  19. 8
      eng/common/core-templates/steps/source-index-stage1-publish.yml
  20. 11
      eng/common/cross/arm/sources.list.bionic
  21. 11
      eng/common/cross/arm/sources.list.focal
  22. 11
      eng/common/cross/arm/sources.list.jammy
  23. 3
      eng/common/cross/arm/sources.list.jessie
  24. 11
      eng/common/cross/arm/sources.list.trusty
  25. 11
      eng/common/cross/arm/sources.list.xenial
  26. 11
      eng/common/cross/arm/sources.list.zesty
  27. 35
      eng/common/cross/arm/tizen-build-rootfs.sh
  28. 170
      eng/common/cross/arm/tizen-fetch.sh
  29. 71
      eng/common/cross/arm/trusty-lttng-2.4.patch
  30. 97
      eng/common/cross/arm/trusty.patch
  31. 11
      eng/common/cross/arm64/sources.list.bionic
  32. 11
      eng/common/cross/arm64/sources.list.buster
  33. 11
      eng/common/cross/arm64/sources.list.focal
  34. 11
      eng/common/cross/arm64/sources.list.jammy
  35. 12
      eng/common/cross/arm64/sources.list.stretch
  36. 11
      eng/common/cross/arm64/sources.list.trusty
  37. 11
      eng/common/cross/arm64/sources.list.xenial
  38. 11
      eng/common/cross/arm64/sources.list.zesty
  39. 35
      eng/common/cross/arm64/tizen-build-rootfs.sh
  40. 170
      eng/common/cross/arm64/tizen-fetch.sh
  41. 43
      eng/common/cross/armel/armel.jessie.patch
  42. 3
      eng/common/cross/armel/sources.list.jessie
  43. 35
      eng/common/cross/armel/tizen-build-rootfs.sh
  44. 170
      eng/common/cross/armel/tizen-fetch.sh
  45. 50
      eng/common/cross/armel/tizen/tizen-dotnet.ks
  46. 2
      eng/common/cross/armv6/sources.list.buster
  47. 8
      eng/common/cross/build-rootfs.sh
  48. 11
      eng/common/cross/ppc64le/sources.list.bionic
  49. 1
      eng/common/cross/riscv64/sources.list.sid
  50. 11
      eng/common/cross/s390x/sources.list.bionic
  51. 11
      eng/common/cross/x86/sources.list.bionic
  52. 11
      eng/common/cross/x86/sources.list.focal
  53. 11
      eng/common/cross/x86/sources.list.jammy
  54. 11
      eng/common/cross/x86/sources.list.trusty
  55. 11
      eng/common/cross/x86/sources.list.xenial
  56. 35
      eng/common/cross/x86/tizen-build-rootfs.sh
  57. 170
      eng/common/cross/x86/tizen-fetch.sh
  58. 2
      eng/common/darc-init.sh
  59. 774
      eng/common/dotnet-install-scripts/dotnet-install.ps1
  60. 1133
      eng/common/dotnet-install-scripts/dotnet-install.sh
  61. 2
      eng/common/dotnet-install.sh
  62. 2
      eng/common/dotnet.sh
  63. 86
      eng/common/generate-graph-files.ps1
  64. 2
      eng/common/internal-feed-operations.ps1
  65. 2
      eng/common/internal-feed-operations.sh
  66. 121
      eng/common/native/find-native-compiler.sh
  67. 11
      eng/common/native/install-dependencies.sh
  68. 30
      eng/common/performance/blazor_perf.proj
  69. 69
      eng/common/performance/crossgen_perf.proj
  70. 144
      eng/common/performance/microbenchmarks.proj
  71. 121
      eng/common/performance/perfhelixpublish.proj
  72. 147
      eng/common/performance/performance-setup.ps1
  73. 289
      eng/common/performance/performance-setup.sh
  74. 48
      eng/common/post-build/add-build-to-channel.ps1
  75. 2
      eng/common/post-build/nuget-verification.ps1
  76. 91
      eng/common/post-build/post-build-utils.ps1
  77. 4
      eng/common/post-build/publish-using-darc.ps1
  78. 4
      eng/common/post-build/redact-logs.ps1
  79. 64
      eng/common/post-build/trigger-subscriptions.ps1
  80. 4
      eng/common/sdk-task.ps1
  81. 69
      eng/common/sdl/push-gdn.ps1
  82. 139
      eng/common/templates/job/execute-sdl.yml
  83. 48
      eng/common/templates/job/generate-graph-files.yml
  84. 95
      eng/common/templates/job/performance.yml
  85. 130
      eng/common/templates/phases/base.yml
  86. 52
      eng/common/templates/phases/publish-build-assets.yml
  87. 190
      eng/common/templates/post-build/channels/generic-internal-channel.yml
  88. 192
      eng/common/templates/post-build/channels/generic-public-channel.yml
  89. 13
      eng/common/templates/post-build/trigger-subscription.yml
  90. 13
      eng/common/templates/steps/add-build-to-channel.yml
  91. 12
      eng/common/templates/steps/build-reason.yml
  92. 32
      eng/common/templates/steps/execute-codeql.yml
  93. 88
      eng/common/templates/steps/execute-sdl.yml
  94. 50
      eng/common/templates/steps/perf-send-to-helix.yml
  95. 7
      eng/common/templates/steps/run-on-unix.yml
  96. 7
      eng/common/templates/steps/run-on-windows.yml
  97. 33
      eng/common/templates/steps/run-script-ifequalelse.yml
  98. 102
      eng/common/templates/steps/telemetry-end.yml
  99. 241
      eng/common/templates/steps/telemetry-start.yml
  100. 2
      eng/common/templates/variables/pool-providers.yml

19
Directory.Build.props

@ -108,33 +108,40 @@
</NetFrameworkTargetFrameworks>
<NetCoreTargetFrameworks Condition=" '$(NetCoreTargetFrameworks)' == '' ">
net10.0
net10.0;
net11.0
</NetCoreTargetFrameworks>
<NetCoreAndroidTargetFrameworks
Condition=" '$(NetCoreAndroidTargetFrameworks)' == '' And '$(SupportsAndroidTargeting)' == 'true' ">
net10.0-android
net10.0-android;
net11.0-android
</NetCoreAndroidTargetFrameworks>
<NetCoreIOSTargetFrameworks
Condition=" '$(NetCoreIOSTargetFrameworks)' == '' And '$(SupportsIOSTargeting)' == 'true' ">
net10.0-ios
net10.0-ios;
net11.0-ios
</NetCoreIOSTargetFrameworks>
<NetCoreMacCatalystTargetFrameworks
Condition=" '$(NetCoreMacCatalystTargetFrameworks)' == '' And '$(SupportsMacCatalystTargeting)' == 'true' ">
net10.0-maccatalyst
net10.0-maccatalyst;
net11.0-maccatalyst
</NetCoreMacCatalystTargetFrameworks>
<NetCoreMacOSTargetFrameworks
Condition=" '$(NetCoreMacOSTargetFrameworks)' == '' And '$(SupportsMacOSTargeting)' == 'true' ">
net10.0-macos
net10.0-macos;
net11.0-macos
</NetCoreMacOSTargetFrameworks>
<NetCoreWindowsTargetFrameworks
Condition=" '$(NetCoreWindowsTargetFrameworks)' == '' And '$(SupportsWindowsTargeting)' == 'true' ">
net10.0-windows7.0;
net10.0-windows10.0.17763
net10.0-windows10.0.17763;
net11.0-windows7.0;
net11.0-windows10.0.17763
</NetCoreWindowsTargetFrameworks>
</PropertyGroup>

2
Directory.Build.targets

@ -34,7 +34,7 @@
</PropertyGroup>
<PropertyGroup Condition=" '$(TargetFrameworkIdentifier)' == '.NETCoreApp' ">
<SupportedOSPlatformVersion Condition=" '$(TargetPlatformIdentifier)' == 'Android' ">21.0</SupportedOSPlatformVersion>
<SupportedOSPlatformVersion Condition=" '$(TargetPlatformIdentifier)' == 'Android' ">24.0</SupportedOSPlatformVersion>
<SupportedOSPlatformVersion Condition=" '$(TargetPlatformIdentifier)' == 'iOS' ">12.2</SupportedOSPlatformVersion>
<SupportedOSPlatformVersion Condition=" '$(TargetPlatformIdentifier)' == 'MacCatalyst' ">15.0</SupportedOSPlatformVersion>
<SupportedOSPlatformVersion Condition=" '$(TargetPlatformIdentifier)' == 'macOS' ">12.0</SupportedOSPlatformVersion>

141
Directory.Packages.props

@ -27,41 +27,41 @@
Condition=" '$(TargetFrameworkIdentifier)' == '.NETFramework' And $([MSBuild]::VersionEquals($(TargetFrameworkVersion), '4.8.0')) ">
<PackageVersion Include="EntityFramework" Version="6.5.2" />
<PackageVersion Include="BouncyCastle.Cryptography" Version="2.7.0" />
<PackageVersion Include="Microsoft.AspNetCore.DataProtection" Version="10.0.14" />
<PackageVersion Include="Microsoft.Bcl.Cryptography" Version="10.0.14" />
<PackageVersion Include="Microsoft.AspNetCore.DataProtection" Version="10.0.15" />
<PackageVersion Include="Microsoft.Bcl.Cryptography" Version="10.0.15" />
<PackageVersion Include="Microsoft.Bcl.HashCode" Version="6.0.0" />
<PackageVersion Include="Microsoft.Bcl.Memory" Version="10.0.14" />
<PackageVersion Include="Microsoft.Extensions.Caching.Memory" Version="10.0.14" />
<PackageVersion Include="Microsoft.Extensions.DependencyInjection.Abstractions" Version="10.0.14" />
<PackageVersion Include="Microsoft.Extensions.Hosting.Abstractions" Version="10.0.14" />
<PackageVersion Include="Microsoft.Extensions.Http.Polly" Version="10.0.14" />
<PackageVersion Include="Microsoft.Extensions.Logging" Version="10.0.14" />
<PackageVersion Include="Microsoft.Extensions.Options" Version="10.0.14" />
<PackageVersion Include="Microsoft.Extensions.Primitives" Version="10.0.14" />
<PackageVersion Include="Microsoft.Extensions.WebEncoders" Version="10.0.14" />
<PackageVersion Include="Microsoft.Bcl.Memory" Version="10.0.15" />
<PackageVersion Include="Microsoft.Extensions.Caching.Memory" Version="10.0.15" />
<PackageVersion Include="Microsoft.Extensions.DependencyInjection.Abstractions" Version="10.0.15" />
<PackageVersion Include="Microsoft.Extensions.Hosting.Abstractions" Version="10.0.15" />
<PackageVersion Include="Microsoft.Extensions.Http.Polly" Version="10.0.15" />
<PackageVersion Include="Microsoft.Extensions.Logging" Version="10.0.15" />
<PackageVersion Include="Microsoft.Extensions.Options" Version="10.0.15" />
<PackageVersion Include="Microsoft.Extensions.Primitives" Version="10.0.15" />
<PackageVersion Include="Microsoft.Extensions.WebEncoders" Version="10.0.15" />
<PackageVersion Include="Microsoft.IdentityModel.JsonWebTokens" Version="8.23.0" />
<PackageVersion Include="Microsoft.IdentityModel.Protocols" Version="8.23.0" />
<PackageVersion Include="Microsoft.IdentityModel.Tokens" Version="8.23.0" />
<PackageVersion Include="Microsoft.Owin.Security" Version="4.2.3" />
<PackageVersion Include="Microsoft.Windows.SDK.Contracts" Version="10.0.17763.1000" />
<PackageVersion Include="MongoDB.Bson" Version="3.13.0" />
<PackageVersion Include="MongoDB.Driver" Version="3.13.0" />
<PackageVersion Include="MongoDB.Bson" Version="3.14.0" />
<PackageVersion Include="MongoDB.Driver" Version="3.14.0" />
<PackageVersion Include="MSBuild.Microsoft.VisualStudio.Web.targets" Version="14.0.0.3" />
<PackageVersion Include="NuGet.Build.Tasks.Pack" Version="6.14.0" />
<PackageVersion Include="SharpCompress" Version="1.0.0" />
<PackageVersion Include="Snappier" Version="1.3.1" />
<PackageVersion Include="System.Buffers" Version="4.6.1" />
<PackageVersion Include="System.Collections.Immutable" Version="10.0.14" />
<PackageVersion Include="System.Net.Http.Json" Version="10.0.14" />
<PackageVersion Include="System.Security.Cryptography.Xml" Version="10.0.14" />
<PackageVersion Include="System.Text.Json" Version="10.0.14" />
<PackageVersion Include="System.Collections.Immutable" Version="10.0.15" />
<PackageVersion Include="System.Net.Http.Json" Version="10.0.15" />
<PackageVersion Include="System.Security.Cryptography.Xml" Version="10.0.15" />
<PackageVersion Include="System.Text.Json" Version="10.0.15" />
<!--
Note: the following references are exclusively used in the test projects:
-->
<PackageVersion Include="AngleSharp" Version="1.9.0" />
<PackageVersion Include="AngleSharp" Version="1.10.3" />
<PackageVersion Include="MartinCostello.Logging.XUnit" Version="0.7.1" />
<PackageVersion Include="Microsoft.Extensions.DependencyInjection" Version="10.0.14" />
<PackageVersion Include="Microsoft.Extensions.DependencyInjection" Version="10.0.15" />
<PackageVersion Include="Microsoft.Extensions.TimeProvider.Testing" Version="10.11.0" />
<PackageVersion Include="Microsoft.Owin.Testing" Version="4.2.3" />
<PackageVersion Include="Moq" Version="4.18.4" />
@ -85,8 +85,8 @@
<PackageVersion Include="Microsoft.AspNet.Web.Optimization" Version="1.1.3" />
<PackageVersion Include="Microsoft.AspNet.WebApi.Owin" Version="5.3.0" />
<PackageVersion Include="Microsoft.CodeDom.Providers.DotNetCompilerPlatform" Version="4.1.0" />
<PackageVersion Include="Microsoft.Extensions.Hosting" Version="10.0.14" />
<PackageVersion Include="Microsoft.Net.Compilers.Toolset" Version="5.9.0" />
<PackageVersion Include="Microsoft.Extensions.Hosting" Version="10.0.15" />
<PackageVersion Include="Microsoft.Net.Compilers.Toolset" Version="5.12.0" />
<PackageVersion Include="Microsoft.Owin.Host.SystemWeb" Version="4.2.3" />
<PackageVersion Include="Microsoft.Owin.Security.Cookies" Version="4.2.3" />
<PackageVersion Include="Microsoft.Owin.Security.OAuth" Version="4.2.3" />
@ -110,36 +110,79 @@
<ItemGroup Label="Package versions for .NET 10.0"
Condition=" '$(TargetFrameworkIdentifier)' == '.NETCoreApp' And $([MSBuild]::VersionEquals($(TargetFrameworkVersion), '10.0')) ">
<PackageVersion Include="EntityFramework" Version="6.5.2" />
<PackageVersion Include="Microsoft.AspNetCore.DataProtection" Version="10.0.14" />
<PackageVersion Include="Microsoft.EntityFrameworkCore.Relational" Version="10.0.14" />
<PackageVersion Include="Microsoft.Extensions.Caching.Memory" Version="10.0.14" />
<PackageVersion Include="Microsoft.Extensions.DependencyInjection.Abstractions" Version="10.0.14" />
<PackageVersion Include="Microsoft.Extensions.Hosting.Abstractions" Version="10.0.14" />
<PackageVersion Include="Microsoft.Extensions.Http.Polly" Version="10.0.14" />
<PackageVersion Include="Microsoft.AspNetCore.DataProtection" Version="10.0.15" />
<PackageVersion Include="Microsoft.EntityFrameworkCore.Relational" Version="10.0.15" />
<PackageVersion Include="Microsoft.Extensions.Caching.Memory" Version="10.0.15" />
<PackageVersion Include="Microsoft.Extensions.DependencyInjection.Abstractions" Version="10.0.15" />
<PackageVersion Include="Microsoft.Extensions.Hosting.Abstractions" Version="10.0.15" />
<PackageVersion Include="Microsoft.Extensions.Http.Polly" Version="10.0.15" />
<PackageVersion Include="Microsoft.Extensions.Http.Resilience" Version="10.11.0" />
<PackageVersion Include="Microsoft.IdentityModel.JsonWebTokens" Version="8.23.0" />
<PackageVersion Include="Microsoft.IdentityModel.Protocols" Version="8.23.0" />
<PackageVersion Include="Microsoft.IdentityModel.Tokens" Version="8.23.0" />
<PackageVersion Include="Microsoft.Extensions.Logging" Version="10.0.14" />
<PackageVersion Include="Microsoft.Extensions.Options" Version="10.0.14" />
<PackageVersion Include="Microsoft.Extensions.Primitives" Version="10.0.14" />
<PackageVersion Include="MongoDB.Bson" Version="3.13.0" />
<PackageVersion Include="MongoDB.Driver" Version="3.13.0" />
<PackageVersion Include="Microsoft.Extensions.Logging" Version="10.0.15" />
<PackageVersion Include="Microsoft.Extensions.Options" Version="10.0.15" />
<PackageVersion Include="Microsoft.Extensions.Primitives" Version="10.0.15" />
<PackageVersion Include="MongoDB.Bson" Version="3.14.0" />
<PackageVersion Include="MongoDB.Driver" Version="3.14.0" />
<PackageVersion Include="SharpCompress" Version="1.0.0" />
<PackageVersion Include="Snappier" Version="1.3.1" />
<PackageVersion Include="System.Security.Cryptography.Xml" Version="10.0.14" />
<PackageVersion Include="System.Security.Cryptography.Xml" Version="10.0.15" />
<PackageVersion Include="Xamarin.AndroidX.Browser" Version="1.10.0.1" />
<!--
Note: the following references are exclusively used in the test projects:
-->
<PackageVersion Include="AngleSharp" Version="1.9.0" />
<PackageVersion Include="AngleSharp" Version="1.10.3" />
<PackageVersion Include="MartinCostello.Logging.XUnit" Version="0.7.1" />
<PackageVersion Include="Microsoft.AspNetCore.TestHost" Version="10.0.14" />
<PackageVersion Include="Microsoft.Extensions.DependencyInjection" Version="10.0.14" />
<PackageVersion Include="Microsoft.AspNetCore.TestHost" Version="10.0.15" />
<PackageVersion Include="Microsoft.Extensions.DependencyInjection" Version="10.0.15" />
<PackageVersion Include="Microsoft.Extensions.TimeProvider.Testing" Version="10.11.0" />
<PackageVersion Include="Moq" Version="4.18.4" />
<PackageVersion Include="Xunit.SkippableFact" Version="1.5.85" />
</ItemGroup>
<!--
▄▄▄▄▄▄▄▄▄▄▄▄▄▄▄▄▄▄▄▄▄▄▄▄▄▄▄▄▄▄▄▄▄▄▄▄▄▄▄▄▄▄
█████ ▀██ ██ ▄▄▄█▄▄ ▄▄███▀ ██▀ █████ ▄▄ ██
█▀▀██ █ █ ██ ▄▄▄███ ██████ ███ ██▀▀█ ▀▄ ██
█▄▄██ ██▄ ██ ▀▀▀███ █████▀ ▀█▀ ▀█▄▄█ ▀▀ ██
▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀▀
-->
<ItemGroup Label="Package versions for .NET 11.0"
Condition=" '$(TargetFrameworkIdentifier)' == '.NETCoreApp' And $([MSBuild]::VersionEquals($(TargetFrameworkVersion), '11.0')) ">
<PackageVersion Include="EntityFramework" Version="6.5.2" />
<PackageVersion Include="Microsoft.AspNetCore.DataProtection" Version="11.0.0" />
<PackageVersion Include="Microsoft.EntityFrameworkCore.Relational" Version="11.0.0" />
<PackageVersion Include="Microsoft.Extensions.Caching.Memory" Version="11.0.0" />
<PackageVersion Include="Microsoft.Extensions.DependencyInjection.Abstractions" Version="11.0.0" />
<PackageVersion Include="Microsoft.Extensions.Hosting.Abstractions" Version="11.0.0" />
<PackageVersion Include="Microsoft.Extensions.Http.Polly" Version="11.0.0" />
<PackageVersion Include="Microsoft.Extensions.Http.Resilience" Version="10.11.0" />
<PackageVersion Include="Microsoft.IdentityModel.JsonWebTokens" Version="8.23.0" />
<PackageVersion Include="Microsoft.IdentityModel.Protocols" Version="8.23.0" />
<PackageVersion Include="Microsoft.IdentityModel.Tokens" Version="8.23.0" />
<PackageVersion Include="Microsoft.Extensions.Logging" Version="11.0.0" />
<PackageVersion Include="Microsoft.Extensions.Options" Version="11.0.0" />
<PackageVersion Include="Microsoft.Extensions.Primitives" Version="11.0.0" />
<PackageVersion Include="MongoDB.Bson" Version="3.14.0" />
<PackageVersion Include="MongoDB.Driver" Version="3.14.0" />
<PackageVersion Include="SharpCompress" Version="1.0.0" />
<PackageVersion Include="Snappier" Version="1.3.1" />
<PackageVersion Include="System.Security.Cryptography.Xml" Version="11.0.0" />
<PackageVersion Include="Xamarin.AndroidX.Browser" Version="1.10.0.1" />
<!--
Note: the following references are exclusively used in the test projects:
-->
<PackageVersion Include="AngleSharp" Version="1.10.3" />
<PackageVersion Include="MartinCostello.Logging.XUnit" Version="0.7.1" />
<PackageVersion Include="Microsoft.AspNetCore.TestHost" Version="11.0.0" />
<PackageVersion Include="Microsoft.Extensions.DependencyInjection" Version="11.0.0" />
<PackageVersion Include="Microsoft.Extensions.TimeProvider.Testing" Version="11.0.0" />
<PackageVersion Include="Moq" Version="4.18.4" />
<PackageVersion Include="Xunit.SkippableFact" Version="1.5.85" />
<!--
Note: the following references are exclusively used in the samples:
@ -147,14 +190,12 @@
<PackageVersion Include="Dapplo.Microsoft.Extensions.Hosting.AppServices" Version="1.0.14" />
<PackageVersion Include="Dapplo.Microsoft.Extensions.Hosting.WinForms" Version="1.0.14" />
<PackageVersion Include="Dapplo.Microsoft.Extensions.Hosting.Wpf" Version="1.0.14" />
<PackageVersion Include="Microsoft.AspNetCore.Identity.EntityFrameworkCore" Version="10.0.14" />
<PackageVersion Include="Microsoft.EntityFrameworkCore.Sqlite" Version="10.0.14" />
<PackageVersion Include="Microsoft.EntityFrameworkCore.SqlServer" Version="10.0.14" />
<PackageVersion Include="Microsoft.Extensions.Hosting" Version="10.0.14" />
<PackageVersion Include="Microsoft.Maui.Controls" Version="10.0.120" />
<PackageVersion Include="Microsoft.Maui.Controls.Compatibility" Version="10.0.120" />
<PackageVersion Include="Microsoft.AspNetCore.Identity.EntityFrameworkCore" Version="11.0.0" />
<PackageVersion Include="Microsoft.EntityFrameworkCore.Sqlite" Version="11.0.0" />
<PackageVersion Include="Microsoft.EntityFrameworkCore.SqlServer" Version="11.0.0" />
<PackageVersion Include="Microsoft.Extensions.Hosting" Version="11.0.0" />
<PackageVersion Include="Microsoft.Maui.Controls" Version="11.0.10" />
<PackageVersion Include="Spectre.Console" Version="0.57.2" />
<PackageVersion Include="System.Security.Cryptography.Pkcs" Version="10.0.14" />
</ItemGroup>
<!--
@ -172,13 +213,13 @@
-->
<PackageVersion Include="Fluid.Core" Version="3.0.0-beta.9" />
<PackageVersion Include="Fluid.SourceGenerator" Version="3.0.0-beta.9" />
<PackageVersion Include="Microsoft.Bcl.AsyncInterfaces" Version="10.0.14" />
<PackageVersion Include="Microsoft.Bcl.Memory" Version="10.0.14" />
<PackageVersion Include="Microsoft.Extensions.FileProviders.Abstractions" Version="10.0.14" />
<PackageVersion Include="Microsoft.CodeAnalysis.Analyzers" Version="5.9.0" />
<PackageVersion Include="Microsoft.CodeAnalysis.CSharp" Version="5.9.0" />
<PackageVersion Include="Parlot" Version="2.2.1" />
<PackageVersion Include="System.Linq.AsyncEnumerable" Version="10.0.14" />
<PackageVersion Include="Microsoft.Bcl.AsyncInterfaces" Version="10.0.15" />
<PackageVersion Include="Microsoft.Bcl.Memory" Version="10.0.15" />
<PackageVersion Include="Microsoft.Extensions.FileProviders.Abstractions" Version="10.0.15" />
<PackageVersion Include="Microsoft.CodeAnalysis.Analyzers" Version="5.12.0" />
<PackageVersion Include="Microsoft.CodeAnalysis.CSharp" Version="5.12.0" />
<PackageVersion Include="Parlot" Version="2.3.1" />
<PackageVersion Include="System.Linq.AsyncEnumerable" Version="10.0.15" />
</ItemGroup>
</Project>

34
WorkloadRollback.json

@ -1,18 +1,20 @@
{
"microsoft.net.sdk.android": "36.1.118/10.0.100",
"microsoft.net.sdk.ios": "26.5.10315/10.0.100",
"microsoft.net.sdk.maccatalyst": "26.5.10315/10.0.100",
"microsoft.net.sdk.macos": "26.5.10315/10.0.100",
"microsoft.net.sdk.maui": "10.0.110/10.0.100",
"microsoft.net.sdk.tvos": "26.5.10315/10.0.100",
"microsoft.net.workload.mono.toolchain.current": "10.0.114/10.0.100",
"microsoft.net.workload.emscripten.current": "10.0.114/10.0.100",
"microsoft.net.workload.emscripten.net6": "10.0.114/10.0.100",
"microsoft.net.workload.emscripten.net7": "10.0.114/10.0.100",
"microsoft.net.workload.emscripten.net8": "10.0.114/10.0.100",
"microsoft.net.workload.emscripten.net9": "10.0.114/10.0.100",
"microsoft.net.workload.mono.toolchain.net6": "10.0.114/10.0.100",
"microsoft.net.workload.mono.toolchain.net7": "10.0.114/10.0.100",
"microsoft.net.workload.mono.toolchain.net8": "10.0.114/10.0.100",
"microsoft.net.workload.mono.toolchain.net9": "10.0.114/10.0.100"
"microsoft.net.sdk.android": "37.2.29/11.0.100",
"microsoft.net.sdk.ios": "27.2.11039/11.0.100",
"microsoft.net.sdk.maccatalyst": "27.2.11039/11.0.100",
"microsoft.net.sdk.macos": "27.2.11039/11.0.100",
"microsoft.net.sdk.maui": "10.0.0/10.0.100",
"microsoft.net.sdk.tvos": "27.2.11039/11.0.100",
"microsoft.net.workload.mono.toolchain.current": "11.0.100/11.0.100",
"microsoft.net.workload.emscripten.current": "11.0.100/11.0.100",
"microsoft.net.workload.emscripten.net6": "11.0.100/11.0.100",
"microsoft.net.workload.emscripten.net7": "11.0.100/11.0.100",
"microsoft.net.workload.emscripten.net8": "11.0.100/11.0.100",
"microsoft.net.workload.emscripten.net9": "11.0.100/11.0.100",
"microsoft.net.workload.emscripten.net10": "11.0.100/11.0.100",
"microsoft.net.workload.mono.toolchain.net6": "11.0.100/11.0.100",
"microsoft.net.workload.mono.toolchain.net7": "11.0.100/11.0.100",
"microsoft.net.workload.mono.toolchain.net8": "11.0.100/11.0.100",
"microsoft.net.workload.mono.toolchain.net9": "11.0.100/11.0.100",
"microsoft.net.workload.mono.toolchain.net10": "11.0.100/11.0.100"
}

2
eng/Tools.props

@ -2,7 +2,7 @@
<ItemGroup>
<PackageReference Include="MSBuild.Microsoft.VisualStudio.Web.targets" Version="14.0.0.3" IsImplicitlyDefined="true" />
<PackageReference Include="NuGet.Build.Tasks.Pack" Version="5.10.0" IsImplicitlyDefined="true" />
<PackageReference Include="NuGet.Build.Tasks.Pack" Version="6.14.0" IsImplicitlyDefined="true" />
</ItemGroup>
<!--

4
eng/Version.Details.xml

@ -5,12 +5,12 @@
</ProductDependencies>
<ToolsetDependencies>
<Dependency Name="Microsoft.DotNet.Arcade.Sdk" Version="10.0.0-beta.25562.108">
<Dependency Name="Microsoft.DotNet.Arcade.Sdk" Version="11.0.0-beta.26514.117">
<Uri>https://github.com/dotnet/dotnet</Uri>
<Sha>f4701e048e6a684237a4b52b745e21b1d857278d</Sha>
</Dependency>
<Dependency Name="Microsoft.DotNet.Helix.Sdk" Version="10.0.0-beta.25562.108">
<Dependency Name="Microsoft.DotNet.Helix.Sdk" Version="11.0.0-beta.26514.117">
<Uri>https://github.com/dotnet/dotnet</Uri>
<Sha>f4701e048e6a684237a4b52b745e21b1d857278d</Sha>
</Dependency>

17
eng/common/SetupNugetSources.ps1

@ -1,7 +1,6 @@
# This script adds internal feeds required to build commits that depend on internal package sources. For instance,
# dotnet6-internal would be added automatically if dotnet6 was found in the nuget.config file. Similarly,
# dotnet-eng-internal and dotnet-tools-internal are added if dotnet-eng and dotnet-tools are present.
# In addition, this script also enables disabled internal Maestro (darc-int*) feeds.
# dotnet6-internal would be added automatically if dotnet6 was found in the nuget.config file. In addition also enables
# disabled internal Maestro (darc-int*) feeds.
#
# Optionally, this script also adds a credential entry for each of the internal feeds if supplied.
#
@ -174,16 +173,4 @@ foreach ($dotnetVersion in $dotnetVersions) {
}
}
# Check for dotnet-eng and add dotnet-eng-internal if present
$dotnetEngSource = $sources.SelectSingleNode("add[@key='dotnet-eng']")
if ($dotnetEngSource -ne $null) {
AddOrEnablePackageSource -Sources $sources -DisabledPackageSources $disabledSources -SourceName "dotnet-eng-internal" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/internal/_packaging/dotnet-eng-internal/nuget/$feedSuffix" -Creds $creds -Username $userName -pwd $Password
}
# Check for dotnet-tools and add dotnet-tools-internal if present
$dotnetToolsSource = $sources.SelectSingleNode("add[@key='dotnet-tools']")
if ($dotnetToolsSource -ne $null) {
AddOrEnablePackageSource -Sources $sources -DisabledPackageSources $disabledSources -SourceName "dotnet-tools-internal" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/internal/_packaging/dotnet-tools-internal/nuget/$feedSuffix" -Creds $creds -Username $userName -pwd $Password
}
$doc.Save($filename)

17
eng/common/SetupNugetSources.sh

@ -1,9 +1,8 @@
#!/usr/bin/env bash
# This script adds internal feeds required to build commits that depend on internal package sources. For instance,
# dotnet6-internal would be added automatically if dotnet6 was found in the nuget.config file. Similarly,
# dotnet-eng-internal and dotnet-tools-internal are added if dotnet-eng and dotnet-tools are present.
# In addition, this script also enables disabled internal Maestro (darc-int*) feeds.
# dotnet6-internal would be added automatically if dotnet6 was found in the nuget.config file. In addition also enables
# disabled internal Maestro (darc-int*) feeds.
#
# Optionally, this script also adds a credential entry for each of the internal feeds if supplied.
#
@ -174,18 +173,6 @@ for DotNetVersion in ${DotNetVersions[@]} ; do
fi
done
# Check for dotnet-eng and add dotnet-eng-internal if present
grep -i "<add key=\"dotnet-eng\"" $ConfigFile > /dev/null
if [ "$?" == "0" ]; then
AddOrEnablePackageSource "dotnet-eng-internal" "https://pkgs.dev.azure.com/dnceng/internal/_packaging/dotnet-eng-internal/nuget/$FeedSuffix"
fi
# Check for dotnet-tools and add dotnet-tools-internal if present
grep -i "<add key=\"dotnet-tools\"" $ConfigFile > /dev/null
if [ "$?" == "0" ]; then
AddOrEnablePackageSource "dotnet-tools-internal" "https://pkgs.dev.azure.com/dnceng/internal/_packaging/dotnet-tools-internal/nuget/$FeedSuffix"
fi
# I want things split line by line
PrevIFS=$IFS
IFS=$'\n'

2
eng/common/build.sh

@ -92,7 +92,7 @@ runtime_source_feed=''
runtime_source_feed_key=''
properties=()
while [[ $# > 0 ]]; do
while [[ $# -gt 0 ]]; do
opt="$(echo "${1/#--/-}" | tr "[:upper:]" "[:lower:]")"
case "$opt" in
-help|-h)

8
eng/common/core-templates/job/job.yml

@ -19,6 +19,8 @@ parameters:
# publishing defaults
artifacts: ''
enableMicrobuild: false
enablePreviewMicrobuild: false
microbuildPluginVersion: 'latest'
enableMicrobuildForMacAndLinux: false
microbuildUseESRP: true
enablePublishBuildArtifacts: false
@ -71,6 +73,8 @@ jobs:
templateContext: ${{ parameters.templateContext }}
variables:
- name: AllowPtrToDetectTestRunRetryFiles
value: true
- ${{ if ne(parameters.enableTelemetry, 'false') }}:
- name: DOTNET_CLI_TELEMETRY_PROFILE
value: '$(Build.Repository.Uri)'
@ -128,6 +132,8 @@ jobs:
- template: /eng/common/core-templates/steps/install-microbuild.yml
parameters:
enableMicrobuild: ${{ parameters.enableMicrobuild }}
enablePreviewMicrobuild: ${{ parameters.enablePreviewMicrobuild }}
microbuildPluginVersion: ${{ parameters.microbuildPluginVersion }}
enableMicrobuildForMacAndLinux: ${{ parameters.enableMicrobuildForMacAndLinux }}
microbuildUseESRP: ${{ parameters.microbuildUseESRP }}
continueOnError: ${{ parameters.continueOnError }}
@ -153,6 +159,8 @@ jobs:
- template: /eng/common/core-templates/steps/cleanup-microbuild.yml
parameters:
enableMicrobuild: ${{ parameters.enableMicrobuild }}
enablePreviewMicrobuild: ${{ parameters.enablePreviewMicrobuild }}
microbuildPluginVersion: ${{ parameters.microbuildPluginVersion }}
enableMicrobuildForMacAndLinux: ${{ parameters.enableMicrobuildForMacAndLinux }}
continueOnError: ${{ parameters.continueOnError }}

29
eng/common/core-templates/job/publish-build-assets.yml

@ -80,7 +80,7 @@ jobs:
# If it's not devdiv, it's dnceng
${{ if ne(variables['System.TeamProject'], 'DevDiv') }}:
name: NetCore1ESPool-Publishing-Internal
image: windows.vs2019.amd64
image: windows.vs2022.amd64
os: windows
steps:
- ${{ if eq(parameters.is1ESPipeline, '') }}:
@ -91,8 +91,8 @@ jobs:
fetchDepth: 3
clean: true
- ${{ if eq(parameters.isAssetlessBuild, 'false') }}:
- ${{ if eq(parameters.publishingVersion, 3) }}:
- ${{ if eq(parameters.isAssetlessBuild, 'false') }}:
- ${{ if eq(parameters.publishingVersion, 3) }}:
- task: DownloadPipelineArtifact@2
displayName: Download Asset Manifests
inputs:
@ -117,9 +117,17 @@ jobs:
flattenFolders: true
condition: ${{ parameters.condition }}
continueOnError: ${{ parameters.continueOnError }}
- task: NuGetAuthenticate@1
# Populate internal runtime variables.
- template: /eng/common/templates/steps/enable-internal-sources.yml
${{ if eq(variables['System.TeamProject'], 'DevDiv') }}:
parameters:
legacyCredential: $(dn-bot-dnceng-artifact-feeds-rw)
- template: /eng/common/templates/steps/enable-internal-runtimes.yml
- task: AzureCLI@2
displayName: Publish Build Assets
inputs:
@ -132,9 +140,12 @@ jobs:
/p:IsAssetlessBuild=${{ parameters.isAssetlessBuild }}
/p:MaestroApiEndpoint=https://maestro.dot.net
/p:OfficialBuildId=$(OfficialBuildId)
-runtimeSourceFeed https://ci.dot.net/internal
-runtimeSourceFeedKey '$(dotnetbuilds-internal-container-read-token-base64)'
condition: ${{ parameters.condition }}
continueOnError: ${{ parameters.continueOnError }}
- task: powershell@2
displayName: Create ReleaseConfigs Artifact
inputs:
@ -162,7 +173,7 @@ jobs:
artifactName: AssetManifests
displayName: 'Publish Merged Manifest'
retryCountOnTaskFailure: 10 # for any logs being locked
sbomEnabled: false # we don't need SBOM for logs
sbomEnabled: false # we don't need SBOM for logs
- template: /eng/common/core-templates/steps/publish-build-artifacts.yml
parameters:
@ -179,7 +190,7 @@ jobs:
BARBuildId: ${{ parameters.BARBuildId }}
PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }}
is1ESPipeline: ${{ parameters.is1ESPipeline }}
# Darc is targeting 8.0, so make sure it's installed
- task: UseDotNet@2
inputs:
@ -200,9 +211,11 @@ jobs:
-ArtifactsPublishingAdditionalParameters '${{ parameters.artifactsPublishingAdditionalParameters }}'
-SymbolPublishingAdditionalParameters '${{ parameters.symbolPublishingAdditionalParameters }}'
-SkipAssetsPublishing '${{ parameters.isAssetlessBuild }}'
-runtimeSourceFeed https://ci.dot.net/internal
-runtimeSourceFeedKey '$(dotnetbuilds-internal-container-read-token-base64)'
- ${{ if eq(parameters.enablePublishBuildArtifacts, 'true') }}:
- template: /eng/common/core-templates/steps/publish-logs.yml
parameters:
is1ESPipeline: ${{ parameters.is1ESPipeline }}
JobLabel: 'Publish_Artifacts_Logs'
JobLabel: 'Publish_Artifacts_Logs'

8
eng/common/core-templates/job/source-build.yml

@ -60,19 +60,19 @@ jobs:
pool:
${{ if eq(variables['System.TeamProject'], 'public') }}:
name: $[replace(replace(eq(contains(coalesce(variables['System.PullRequest.TargetBranch'], variables['Build.SourceBranch'], 'refs/heads/main'), 'release'), 'true'), True, 'NetCore-Svc-Public' ), False, 'NetCore-Public')]
demands: ImageOverride -equals build.ubuntu.2004.amd64
demands: ImageOverride -equals build.azurelinux.3.amd64.open
${{ if eq(variables['System.TeamProject'], 'internal') }}:
name: $[replace(replace(eq(contains(coalesce(variables['System.PullRequest.TargetBranch'], variables['Build.SourceBranch'], 'refs/heads/main'), 'release'), 'true'), True, 'NetCore1ESPool-Svc-Internal'), False, 'NetCore1ESPool-Internal')]
image: 1es-mariner-2
image: build.azurelinux.3.amd64
os: linux
${{ else }}:
pool:
${{ if eq(variables['System.TeamProject'], 'public') }}:
name: $[replace(replace(eq(contains(coalesce(variables['System.PullRequest.TargetBranch'], variables['Build.SourceBranch'], 'refs/heads/main'), 'release'), 'true'), True, 'NetCore-Svc-Public' ), False, 'NetCore-Public')]
demands: ImageOverride -equals Build.Ubuntu.2204.Amd64.Open
demands: ImageOverride -equals build.azurelinux.3.amd64.open
${{ if eq(variables['System.TeamProject'], 'internal') }}:
name: $[replace(replace(eq(contains(coalesce(variables['System.PullRequest.TargetBranch'], variables['Build.SourceBranch'], 'refs/heads/main'), 'release'), 'true'), True, 'NetCore1ESPool-Svc-Internal'), False, 'NetCore1ESPool-Internal')]
demands: ImageOverride -equals Build.Ubuntu.2204.Amd64
demands: ImageOverride -equals build.azurelinux.3.amd64
${{ if ne(parameters.platform.pool, '') }}:
pool: ${{ parameters.platform.pool }}

8
eng/common/core-templates/job/source-index-stage1.yml

@ -3,7 +3,7 @@ parameters:
sourceIndexBuildCommand: powershell -NoLogo -NoProfile -ExecutionPolicy Bypass -Command "eng/common/build.ps1 -restore -build -binarylog -ci"
preSteps: []
binlogPath: artifacts/log/Debug/Build.binlog
condition: ''
condition: eq(variables['Build.SourceBranch'], 'refs/heads/main')
dependsOn: ''
pool: ''
is1ESPipeline: ''
@ -25,10 +25,10 @@ jobs:
pool:
${{ if eq(variables['System.TeamProject'], 'public') }}:
name: $(DncEngPublicBuildPool)
image: windows.vs2022.amd64.open
image: windows.vs2026preview.scout.amd64.open
${{ if eq(variables['System.TeamProject'], 'internal') }}:
name: $(DncEngInternalBuildPool)
image: windows.vs2022.amd64
image: windows.vs2026preview.scout.amd64
steps:
- ${{ if eq(parameters.is1ESPipeline, '') }}:
@ -41,4 +41,4 @@ jobs:
- template: /eng/common/core-templates/steps/source-index-stage1-publish.yml
parameters:
binLogPath: ${{ parameters.binLogPath }}
binLogPath: ${{ parameters.binLogPath }}

466
eng/common/core-templates/post-build/post-build.yml

@ -1,106 +1,106 @@
parameters:
# Which publishing infra should be used. THIS SHOULD MATCH THE VERSION ON THE BUILD MANIFEST.
# Publishing V1 is no longer supported
# Publishing V2 is no longer supported
# Publishing V3 is the default
- name: publishingInfraVersion
displayName: Which version of publishing should be used to promote the build definition?
type: number
default: 3
values:
- 3
- name: BARBuildId
displayName: BAR Build Id
type: number
default: 0
- name: PromoteToChannelIds
displayName: Channel to promote BARBuildId to
type: string
default: ''
- name: enableSourceLinkValidation
displayName: Enable SourceLink validation
type: boolean
default: false
- name: enableSigningValidation
displayName: Enable signing validation
type: boolean
default: true
- name: enableSymbolValidation
displayName: Enable symbol validation
type: boolean
default: false
- name: enableNugetValidation
displayName: Enable NuGet validation
type: boolean
default: true
- name: publishInstallersAndChecksums
displayName: Publish installers and checksums
type: boolean
default: true
- name: requireDefaultChannels
displayName: Fail the build if there are no default channel(s) registrations for the current build
type: boolean
default: false
- name: SDLValidationParameters
type: object
default:
enable: false
publishGdn: false
continueOnError: false
params: ''
artifactNames: ''
downloadArtifacts: true
- name: isAssetlessBuild
type: boolean
displayName: Is Assetless Build
default: false
# These parameters let the user customize the call to sdk-task.ps1 for publishing
# symbols & general artifacts as well as for signing validation
- name: symbolPublishingAdditionalParameters
displayName: Symbol publishing additional parameters
type: string
default: ''
- name: artifactsPublishingAdditionalParameters
displayName: Artifact publishing additional parameters
type: string
default: ''
- name: signingValidationAdditionalParameters
displayName: Signing validation additional parameters
type: string
default: ''
# Which stages should finish execution before post-build stages start
- name: validateDependsOn
type: object
default:
- build
- name: publishDependsOn
type: object
default:
- Validate
# Optional: Call asset publishing rather than running in a separate stage
- name: publishAssetsImmediately
type: boolean
default: false
- name: is1ESPipeline
type: boolean
default: false
# Which publishing infra should be used. THIS SHOULD MATCH THE VERSION ON THE BUILD MANIFEST.
# Publishing V1 is no longer supported
# Publishing V2 is no longer supported
# Publishing V3 is the default
- name: publishingInfraVersion
displayName: Which version of publishing should be used to promote the build definition?
type: number
default: 3
values:
- 3
- name: BARBuildId
displayName: BAR Build Id
type: number
default: 0
- name: PromoteToChannelIds
displayName: Channel to promote BARBuildId to
type: string
default: ''
- name: enableSourceLinkValidation
displayName: Enable SourceLink validation
type: boolean
default: false
- name: enableSigningValidation
displayName: Enable signing validation
type: boolean
default: true
- name: enableSymbolValidation
displayName: Enable symbol validation
type: boolean
default: false
- name: enableNugetValidation
displayName: Enable NuGet validation
type: boolean
default: true
- name: publishInstallersAndChecksums
displayName: Publish installers and checksums
type: boolean
default: true
- name: requireDefaultChannels
displayName: Fail the build if there are no default channel(s) registrations for the current build
type: boolean
default: false
- name: SDLValidationParameters
type: object
default:
enable: false
publishGdn: false
continueOnError: false
params: ''
artifactNames: ''
downloadArtifacts: true
- name: isAssetlessBuild
type: boolean
displayName: Is Assetless Build
default: false
# These parameters let the user customize the call to sdk-task.ps1 for publishing
# symbols & general artifacts as well as for signing validation
- name: symbolPublishingAdditionalParameters
displayName: Symbol publishing additional parameters
type: string
default: ''
- name: artifactsPublishingAdditionalParameters
displayName: Artifact publishing additional parameters
type: string
default: ''
- name: signingValidationAdditionalParameters
displayName: Signing validation additional parameters
type: string
default: ''
# Which stages should finish execution before post-build stages start
- name: validateDependsOn
type: object
default:
- build
- name: publishDependsOn
type: object
default:
- Validate
# Optional: Call asset publishing rather than running in a separate stage
- name: publishAssetsImmediately
type: boolean
default: false
- name: is1ESPipeline
type: boolean
default: false
stages:
- ${{ if or(eq( parameters.enableNugetValidation, 'true'), eq(parameters.enableSigningValidation, 'true'), eq(parameters.enableSourceLinkValidation, 'true'), eq(parameters.SDLValidationParameters.enable, 'true')) }}:
@ -108,10 +108,10 @@ stages:
dependsOn: ${{ parameters.validateDependsOn }}
displayName: Validate Build Assets
variables:
- template: /eng/common/core-templates/post-build/common-variables.yml
- template: /eng/common/core-templates/variables/pool-providers.yml
parameters:
is1ESPipeline: ${{ parameters.is1ESPipeline }}
- template: /eng/common/core-templates/post-build/common-variables.yml
- template: /eng/common/core-templates/variables/pool-providers.yml
parameters:
is1ESPipeline: ${{ parameters.is1ESPipeline }}
jobs:
- job:
displayName: NuGet Validation
@ -127,35 +127,35 @@ stages:
${{ else }}:
${{ if eq(parameters.is1ESPipeline, true) }}:
name: $(DncEngInternalBuildPool)
image: windows.vs2022.amd64
image: windows.vs2026preview.scout.amd64
os: windows
${{ else }}:
name: $(DncEngInternalBuildPool)
demands: ImageOverride -equals windows.vs2022.amd64
demands: ImageOverride -equals windows.vs2026preview.scout.amd64
steps:
- template: /eng/common/core-templates/post-build/setup-maestro-vars.yml
parameters:
BARBuildId: ${{ parameters.BARBuildId }}
PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }}
is1ESPipeline: ${{ parameters.is1ESPipeline }}
- task: DownloadBuildArtifacts@0
displayName: Download Package Artifacts
inputs:
buildType: specific
buildVersionToDownload: specific
project: $(AzDOProjectName)
pipeline: $(AzDOPipelineId)
buildId: $(AzDOBuildId)
artifactName: PackageArtifacts
checkDownloadedFiles: true
- task: PowerShell@2
displayName: Validate
inputs:
filePath: $(System.DefaultWorkingDirectory)/eng/common/post-build/nuget-validation.ps1
arguments: -PackagesPath $(Build.ArtifactStagingDirectory)/PackageArtifacts/
- template: /eng/common/core-templates/post-build/setup-maestro-vars.yml
parameters:
BARBuildId: ${{ parameters.BARBuildId }}
PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }}
is1ESPipeline: ${{ parameters.is1ESPipeline }}
- task: DownloadBuildArtifacts@0
displayName: Download Package Artifacts
inputs:
buildType: specific
buildVersionToDownload: specific
project: $(AzDOProjectName)
pipeline: $(AzDOPipelineId)
buildId: $(AzDOBuildId)
artifactName: PackageArtifacts
checkDownloadedFiles: true
- task: PowerShell@2
displayName: Validate
inputs:
filePath: $(System.DefaultWorkingDirectory)/eng/common/post-build/nuget-validation.ps1
arguments: -PackagesPath $(Build.ArtifactStagingDirectory)/PackageArtifacts/
- job:
displayName: Signing Validation
@ -169,54 +169,54 @@ stages:
os: windows
# If it's not devdiv, it's dnceng
${{ else }}:
${{ if eq(parameters.is1ESPipeline, true) }}:
${{ if eq(parameters.is1ESPipeline, true) }}:
name: $(DncEngInternalBuildPool)
image: 1es-windows-2022
os: windows
${{ else }}:
name: $(DncEngInternalBuildPool)
demands: ImageOverride -equals windows.vs2022.amd64
demands: ImageOverride -equals windows.vs2026preview.scout.amd64
steps:
- template: /eng/common/core-templates/post-build/setup-maestro-vars.yml
parameters:
BARBuildId: ${{ parameters.BARBuildId }}
PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }}
is1ESPipeline: ${{ parameters.is1ESPipeline }}
- task: DownloadBuildArtifacts@0
displayName: Download Package Artifacts
inputs:
buildType: specific
buildVersionToDownload: specific
project: $(AzDOProjectName)
pipeline: $(AzDOPipelineId)
buildId: $(AzDOBuildId)
artifactName: PackageArtifacts
checkDownloadedFiles: true
# This is necessary whenever we want to publish/restore to an AzDO private feed
# Since sdk-task.ps1 tries to restore packages we need to do this authentication here
# otherwise it'll complain about accessing a private feed.
- task: NuGetAuthenticate@1
displayName: 'Authenticate to AzDO Feeds'
# Signing validation will optionally work with the buildmanifest file which is downloaded from
# Azure DevOps above.
- task: PowerShell@2
displayName: Validate
inputs:
filePath: eng\common\sdk-task.ps1
arguments: -task SigningValidation -restore -msbuildEngine vs
/p:PackageBasePath='$(Build.ArtifactStagingDirectory)/PackageArtifacts'
/p:SignCheckExclusionsFile='$(System.DefaultWorkingDirectory)/eng/SignCheckExclusionsFile.txt'
${{ parameters.signingValidationAdditionalParameters }}
- template: /eng/common/core-templates/steps/publish-logs.yml
parameters:
is1ESPipeline: ${{ parameters.is1ESPipeline }}
StageLabel: 'Validation'
JobLabel: 'Signing'
BinlogToolVersion: $(BinlogToolVersion)
- template: /eng/common/core-templates/post-build/setup-maestro-vars.yml
parameters:
BARBuildId: ${{ parameters.BARBuildId }}
PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }}
is1ESPipeline: ${{ parameters.is1ESPipeline }}
- task: DownloadBuildArtifacts@0
displayName: Download Package Artifacts
inputs:
buildType: specific
buildVersionToDownload: specific
project: $(AzDOProjectName)
pipeline: $(AzDOPipelineId)
buildId: $(AzDOBuildId)
artifactName: PackageArtifacts
checkDownloadedFiles: true
# This is necessary whenever we want to publish/restore to an AzDO private feed
# Since sdk-task.ps1 tries to restore packages we need to do this authentication here
# otherwise it'll complain about accessing a private feed.
- task: NuGetAuthenticate@1
displayName: 'Authenticate to AzDO Feeds'
# Signing validation will optionally work with the buildmanifest file which is downloaded from
# Azure DevOps above.
- task: PowerShell@2
displayName: Validate
inputs:
filePath: eng\common\sdk-task.ps1
arguments: -task SigningValidation -restore -msbuildEngine vs
/p:PackageBasePath='$(Build.ArtifactStagingDirectory)/PackageArtifacts'
/p:SignCheckExclusionsFile='$(System.DefaultWorkingDirectory)/eng/SignCheckExclusionsFile.txt'
${{ parameters.signingValidationAdditionalParameters }}
- template: /eng/common/core-templates/steps/publish-logs.yml
parameters:
is1ESPipeline: ${{ parameters.is1ESPipeline }}
StageLabel: 'Validation'
JobLabel: 'Signing'
BinlogToolVersion: $(BinlogToolVersion)
- job:
displayName: SourceLink Validation
@ -230,41 +230,41 @@ stages:
os: windows
# If it's not devdiv, it's dnceng
${{ else }}:
${{ if eq(parameters.is1ESPipeline, true) }}:
${{ if eq(parameters.is1ESPipeline, true) }}:
name: $(DncEngInternalBuildPool)
image: 1es-windows-2022
os: windows
${{ else }}:
name: $(DncEngInternalBuildPool)
demands: ImageOverride -equals windows.vs2022.amd64
demands: ImageOverride -equals windows.vs2026preview.scout.amd64
steps:
- template: /eng/common/core-templates/post-build/setup-maestro-vars.yml
parameters:
BARBuildId: ${{ parameters.BARBuildId }}
PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }}
is1ESPipeline: ${{ parameters.is1ESPipeline }}
- task: DownloadBuildArtifacts@0
displayName: Download Blob Artifacts
inputs:
buildType: specific
buildVersionToDownload: specific
project: $(AzDOProjectName)
pipeline: $(AzDOPipelineId)
buildId: $(AzDOBuildId)
artifactName: BlobArtifacts
checkDownloadedFiles: true
- task: PowerShell@2
displayName: Validate
inputs:
filePath: $(System.DefaultWorkingDirectory)/eng/common/post-build/sourcelink-validation.ps1
arguments: -InputPath $(Build.ArtifactStagingDirectory)/BlobArtifacts/
-ExtractPath $(Agent.BuildDirectory)/Extract/
-GHRepoName $(Build.Repository.Name)
-GHCommit $(Build.SourceVersion)
-SourcelinkCliVersion $(SourceLinkCLIVersion)
continueOnError: true
- template: /eng/common/core-templates/post-build/setup-maestro-vars.yml
parameters:
BARBuildId: ${{ parameters.BARBuildId }}
PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }}
is1ESPipeline: ${{ parameters.is1ESPipeline }}
- task: DownloadBuildArtifacts@0
displayName: Download Blob Artifacts
inputs:
buildType: specific
buildVersionToDownload: specific
project: $(AzDOProjectName)
pipeline: $(AzDOPipelineId)
buildId: $(AzDOBuildId)
artifactName: BlobArtifacts
checkDownloadedFiles: true
- task: PowerShell@2
displayName: Validate
inputs:
filePath: $(System.DefaultWorkingDirectory)/eng/common/post-build/sourcelink-validation.ps1
arguments: -InputPath $(Build.ArtifactStagingDirectory)/BlobArtifacts/
-ExtractPath $(Agent.BuildDirectory)/Extract/
-GHRepoName $(Build.Repository.Name)
-GHCommit $(Build.SourceVersion)
-SourcelinkCliVersion $(SourceLinkCLIVersion)
continueOnError: true
- ${{ if ne(parameters.publishAssetsImmediately, 'true') }}:
- stage: publish_using_darc
@ -274,10 +274,10 @@ stages:
dependsOn: ${{ parameters.validateDependsOn }}
displayName: Publish using Darc
variables:
- template: /eng/common/core-templates/post-build/common-variables.yml
- template: /eng/common/core-templates/variables/pool-providers.yml
parameters:
is1ESPipeline: ${{ parameters.is1ESPipeline }}
- template: /eng/common/core-templates/post-build/common-variables.yml
- template: /eng/common/core-templates/variables/pool-providers.yml
parameters:
is1ESPipeline: ${{ parameters.is1ESPipeline }}
jobs:
- job:
displayName: Publish Using Darc
@ -291,35 +291,41 @@ stages:
os: windows
# If it's not devdiv, it's dnceng
${{ else }}:
${{ if eq(parameters.is1ESPipeline, true) }}:
${{ if eq(parameters.is1ESPipeline, true) }}:
name: NetCore1ESPool-Publishing-Internal
image: windows.vs2019.amd64
image: windows.vs2022.amd64
os: windows
${{ else }}:
name: NetCore1ESPool-Publishing-Internal
demands: ImageOverride -equals windows.vs2019.amd64
demands: ImageOverride -equals windows.vs2022.amd64
steps:
- template: /eng/common/core-templates/post-build/setup-maestro-vars.yml
parameters:
BARBuildId: ${{ parameters.BARBuildId }}
PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }}
is1ESPipeline: ${{ parameters.is1ESPipeline }}
- task: NuGetAuthenticate@1
# Darc is targeting 8.0, so make sure it's installed
- task: UseDotNet@2
inputs:
version: 8.0.x
- task: AzureCLI@2
displayName: Publish Using Darc
inputs:
azureSubscription: "Darc: Maestro Production"
scriptType: ps
scriptLocation: scriptPath
scriptPath: $(System.DefaultWorkingDirectory)/eng/common/post-build/publish-using-darc.ps1
arguments: >
- template: /eng/common/core-templates/post-build/setup-maestro-vars.yml
parameters:
BARBuildId: ${{ parameters.BARBuildId }}
PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }}
is1ESPipeline: ${{ parameters.is1ESPipeline }}
- task: NuGetAuthenticate@1
# Populate internal runtime variables.
- template: /eng/common/templates/steps/enable-internal-sources.yml
parameters:
legacyCredential: $(dn-bot-dnceng-artifact-feeds-rw)
- template: /eng/common/templates/steps/enable-internal-runtimes.yml
- task: UseDotNet@2
inputs:
version: 8.0.x
- task: AzureCLI@2
displayName: Publish Using Darc
inputs:
azureSubscription: "Darc: Maestro Production"
scriptType: ps
scriptLocation: scriptPath
scriptPath: $(System.DefaultWorkingDirectory)/eng/common/post-build/publish-using-darc.ps1
arguments: >
-BuildId $(BARBuildId)
-PublishingInfraVersion ${{ parameters.publishingInfraVersion }}
-AzdoToken '$(System.AccessToken)'
@ -328,3 +334,5 @@ stages:
-ArtifactsPublishingAdditionalParameters '${{ parameters.artifactsPublishingAdditionalParameters }}'
-SymbolPublishingAdditionalParameters '${{ parameters.symbolPublishingAdditionalParameters }}'
-SkipAssetsPublishing '${{ parameters.isAssetlessBuild }}'
-runtimeSourceFeed https://ci.dot.net/internal
-runtimeSourceFeedKey '$(dotnetbuilds-internal-container-read-token-base64)'

2
eng/common/core-templates/steps/generate-sbom.yml

@ -5,7 +5,7 @@
# IgnoreDirectories - Directories to ignore for SBOM generation. This will be passed through to the CG component detector.
parameters:
PackageVersion: 10.0.0
PackageVersion: 11.0.0
BuildDropPath: '$(System.DefaultWorkingDirectory)/artifacts'
PackageName: '.NET'
ManifestDirPath: $(Build.ArtifactStagingDirectory)/sbom

34
eng/common/core-templates/steps/install-microbuild-impl.yml

@ -0,0 +1,34 @@
parameters:
- name: microbuildTaskInputs
type: object
default: {}
- name: microbuildEnv
type: object
default: {}
- name: enablePreviewMicrobuild
type: boolean
default: false
- name: condition
type: string
- name: continueOnError
type: boolean
steps:
- ${{ if eq(parameters.enablePreviewMicrobuild, true) }}:
- task: MicroBuildSigningPluginPreview@4
displayName: Install Preview MicroBuild plugin
inputs: ${{ parameters.microbuildTaskInputs }}
env: ${{ parameters.microbuildEnv }}
continueOnError: ${{ parameters.continueOnError }}
condition: ${{ parameters.condition }}
- ${{ else }}:
- task: MicroBuildSigningPlugin@4
displayName: Install MicroBuild plugin
inputs: ${{ parameters.microbuildTaskInputs }}
env: ${{ parameters.microbuildEnv }}
continueOnError: ${{ parameters.continueOnError }}
condition: ${{ parameters.condition }}

94
eng/common/core-templates/steps/install-microbuild.yml

@ -4,6 +4,8 @@ parameters:
# Enable install tasks for MicroBuild on Mac and Linux
# Will be ignored if 'enableMicrobuild' is false or 'Agent.Os' is 'Windows_NT'
enableMicrobuildForMacAndLinux: false
# Enable preview version of MB signing plugin
enablePreviewMicrobuild: false
# Determines whether the ESRP service connection information should be passed to the signing plugin.
# This overlaps with _SignType to some degree. We only need the service connection for real signing.
# It's important that the service connection not be passed to the MicroBuildSigningPlugin task in this place.
@ -11,22 +13,43 @@ parameters:
# Unfortunately, _SignType can't be used to exclude the use of the service connection in non-real sign scenarios. The
# variable is not available in template expression. _SignType has a very large proliferation across .NET, so replacing it is tough.
microbuildUseESRP: true
# Microbuild installation directory
microBuildOutputFolder: $(Agent.TempDirectory)/MicroBuild
# Microbuild version
microbuildPluginVersion: 'latest'
continueOnError: false
steps:
- ${{ if eq(parameters.enableMicrobuild, 'true') }}:
- ${{ if eq(parameters.enableMicrobuildForMacAndLinux, 'true') }}:
# Installing .NET 8 is required to use the MicroBuild signing plugin on non-Windows platforms
# Needed to download the MicroBuild plugin nupkgs on Mac and Linux when nuget.exe is unavailable
- task: UseDotNet@2
displayName: Install .NET 8.0 SDK for MicroBuild Plugin
inputs:
packageType: sdk
version: 8.0.x
# Installing the SDK in a '.dotnet-microbuild' directory is required for signing.
# See target FindDotNetPathForMicroBuild in arcade/src/Microsoft.DotNet.Arcade.Sdk/tools/Sign.proj
# Do not remove '.dotnet-microbuild' from the path without changing the corresponding logic.
installationPath: $(Agent.TempDirectory)/.dotnet-microbuild
installationPath: ${{ parameters.microBuildOutputFolder }}/.dotnet-microbuild
condition: and(succeeded(), ne(variables['Agent.Os'], 'Windows_NT'))
- script: |
set -euo pipefail
# UseDotNet@2 prepends the dotnet executable path to the PATH variable, so we can call dotnet directly
version=$(dotnet --version)
cat << 'EOF' > ${{ parameters.microBuildOutputFolder }}/global.json
{
"sdk": {
"version": "$version",
"paths": [
"${{ parameters.microBuildOutputFolder }}/.dotnet-microbuild"
],
"errorMessage": "The .NET SDK version $version is required to install the MicroBuild signing plugin."
}
}
EOF
displayName: 'Add global.json to MicroBuild Installation path'
workingDirectory: ${{ parameters.microBuildOutputFolder }}
condition: and(succeeded(), ne(variables['Agent.Os'], 'Windows_NT'))
- script: |
@ -50,41 +73,46 @@ steps:
# YAML expansion, and Windows vs. Linux/Mac uses different service connections. However,
# we can avoid including the MB install step if not enabled at all. This avoids a bunch of
# extra pipeline authorizations, since most pipelines do not sign on non-Windows.
- task: MicroBuildSigningPlugin@4
displayName: Install MicroBuild plugin (Windows)
inputs:
signType: $(_SignType)
zipSources: false
feedSource: https://dnceng.pkgs.visualstudio.com/_packaging/MicroBuildToolset/nuget/v3/index.json
${{ if eq(parameters.microbuildUseESRP, true) }}:
ConnectedServiceName: 'MicroBuild Signing Task (DevDiv)'
${{ if eq(variables['System.TeamProject'], 'DevDiv') }}:
ConnectedPMEServiceName: 6cc74545-d7b9-4050-9dfa-ebefcc8961ea
${{ else }}:
ConnectedPMEServiceName: 248d384a-b39b-46e3-8ad5-c2c210d5e7ca
env:
TeamName: $(_TeamName)
MicroBuildOutputFolderOverride: $(Agent.TempDirectory)/MicroBuild
SYSTEM_ACCESSTOKEN: $(System.AccessToken)
continueOnError: ${{ parameters.continueOnError }}
condition: and(succeeded(), eq(variables['Agent.Os'], 'Windows_NT'), in(variables['_SignType'], 'real', 'test'))
- ${{ if eq(parameters.enableMicrobuildForMacAndLinux, true) }}:
- task: MicroBuildSigningPlugin@4
displayName: Install MicroBuild plugin (non-Windows)
inputs:
- template: /eng/common/core-templates/steps/install-microbuild-impl.yml@self
parameters:
enablePreviewMicrobuild: ${{ parameters.enablePreviewMicrobuild }}
microbuildTaskInputs:
signType: $(_SignType)
zipSources: false
feedSource: https://dnceng.pkgs.visualstudio.com/_packaging/MicroBuildToolset/nuget/v3/index.json
version: ${{ parameters.microbuildPluginVersion }}
${{ if eq(parameters.microbuildUseESRP, true) }}:
ConnectedServiceName: 'MicroBuild Signing Task (DevDiv)'
${{ if eq(variables['System.TeamProject'], 'DevDiv') }}:
ConnectedPMEServiceName: beb8cb23-b303-4c95-ab26-9e44bc958d39
ConnectedPMEServiceName: 6cc74545-d7b9-4050-9dfa-ebefcc8961ea
${{ else }}:
ConnectedPMEServiceName: c24de2a5-cc7a-493d-95e4-8e5ff5cad2bc
env:
ConnectedPMEServiceName: 248d384a-b39b-46e3-8ad5-c2c210d5e7ca
microbuildEnv:
TeamName: $(_TeamName)
MicroBuildOutputFolderOverride: $(Agent.TempDirectory)/MicroBuild
MicroBuildOutputFolderOverride: ${{ parameters.microBuildOutputFolder }}
SYSTEM_ACCESSTOKEN: $(System.AccessToken)
continueOnError: ${{ parameters.continueOnError }}
condition: and(succeeded(), ne(variables['Agent.Os'], 'Windows_NT'), eq(variables['_SignType'], 'real'))
condition: and(succeeded(), eq(variables['Agent.Os'], 'Windows_NT'), in(variables['_SignType'], 'real', 'test'))
- ${{ if eq(parameters.enableMicrobuildForMacAndLinux, true) }}:
- template: /eng/common/core-templates/steps/install-microbuild-impl.yml@self
parameters:
enablePreviewMicrobuild: ${{ parameters.enablePreviewMicrobuild }}
microbuildTaskInputs:
signType: $(_SignType)
zipSources: false
feedSource: https://dnceng.pkgs.visualstudio.com/_packaging/MicroBuildToolset/nuget/v3/index.json
version: ${{ parameters.microbuildPluginVersion }}
workingDirectory: ${{ parameters.microBuildOutputFolder }}
${{ if eq(parameters.microbuildUseESRP, true) }}:
ConnectedServiceName: 'MicroBuild Signing Task (DevDiv)'
${{ if eq(variables['System.TeamProject'], 'DevDiv') }}:
ConnectedPMEServiceName: beb8cb23-b303-4c95-ab26-9e44bc958d39
${{ else }}:
ConnectedPMEServiceName: c24de2a5-cc7a-493d-95e4-8e5ff5cad2bc
microbuildEnv:
TeamName: $(_TeamName)
MicroBuildOutputFolderOverride: ${{ parameters.microBuildOutputFolder }}
SYSTEM_ACCESSTOKEN: $(System.AccessToken)
continueOnError: ${{ parameters.continueOnError }}
condition: and(succeeded(), ne(variables['Agent.Os'], 'Windows_NT'), eq(variables['_SignType'], 'real'))

4
eng/common/core-templates/steps/publish-logs.yml

@ -26,8 +26,10 @@ steps:
# If the file exists - sensitive data for redaction will be sourced from it
# (single entry per line, lines starting with '# ' are considered comments and skipped)
arguments: -InputPath '$(System.DefaultWorkingDirectory)/PostBuildLogs'
-BinlogToolVersion ${{parameters.BinlogToolVersion}}
-BinlogToolVersion '${{parameters.BinlogToolVersion}}'
-TokensFilePath '$(System.DefaultWorkingDirectory)/eng/BinlogSecretsRedactionFile.txt'
-runtimeSourceFeed https://ci.dot.net/internal
-runtimeSourceFeedKey '$(dotnetbuilds-internal-container-read-token-base64)'
'$(publishing-dnceng-devdiv-code-r-build-re)'
'$(MaestroAccessToken)'
'$(dn-bot-all-orgs-artifact-feeds-rw)'

8
eng/common/core-templates/steps/source-index-stage1-publish.yml

@ -1,6 +1,6 @@
parameters:
sourceIndexUploadPackageVersion: 2.0.0-20250818.1
sourceIndexProcessBinlogPackageVersion: 1.0.1-20250818.1
sourceIndexUploadPackageVersion: 2.0.0-20250906.1
sourceIndexProcessBinlogPackageVersion: 1.0.1-20250906.1
sourceIndexPackageSource: https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools/nuget/v3/index.json
binlogPath: artifacts/log/Debug/Build.binlog
@ -14,8 +14,8 @@ steps:
workingDirectory: $(Agent.TempDirectory)
- script: |
$(Agent.TempDirectory)/dotnet/dotnet tool install BinLogToSln --version ${{parameters.sourceIndexProcessBinlogPackageVersion}} --add-source ${{parameters.SourceIndexPackageSource}} --tool-path $(Agent.TempDirectory)/.source-index/tools
$(Agent.TempDirectory)/dotnet/dotnet tool install UploadIndexStage1 --version ${{parameters.sourceIndexUploadPackageVersion}} --add-source ${{parameters.SourceIndexPackageSource}} --tool-path $(Agent.TempDirectory)/.source-index/tools
$(Agent.TempDirectory)/dotnet/dotnet tool install BinLogToSln --version ${{parameters.sourceIndexProcessBinlogPackageVersion}} --source ${{parameters.sourceIndexPackageSource}} --tool-path $(Agent.TempDirectory)/.source-index/tools
$(Agent.TempDirectory)/dotnet/dotnet tool install UploadIndexStage1 --version ${{parameters.sourceIndexUploadPackageVersion}} --source ${{parameters.sourceIndexPackageSource}} --tool-path $(Agent.TempDirectory)/.source-index/tools
displayName: "Source Index: Download netsourceindex Tools"
# Set working directory to temp directory so 'dotnet' doesn't try to use global.json and use the repo's sdk.
workingDirectory: $(Agent.TempDirectory)

11
eng/common/cross/arm/sources.list.bionic

@ -1,11 +0,0 @@
deb http://ports.ubuntu.com/ubuntu-ports/ bionic main restricted universe
deb-src http://ports.ubuntu.com/ubuntu-ports/ bionic main restricted universe
deb http://ports.ubuntu.com/ubuntu-ports/ bionic-updates main restricted universe
deb-src http://ports.ubuntu.com/ubuntu-ports/ bionic-updates main restricted universe
deb http://ports.ubuntu.com/ubuntu-ports/ bionic-backports main restricted
deb-src http://ports.ubuntu.com/ubuntu-ports/ bionic-backports main restricted
deb http://ports.ubuntu.com/ubuntu-ports/ bionic-security main restricted universe multiverse
deb-src http://ports.ubuntu.com/ubuntu-ports/ bionic-security main restricted universe multiverse

11
eng/common/cross/arm/sources.list.focal

@ -1,11 +0,0 @@
deb http://ports.ubuntu.com/ubuntu-ports/ focal main restricted universe
deb-src http://ports.ubuntu.com/ubuntu-ports/ focal main restricted universe
deb http://ports.ubuntu.com/ubuntu-ports/ focal-updates main restricted universe
deb-src http://ports.ubuntu.com/ubuntu-ports/ focal-updates main restricted universe
deb http://ports.ubuntu.com/ubuntu-ports/ focal-backports main restricted
deb-src http://ports.ubuntu.com/ubuntu-ports/ focal-backports main restricted
deb http://ports.ubuntu.com/ubuntu-ports/ focal-security main restricted universe multiverse
deb-src http://ports.ubuntu.com/ubuntu-ports/ focal-security main restricted universe multiverse

11
eng/common/cross/arm/sources.list.jammy

@ -1,11 +0,0 @@
deb http://ports.ubuntu.com/ubuntu-ports/ jammy main restricted universe
deb-src http://ports.ubuntu.com/ubuntu-ports/ jammy main restricted universe
deb http://ports.ubuntu.com/ubuntu-ports/ jammy-updates main restricted universe
deb-src http://ports.ubuntu.com/ubuntu-ports/ jammy-updates main restricted universe
deb http://ports.ubuntu.com/ubuntu-ports/ jammy-backports main restricted
deb-src http://ports.ubuntu.com/ubuntu-ports/ jammy-backports main restricted
deb http://ports.ubuntu.com/ubuntu-ports/ jammy-security main restricted universe multiverse
deb-src http://ports.ubuntu.com/ubuntu-ports/ jammy-security main restricted universe multiverse

3
eng/common/cross/arm/sources.list.jessie

@ -1,3 +0,0 @@
# Debian (sid) # UNSTABLE
deb http://ftp.debian.org/debian/ sid main contrib non-free
deb-src http://ftp.debian.org/debian/ sid main contrib non-free

11
eng/common/cross/arm/sources.list.trusty

@ -1,11 +0,0 @@
deb http://ports.ubuntu.com/ubuntu-ports/ trusty main restricted universe
deb-src http://ports.ubuntu.com/ubuntu-ports/ trusty main restricted universe
deb http://ports.ubuntu.com/ubuntu-ports/ trusty-updates main restricted universe
deb-src http://ports.ubuntu.com/ubuntu-ports/ trusty-updates main restricted universe
deb http://ports.ubuntu.com/ubuntu-ports/ trusty-backports main restricted
deb-src http://ports.ubuntu.com/ubuntu-ports/ trusty-backports main restricted
deb http://ports.ubuntu.com/ubuntu-ports/ trusty-security main restricted universe multiverse
deb-src http://ports.ubuntu.com/ubuntu-ports/ trusty-security main restricted universe multiverse

11
eng/common/cross/arm/sources.list.xenial

@ -1,11 +0,0 @@
deb http://ports.ubuntu.com/ubuntu-ports/ xenial main restricted universe
deb-src http://ports.ubuntu.com/ubuntu-ports/ xenial main restricted universe
deb http://ports.ubuntu.com/ubuntu-ports/ xenial-updates main restricted universe
deb-src http://ports.ubuntu.com/ubuntu-ports/ xenial-updates main restricted universe
deb http://ports.ubuntu.com/ubuntu-ports/ xenial-backports main restricted
deb-src http://ports.ubuntu.com/ubuntu-ports/ xenial-backports main restricted
deb http://ports.ubuntu.com/ubuntu-ports/ xenial-security main restricted universe multiverse
deb-src http://ports.ubuntu.com/ubuntu-ports/ xenial-security main restricted universe multiverse

11
eng/common/cross/arm/sources.list.zesty

@ -1,11 +0,0 @@
deb http://ports.ubuntu.com/ubuntu-ports/ zesty main restricted universe
deb-src http://ports.ubuntu.com/ubuntu-ports/ zesty main restricted universe
deb http://ports.ubuntu.com/ubuntu-ports/ zesty-updates main restricted universe
deb-src http://ports.ubuntu.com/ubuntu-ports/ zesty-updates main restricted universe
deb http://ports.ubuntu.com/ubuntu-ports/ zesty-backports main restricted
deb-src http://ports.ubuntu.com/ubuntu-ports/ zesty-backports main restricted
deb http://ports.ubuntu.com/ubuntu-ports/ zesty-security main restricted universe multiverse
deb-src http://ports.ubuntu.com/ubuntu-ports/ zesty-security main restricted universe multiverse

35
eng/common/cross/arm/tizen-build-rootfs.sh

@ -1,35 +0,0 @@
#!/usr/bin/env bash
set -e
__ARM_HARDFP_CrossDir=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )
__TIZEN_CROSSDIR="$__ARM_HARDFP_CrossDir/tizen"
if [[ -z "$ROOTFS_DIR" ]]; then
echo "ROOTFS_DIR is not defined."
exit 1;
fi
TIZEN_TMP_DIR=$ROOTFS_DIR/tizen_tmp
mkdir -p $TIZEN_TMP_DIR
# Download files
echo ">>Start downloading files"
VERBOSE=1 $__ARM_HARDFP_CrossDir/tizen-fetch.sh $TIZEN_TMP_DIR
echo "<<Finish downloading files"
echo ">>Start constructing Tizen rootfs"
TIZEN_RPM_FILES=`ls $TIZEN_TMP_DIR/*.rpm`
cd $ROOTFS_DIR
for f in $TIZEN_RPM_FILES; do
rpm2cpio $f | cpio -idm --quiet
done
echo "<<Finish constructing Tizen rootfs"
# Cleanup tmp
rm -rf $TIZEN_TMP_DIR
# Configure Tizen rootfs
echo ">>Start configuring Tizen rootfs"
ln -sfn asm-arm ./usr/include/asm
patch -p1 < $__TIZEN_CROSSDIR/tizen.patch
echo "<<Finish configuring Tizen rootfs"

170
eng/common/cross/arm/tizen-fetch.sh

@ -1,170 +0,0 @@
#!/usr/bin/env bash
set -e
if [[ -z "${VERBOSE// }" ]] || [ "$VERBOSE" -ne "$VERBOSE" ] 2>/dev/null; then
VERBOSE=0
fi
Log()
{
if [ $VERBOSE -ge $1 ]; then
echo ${@:2}
fi
}
Inform()
{
Log 1 -e "\x1B[0;34m$@\x1B[m"
}
Debug()
{
Log 2 -e "\x1B[0;32m$@\x1B[m"
}
Error()
{
>&2 Log 0 -e "\x1B[0;31m$@\x1B[m"
}
Fetch()
{
URL=$1
FILE=$2
PROGRESS=$3
if [ $VERBOSE -ge 1 ] && [ $PROGRESS ]; then
CURL_OPT="--progress-bar"
else
CURL_OPT="--silent"
fi
curl $CURL_OPT $URL > $FILE
}
hash curl 2> /dev/null || { Error "Require 'curl' Aborting."; exit 1; }
hash xmllint 2> /dev/null || { Error "Require 'xmllint' Aborting."; exit 1; }
hash sha256sum 2> /dev/null || { Error "Require 'sha256sum' Aborting."; exit 1; }
TMPDIR=$1
if [ ! -d $TMPDIR ]; then
TMPDIR=./tizen_tmp
Debug "Create temporary directory : $TMPDIR"
mkdir -p $TMPDIR
fi
TIZEN_URL=http://download.tizen.org/snapshots/tizen
BUILD_XML=build.xml
REPOMD_XML=repomd.xml
PRIMARY_XML=primary.xml
TARGET_URL="http://__not_initialized"
Xpath_get()
{
XPATH_RESULT=''
XPATH=$1
XML_FILE=$2
RESULT=$(xmllint --xpath $XPATH $XML_FILE)
if [[ -z ${RESULT// } ]]; then
Error "Can not find target from $XML_FILE"
Debug "Xpath = $XPATH"
exit 1
fi
XPATH_RESULT=$RESULT
}
fetch_tizen_pkgs_init()
{
TARGET=$1
PROFILE=$2
Debug "Initialize TARGET=$TARGET, PROFILE=$PROFILE"
TMP_PKG_DIR=$TMPDIR/tizen_${PROFILE}_pkgs
if [ -d $TMP_PKG_DIR ]; then rm -rf $TMP_PKG_DIR; fi
mkdir -p $TMP_PKG_DIR
PKG_URL=$TIZEN_URL/$PROFILE/latest
BUILD_XML_URL=$PKG_URL/$BUILD_XML
TMP_BUILD=$TMP_PKG_DIR/$BUILD_XML
TMP_REPOMD=$TMP_PKG_DIR/$REPOMD_XML
TMP_PRIMARY=$TMP_PKG_DIR/$PRIMARY_XML
TMP_PRIMARYGZ=${TMP_PRIMARY}.gz
Fetch $BUILD_XML_URL $TMP_BUILD
Debug "fetch $BUILD_XML_URL to $TMP_BUILD"
TARGET_XPATH="//build/buildtargets/buildtarget[@name=\"$TARGET\"]/repo[@type=\"binary\"]/text()"
Xpath_get $TARGET_XPATH $TMP_BUILD
TARGET_PATH=$XPATH_RESULT
TARGET_URL=$PKG_URL/$TARGET_PATH
REPOMD_URL=$TARGET_URL/repodata/repomd.xml
PRIMARY_XPATH='string(//*[local-name()="data"][@type="primary"]/*[local-name()="location"]/@href)'
Fetch $REPOMD_URL $TMP_REPOMD
Debug "fetch $REPOMD_URL to $TMP_REPOMD"
Xpath_get $PRIMARY_XPATH $TMP_REPOMD
PRIMARY_XML_PATH=$XPATH_RESULT
PRIMARY_URL=$TARGET_URL/$PRIMARY_XML_PATH
Fetch $PRIMARY_URL $TMP_PRIMARYGZ
Debug "fetch $PRIMARY_URL to $TMP_PRIMARYGZ"
gunzip $TMP_PRIMARYGZ
Debug "unzip $TMP_PRIMARYGZ to $TMP_PRIMARY"
}
fetch_tizen_pkgs()
{
ARCH=$1
PACKAGE_XPATH_TPL='string(//*[local-name()="metadata"]/*[local-name()="package"][*[local-name()="name"][text()="_PKG_"]][*[local-name()="arch"][text()="_ARCH_"]]/*[local-name()="location"]/@href)'
PACKAGE_CHECKSUM_XPATH_TPL='string(//*[local-name()="metadata"]/*[local-name()="package"][*[local-name()="name"][text()="_PKG_"]][*[local-name()="arch"][text()="_ARCH_"]]/*[local-name()="checksum"]/text())'
for pkg in ${@:2}
do
Inform "Fetching... $pkg"
XPATH=${PACKAGE_XPATH_TPL/_PKG_/$pkg}
XPATH=${XPATH/_ARCH_/$ARCH}
Xpath_get $XPATH $TMP_PRIMARY
PKG_PATH=$XPATH_RESULT
XPATH=${PACKAGE_CHECKSUM_XPATH_TPL/_PKG_/$pkg}
XPATH=${XPATH/_ARCH_/$ARCH}
Xpath_get $XPATH $TMP_PRIMARY
CHECKSUM=$XPATH_RESULT
PKG_URL=$TARGET_URL/$PKG_PATH
PKG_FILE=$(basename $PKG_PATH)
PKG_PATH=$TMPDIR/$PKG_FILE
Debug "Download $PKG_URL to $PKG_PATH"
Fetch $PKG_URL $PKG_PATH true
echo "$CHECKSUM $PKG_PATH" | sha256sum -c - > /dev/null
if [ $? -ne 0 ]; then
Error "Fail to fetch $PKG_URL to $PKG_PATH"
Debug "Checksum = $CHECKSUM"
exit 1
fi
done
}
Inform "Initialize arm base"
fetch_tizen_pkgs_init standard base
Inform "fetch common packages"
fetch_tizen_pkgs armv7hl gcc gcc-devel-static glibc glibc-devel libicu libicu-devel libatomic linux-glibc-devel keyutils keyutils-devel libkeyutils
Inform "fetch coreclr packages"
fetch_tizen_pkgs armv7hl lldb lldb-devel libgcc libstdc++ libstdc++-devel libunwind libunwind-devel lttng-ust-devel lttng-ust userspace-rcu-devel userspace-rcu
Inform "fetch corefx packages"
fetch_tizen_pkgs armv7hl libcom_err libcom_err-devel zlib zlib-devel libopenssl11 libopenssl1.1-devel krb5 krb5-devel
Inform "Initialize standard unified"
fetch_tizen_pkgs_init standard unified
Inform "fetch corefx packages"
fetch_tizen_pkgs armv7hl gssdp gssdp-devel tizen-release

71
eng/common/cross/arm/trusty-lttng-2.4.patch

@ -1,71 +0,0 @@
From e72c9d7ead60e3317bd6d1fade995c07021c947b Mon Sep 17 00:00:00 2001
From: Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
Date: Thu, 7 May 2015 13:25:04 -0400
Subject: [PATCH] Fix: building probe providers with C++ compiler
Robert Daniels wrote:
> > I'm attempting to use lttng userspace tracing with a C++ application
> > on an ARM platform. I'm using GCC 4.8.4 on Linux 3.14 with the 2.6
> > release of lttng. I've compiled lttng-modules, lttng-ust, and
> > lttng-tools and have been able to get a simple test working with C
> > code. When I attempt to run the hello.cxx test on my target it will
> > segfault.
>
>
> I spent a little time digging into this issue and finally discovered the
> cause of my segfault with ARM C++ tracepoints.
>
> There is a struct called 'lttng_event' in ust-events.h which contains an
> empty union 'u'. This was the cause of my issue. Under C, this empty union
> compiles to a zero byte member while under C++ it compiles to a one byte
> member, and in my case was four-byte aligned which caused my C++ code to
> have the 'cds_list_head node' offset incorrectly by four bytes. This lead
> to an incorrect linked list structure which caused my issue.
>
> Since this union is empty, I simply removed it from the struct and everything
> worked correctly.
>
> I don't know the history or purpose behind this empty union so I'd like to
> know if this is a safe fix. If it is I can submit a patch with the union
> removed.
That's a very nice catch!
We do not support building tracepoint probe provider with
g++ yet, as stated in lttng-ust(3):
"- Note for C++ support: although an application instrumented with
tracepoints can be compiled with g++, tracepoint probes should be
compiled with gcc (only tested with gcc so far)."
However, if it works fine with this fix, then I'm tempted to take it,
especially because removing the empty union does not appear to affect
the layout of struct lttng_event as seen from liblttng-ust, which must
be compiled with a C compiler, and from probe providers compiled with
a C compiler. So all we are changing is the layout of a probe provider
compiled with a C++ compiler, which is anyway buggy at the moment,
because it is not compatible with the layout expected by liblttng-ust
compiled with a C compiler.
Reported-by: Robert Daniels <robert.daniels@vantagecontrols.com>
Signed-off-by: Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
---
include/lttng/ust-events.h | 2 --
1 file changed, 2 deletions(-)
diff --git a/usr/include/lttng/ust-events.h b/usr/include/lttng/ust-events.h
index 328a875..3d7a274 100644
--- a/usr/include/lttng/ust-events.h
+++ b/usr/include/lttng/ust-events.h
@@ -407,8 +407,6 @@ struct lttng_event {
void *_deprecated1;
struct lttng_ctx *ctx;
enum lttng_ust_instrumentation instrumentation;
- union {
- } u;
struct cds_list_head node; /* Event list in session */
struct cds_list_head _deprecated2;
void *_deprecated3;
--
2.7.4

97
eng/common/cross/arm/trusty.patch

@ -1,97 +0,0 @@
diff -u -r a/usr/include/urcu/uatomic/generic.h b/usr/include/urcu/uatomic/generic.h
--- a/usr/include/urcu/uatomic/generic.h 2014-03-28 06:04:42.000000000 +0900
+++ b/usr/include/urcu/uatomic/generic.h 2017-02-13 10:35:21.189927116 +0900
@@ -65,17 +65,17 @@
switch (len) {
#ifdef UATOMIC_HAS_ATOMIC_BYTE
case 1:
- return __sync_val_compare_and_swap_1(addr, old, _new);
+ return __sync_val_compare_and_swap_1((uint8_t *) addr, old, _new);
#endif
#ifdef UATOMIC_HAS_ATOMIC_SHORT
case 2:
- return __sync_val_compare_and_swap_2(addr, old, _new);
+ return __sync_val_compare_and_swap_2((uint16_t *) addr, old, _new);
#endif
case 4:
- return __sync_val_compare_and_swap_4(addr, old, _new);
+ return __sync_val_compare_and_swap_4((uint32_t *) addr, old, _new);
#if (CAA_BITS_PER_LONG == 64)
case 8:
- return __sync_val_compare_and_swap_8(addr, old, _new);
+ return __sync_val_compare_and_swap_8((uint64_t *) addr, old, _new);
#endif
}
_uatomic_link_error();
@@ -100,20 +100,20 @@
switch (len) {
#ifdef UATOMIC_HAS_ATOMIC_BYTE
case 1:
- __sync_and_and_fetch_1(addr, val);
+ __sync_and_and_fetch_1((uint8_t *) addr, val);
return;
#endif
#ifdef UATOMIC_HAS_ATOMIC_SHORT
case 2:
- __sync_and_and_fetch_2(addr, val);
+ __sync_and_and_fetch_2((uint16_t *) addr, val);
return;
#endif
case 4:
- __sync_and_and_fetch_4(addr, val);
+ __sync_and_and_fetch_4((uint32_t *) addr, val);
return;
#if (CAA_BITS_PER_LONG == 64)
case 8:
- __sync_and_and_fetch_8(addr, val);
+ __sync_and_and_fetch_8((uint64_t *) addr, val);
return;
#endif
}
@@ -139,20 +139,20 @@
switch (len) {
#ifdef UATOMIC_HAS_ATOMIC_BYTE
case 1:
- __sync_or_and_fetch_1(addr, val);
+ __sync_or_and_fetch_1((uint8_t *) addr, val);
return;
#endif
#ifdef UATOMIC_HAS_ATOMIC_SHORT
case 2:
- __sync_or_and_fetch_2(addr, val);
+ __sync_or_and_fetch_2((uint16_t *) addr, val);
return;
#endif
case 4:
- __sync_or_and_fetch_4(addr, val);
+ __sync_or_and_fetch_4((uint32_t *) addr, val);
return;
#if (CAA_BITS_PER_LONG == 64)
case 8:
- __sync_or_and_fetch_8(addr, val);
+ __sync_or_and_fetch_8((uint64_t *) addr, val);
return;
#endif
}
@@ -180,17 +180,17 @@
switch (len) {
#ifdef UATOMIC_HAS_ATOMIC_BYTE
case 1:
- return __sync_add_and_fetch_1(addr, val);
+ return __sync_add_and_fetch_1((uint8_t *) addr, val);
#endif
#ifdef UATOMIC_HAS_ATOMIC_SHORT
case 2:
- return __sync_add_and_fetch_2(addr, val);
+ return __sync_add_and_fetch_2((uint16_t *) addr, val);
#endif
case 4:
- return __sync_add_and_fetch_4(addr, val);
+ return __sync_add_and_fetch_4((uint32_t *) addr, val);
#if (CAA_BITS_PER_LONG == 64)
case 8:
- return __sync_add_and_fetch_8(addr, val);
+ return __sync_add_and_fetch_8((uint64_t *) addr, val);
#endif
}
_uatomic_link_error();

11
eng/common/cross/arm64/sources.list.bionic

@ -1,11 +0,0 @@
deb http://ports.ubuntu.com/ubuntu-ports/ bionic main restricted universe
deb-src http://ports.ubuntu.com/ubuntu-ports/ bionic main restricted universe
deb http://ports.ubuntu.com/ubuntu-ports/ bionic-updates main restricted universe
deb-src http://ports.ubuntu.com/ubuntu-ports/ bionic-updates main restricted universe
deb http://ports.ubuntu.com/ubuntu-ports/ bionic-backports main restricted
deb-src http://ports.ubuntu.com/ubuntu-ports/ bionic-backports main restricted
deb http://ports.ubuntu.com/ubuntu-ports/ bionic-security main restricted universe multiverse
deb-src http://ports.ubuntu.com/ubuntu-ports/ bionic-security main restricted universe multiverse

11
eng/common/cross/arm64/sources.list.buster

@ -1,11 +0,0 @@
deb http://deb.debian.org/debian buster main
deb-src http://deb.debian.org/debian buster main
deb http://deb.debian.org/debian-security/ buster/updates main
deb-src http://deb.debian.org/debian-security/ buster/updates main
deb http://deb.debian.org/debian buster-updates main
deb-src http://deb.debian.org/debian buster-updates main
deb http://deb.debian.org/debian buster-backports main contrib non-free
deb-src http://deb.debian.org/debian buster-backports main contrib non-free

11
eng/common/cross/arm64/sources.list.focal

@ -1,11 +0,0 @@
deb http://ports.ubuntu.com/ubuntu-ports/ focal main restricted universe
deb-src http://ports.ubuntu.com/ubuntu-ports/ focal main restricted universe
deb http://ports.ubuntu.com/ubuntu-ports/ focal-updates main restricted universe
deb-src http://ports.ubuntu.com/ubuntu-ports/ focal-updates main restricted universe
deb http://ports.ubuntu.com/ubuntu-ports/ focal-backports main restricted
deb-src http://ports.ubuntu.com/ubuntu-ports/ focal-backports main restricted
deb http://ports.ubuntu.com/ubuntu-ports/ focal-security main restricted universe multiverse
deb-src http://ports.ubuntu.com/ubuntu-ports/ focal-security main restricted universe multiverse

11
eng/common/cross/arm64/sources.list.jammy

@ -1,11 +0,0 @@
deb http://ports.ubuntu.com/ubuntu-ports/ jammy main restricted universe
deb-src http://ports.ubuntu.com/ubuntu-ports/ jammy main restricted universe
deb http://ports.ubuntu.com/ubuntu-ports/ jammy-updates main restricted universe
deb-src http://ports.ubuntu.com/ubuntu-ports/ jammy-updates main restricted universe
deb http://ports.ubuntu.com/ubuntu-ports/ jammy-backports main restricted
deb-src http://ports.ubuntu.com/ubuntu-ports/ jammy-backports main restricted
deb http://ports.ubuntu.com/ubuntu-ports/ jammy-security main restricted universe multiverse
deb-src http://ports.ubuntu.com/ubuntu-ports/ jammy-security main restricted universe multiverse

12
eng/common/cross/arm64/sources.list.stretch

@ -1,12 +0,0 @@
deb http://deb.debian.org/debian stretch main
deb-src http://deb.debian.org/debian stretch main
deb http://deb.debian.org/debian-security/ stretch/updates main
deb-src http://deb.debian.org/debian-security/ stretch/updates main
deb http://deb.debian.org/debian stretch-updates main
deb-src http://deb.debian.org/debian stretch-updates main
deb http://deb.debian.org/debian stretch-backports main contrib non-free
deb-src http://deb.debian.org/debian stretch-backports main contrib non-free

11
eng/common/cross/arm64/sources.list.trusty

@ -1,11 +0,0 @@
deb http://ports.ubuntu.com/ubuntu-ports/ trusty main restricted universe
deb-src http://ports.ubuntu.com/ubuntu-ports/ trusty main restricted universe
deb http://ports.ubuntu.com/ubuntu-ports/ trusty-updates main restricted universe
deb-src http://ports.ubuntu.com/ubuntu-ports/ trusty-updates main restricted universe
deb http://ports.ubuntu.com/ubuntu-ports/ trusty-backports main restricted
deb-src http://ports.ubuntu.com/ubuntu-ports/ trusty-backports main restricted
deb http://ports.ubuntu.com/ubuntu-ports/ trusty-security main restricted universe multiverse
deb-src http://ports.ubuntu.com/ubuntu-ports/ trusty-security main restricted universe multiverse

11
eng/common/cross/arm64/sources.list.xenial

@ -1,11 +0,0 @@
deb http://ports.ubuntu.com/ubuntu-ports/ xenial main restricted universe
deb-src http://ports.ubuntu.com/ubuntu-ports/ xenial main restricted universe
deb http://ports.ubuntu.com/ubuntu-ports/ xenial-updates main restricted universe
deb-src http://ports.ubuntu.com/ubuntu-ports/ xenial-updates main restricted universe
deb http://ports.ubuntu.com/ubuntu-ports/ xenial-backports main restricted
deb-src http://ports.ubuntu.com/ubuntu-ports/ xenial-backports main restricted
deb http://ports.ubuntu.com/ubuntu-ports/ xenial-security main restricted universe multiverse
deb-src http://ports.ubuntu.com/ubuntu-ports/ xenial-security main restricted universe multiverse

11
eng/common/cross/arm64/sources.list.zesty

@ -1,11 +0,0 @@
deb http://ports.ubuntu.com/ubuntu-ports/ zesty main restricted universe
deb-src http://ports.ubuntu.com/ubuntu-ports/ zesty main restricted universe
deb http://ports.ubuntu.com/ubuntu-ports/ zesty-updates main restricted universe
deb-src http://ports.ubuntu.com/ubuntu-ports/ zesty-updates main restricted universe
deb http://ports.ubuntu.com/ubuntu-ports/ zesty-backports main restricted
deb-src http://ports.ubuntu.com/ubuntu-ports/ zesty-backports main restricted
deb http://ports.ubuntu.com/ubuntu-ports/ zesty-security main restricted universe multiverse
deb-src http://ports.ubuntu.com/ubuntu-ports/ zesty-security main restricted universe multiverse

35
eng/common/cross/arm64/tizen-build-rootfs.sh

@ -1,35 +0,0 @@
#!/usr/bin/env bash
set -e
__CrossDir=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )
__TIZEN_CROSSDIR="$__CrossDir/tizen"
if [[ -z "$ROOTFS_DIR" ]]; then
echo "ROOTFS_DIR is not defined."
exit 1;
fi
TIZEN_TMP_DIR=$ROOTFS_DIR/tizen_tmp
mkdir -p $TIZEN_TMP_DIR
# Download files
echo ">>Start downloading files"
VERBOSE=1 $__CrossDir/tizen-fetch.sh $TIZEN_TMP_DIR
echo "<<Finish downloading files"
echo ">>Start constructing Tizen rootfs"
TIZEN_RPM_FILES=`ls $TIZEN_TMP_DIR/*.rpm`
cd $ROOTFS_DIR
for f in $TIZEN_RPM_FILES; do
rpm2cpio $f | cpio -idm --quiet
done
echo "<<Finish constructing Tizen rootfs"
# Cleanup tmp
rm -rf $TIZEN_TMP_DIR
# Configure Tizen rootfs
echo ">>Start configuring Tizen rootfs"
ln -sfn asm-arm64 ./usr/include/asm
patch -p1 < $__TIZEN_CROSSDIR/tizen.patch
echo "<<Finish configuring Tizen rootfs"

170
eng/common/cross/arm64/tizen-fetch.sh

@ -1,170 +0,0 @@
#!/usr/bin/env bash
set -e
if [[ -z "${VERBOSE// }" ]] || [ "$VERBOSE" -ne "$VERBOSE" ] 2>/dev/null; then
VERBOSE=0
fi
Log()
{
if [ $VERBOSE -ge $1 ]; then
echo ${@:2}
fi
}
Inform()
{
Log 1 -e "\x1B[0;34m$@\x1B[m"
}
Debug()
{
Log 2 -e "\x1B[0;32m$@\x1B[m"
}
Error()
{
>&2 Log 0 -e "\x1B[0;31m$@\x1B[m"
}
Fetch()
{
URL=$1
FILE=$2
PROGRESS=$3
if [ $VERBOSE -ge 1 ] && [ $PROGRESS ]; then
CURL_OPT="--progress-bar"
else
CURL_OPT="--silent"
fi
curl $CURL_OPT $URL > $FILE
}
hash curl 2> /dev/null || { Error "Require 'curl' Aborting."; exit 1; }
hash xmllint 2> /dev/null || { Error "Require 'xmllint' Aborting."; exit 1; }
hash sha256sum 2> /dev/null || { Error "Require 'sha256sum' Aborting."; exit 1; }
TMPDIR=$1
if [ ! -d $TMPDIR ]; then
TMPDIR=./tizen_tmp
Debug "Create temporary directory : $TMPDIR"
mkdir -p $TMPDIR
fi
TIZEN_URL=http://download.tizen.org/snapshots/tizen/
BUILD_XML=build.xml
REPOMD_XML=repomd.xml
PRIMARY_XML=primary.xml
TARGET_URL="http://__not_initialized"
Xpath_get()
{
XPATH_RESULT=''
XPATH=$1
XML_FILE=$2
RESULT=$(xmllint --xpath $XPATH $XML_FILE)
if [[ -z ${RESULT// } ]]; then
Error "Can not find target from $XML_FILE"
Debug "Xpath = $XPATH"
exit 1
fi
XPATH_RESULT=$RESULT
}
fetch_tizen_pkgs_init()
{
TARGET=$1
PROFILE=$2
Debug "Initialize TARGET=$TARGET, PROFILE=$PROFILE"
TMP_PKG_DIR=$TMPDIR/tizen_${PROFILE}_pkgs
if [ -d $TMP_PKG_DIR ]; then rm -rf $TMP_PKG_DIR; fi
mkdir -p $TMP_PKG_DIR
PKG_URL=$TIZEN_URL/$PROFILE/latest
BUILD_XML_URL=$PKG_URL/$BUILD_XML
TMP_BUILD=$TMP_PKG_DIR/$BUILD_XML
TMP_REPOMD=$TMP_PKG_DIR/$REPOMD_XML
TMP_PRIMARY=$TMP_PKG_DIR/$PRIMARY_XML
TMP_PRIMARYGZ=${TMP_PRIMARY}.gz
Fetch $BUILD_XML_URL $TMP_BUILD
Debug "fetch $BUILD_XML_URL to $TMP_BUILD"
TARGET_XPATH="//build/buildtargets/buildtarget[@name=\"$TARGET\"]/repo[@type=\"binary\"]/text()"
Xpath_get $TARGET_XPATH $TMP_BUILD
TARGET_PATH=$XPATH_RESULT
TARGET_URL=$PKG_URL/$TARGET_PATH
REPOMD_URL=$TARGET_URL/repodata/repomd.xml
PRIMARY_XPATH='string(//*[local-name()="data"][@type="primary"]/*[local-name()="location"]/@href)'
Fetch $REPOMD_URL $TMP_REPOMD
Debug "fetch $REPOMD_URL to $TMP_REPOMD"
Xpath_get $PRIMARY_XPATH $TMP_REPOMD
PRIMARY_XML_PATH=$XPATH_RESULT
PRIMARY_URL=$TARGET_URL/$PRIMARY_XML_PATH
Fetch $PRIMARY_URL $TMP_PRIMARYGZ
Debug "fetch $PRIMARY_URL to $TMP_PRIMARYGZ"
gunzip $TMP_PRIMARYGZ
Debug "unzip $TMP_PRIMARYGZ to $TMP_PRIMARY"
}
fetch_tizen_pkgs()
{
ARCH=$1
PACKAGE_XPATH_TPL='string(//*[local-name()="metadata"]/*[local-name()="package"][*[local-name()="name"][text()="_PKG_"]][*[local-name()="arch"][text()="_ARCH_"]]/*[local-name()="location"]/@href)'
PACKAGE_CHECKSUM_XPATH_TPL='string(//*[local-name()="metadata"]/*[local-name()="package"][*[local-name()="name"][text()="_PKG_"]][*[local-name()="arch"][text()="_ARCH_"]]/*[local-name()="checksum"]/text())'
for pkg in ${@:2}
do
Inform "Fetching... $pkg"
XPATH=${PACKAGE_XPATH_TPL/_PKG_/$pkg}
XPATH=${XPATH/_ARCH_/$ARCH}
Xpath_get $XPATH $TMP_PRIMARY
PKG_PATH=$XPATH_RESULT
XPATH=${PACKAGE_CHECKSUM_XPATH_TPL/_PKG_/$pkg}
XPATH=${XPATH/_ARCH_/$ARCH}
Xpath_get $XPATH $TMP_PRIMARY
CHECKSUM=$XPATH_RESULT
PKG_URL=$TARGET_URL/$PKG_PATH
PKG_FILE=$(basename $PKG_PATH)
PKG_PATH=$TMPDIR/$PKG_FILE
Debug "Download $PKG_URL to $PKG_PATH"
Fetch $PKG_URL $PKG_PATH true
echo "$CHECKSUM $PKG_PATH" | sha256sum -c - > /dev/null
if [ $? -ne 0 ]; then
Error "Fail to fetch $PKG_URL to $PKG_PATH"
Debug "Checksum = $CHECKSUM"
exit 1
fi
done
}
Inform "Initialize arm base"
fetch_tizen_pkgs_init standard base
Inform "fetch common packages"
fetch_tizen_pkgs aarch64 gcc glibc glibc-devel libicu libicu-devel libatomic linux-glibc-devel keyutils keyutils-devel libkeyutils
Inform "fetch coreclr packages"
fetch_tizen_pkgs aarch64 lldb lldb-devel libgcc libstdc++ libstdc++-devel libunwind libunwind-devel lttng-ust-devel lttng-ust userspace-rcu-devel userspace-rcu
Inform "fetch corefx packages"
fetch_tizen_pkgs aarch64 libcom_err libcom_err-devel zlib zlib-devel libopenssl11 libopenssl1.1-devel krb5 krb5-devel
Inform "Initialize standard unified"
fetch_tizen_pkgs_init standard unified
Inform "fetch corefx packages"
fetch_tizen_pkgs aarch64 gssdp gssdp-devel tizen-release

43
eng/common/cross/armel/armel.jessie.patch

@ -1,43 +0,0 @@
diff -u -r a/usr/include/urcu/uatomic/generic.h b/usr/include/urcu/uatomic/generic.h
--- a/usr/include/urcu/uatomic/generic.h 2014-10-22 15:00:58.000000000 -0700
+++ b/usr/include/urcu/uatomic/generic.h 2020-10-30 21:38:28.550000000 -0700
@@ -69,10 +69,10 @@
#endif
#ifdef UATOMIC_HAS_ATOMIC_SHORT
case 2:
- return __sync_val_compare_and_swap_2(addr, old, _new);
+ return __sync_val_compare_and_swap_2((uint16_t*) addr, old, _new);
#endif
case 4:
- return __sync_val_compare_and_swap_4(addr, old, _new);
+ return __sync_val_compare_and_swap_4((uint32_t*) addr, old, _new);
#if (CAA_BITS_PER_LONG == 64)
case 8:
return __sync_val_compare_and_swap_8(addr, old, _new);
@@ -109,7 +109,7 @@
return;
#endif
case 4:
- __sync_and_and_fetch_4(addr, val);
+ __sync_and_and_fetch_4((uint32_t*) addr, val);
return;
#if (CAA_BITS_PER_LONG == 64)
case 8:
@@ -148,7 +148,7 @@
return;
#endif
case 4:
- __sync_or_and_fetch_4(addr, val);
+ __sync_or_and_fetch_4((uint32_t*) addr, val);
return;
#if (CAA_BITS_PER_LONG == 64)
case 8:
@@ -187,7 +187,7 @@
return __sync_add_and_fetch_2(addr, val);
#endif
case 4:
- return __sync_add_and_fetch_4(addr, val);
+ return __sync_add_and_fetch_4((uint32_t*) addr, val);
#if (CAA_BITS_PER_LONG == 64)
case 8:
return __sync_add_and_fetch_8(addr, val);

3
eng/common/cross/armel/sources.list.jessie

@ -1,3 +0,0 @@
# Debian (jessie) # Stable
deb http://ftp.debian.org/debian/ jessie main contrib non-free
deb-src http://ftp.debian.org/debian/ jessie main contrib non-free

35
eng/common/cross/armel/tizen-build-rootfs.sh

@ -1,35 +0,0 @@
#!/usr/bin/env bash
set -e
__ARM_SOFTFP_CrossDir=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )
__TIZEN_CROSSDIR="$__ARM_SOFTFP_CrossDir/tizen"
if [[ -z "$ROOTFS_DIR" ]]; then
echo "ROOTFS_DIR is not defined."
exit 1;
fi
TIZEN_TMP_DIR=$ROOTFS_DIR/tizen_tmp
mkdir -p $TIZEN_TMP_DIR
# Download files
echo ">>Start downloading files"
VERBOSE=1 $__ARM_SOFTFP_CrossDir/tizen-fetch.sh $TIZEN_TMP_DIR
echo "<<Finish downloading files"
echo ">>Start constructing Tizen rootfs"
TIZEN_RPM_FILES=`ls $TIZEN_TMP_DIR/*.rpm`
cd $ROOTFS_DIR
for f in $TIZEN_RPM_FILES; do
rpm2cpio $f | cpio -idm --quiet
done
echo "<<Finish constructing Tizen rootfs"
# Cleanup tmp
rm -rf $TIZEN_TMP_DIR
# Configure Tizen rootfs
echo ">>Start configuring Tizen rootfs"
ln -sfn asm-arm ./usr/include/asm
patch -p1 < $__TIZEN_CROSSDIR/tizen.patch
echo "<<Finish configuring Tizen rootfs"

170
eng/common/cross/armel/tizen-fetch.sh

@ -1,170 +0,0 @@
#!/usr/bin/env bash
set -e
if [[ -z "${VERBOSE// }" ]] || [ "$VERBOSE" -ne "$VERBOSE" ] 2>/dev/null; then
VERBOSE=0
fi
Log()
{
if [ $VERBOSE -ge $1 ]; then
echo ${@:2}
fi
}
Inform()
{
Log 1 -e "\x1B[0;34m$@\x1B[m"
}
Debug()
{
Log 2 -e "\x1B[0;32m$@\x1B[m"
}
Error()
{
>&2 Log 0 -e "\x1B[0;31m$@\x1B[m"
}
Fetch()
{
URL=$1
FILE=$2
PROGRESS=$3
if [ $VERBOSE -ge 1 ] && [ $PROGRESS ]; then
CURL_OPT="--progress-bar"
else
CURL_OPT="--silent"
fi
curl $CURL_OPT $URL > $FILE
}
hash curl 2> /dev/null || { Error "Require 'curl' Aborting."; exit 1; }
hash xmllint 2> /dev/null || { Error "Require 'xmllint' Aborting."; exit 1; }
hash sha256sum 2> /dev/null || { Error "Require 'sha256sum' Aborting."; exit 1; }
TMPDIR=$1
if [ ! -d $TMPDIR ]; then
TMPDIR=./tizen_tmp
Debug "Create temporary directory : $TMPDIR"
mkdir -p $TMPDIR
fi
TIZEN_URL=http://download.tizen.org/snapshots/tizen
BUILD_XML=build.xml
REPOMD_XML=repomd.xml
PRIMARY_XML=primary.xml
TARGET_URL="http://__not_initialized"
Xpath_get()
{
XPATH_RESULT=''
XPATH=$1
XML_FILE=$2
RESULT=$(xmllint --xpath $XPATH $XML_FILE)
if [[ -z ${RESULT// } ]]; then
Error "Can not find target from $XML_FILE"
Debug "Xpath = $XPATH"
exit 1
fi
XPATH_RESULT=$RESULT
}
fetch_tizen_pkgs_init()
{
TARGET=$1
PROFILE=$2
Debug "Initialize TARGET=$TARGET, PROFILE=$PROFILE"
TMP_PKG_DIR=$TMPDIR/tizen_${PROFILE}_pkgs
if [ -d $TMP_PKG_DIR ]; then rm -rf $TMP_PKG_DIR; fi
mkdir -p $TMP_PKG_DIR
PKG_URL=$TIZEN_URL/$PROFILE/latest
BUILD_XML_URL=$PKG_URL/$BUILD_XML
TMP_BUILD=$TMP_PKG_DIR/$BUILD_XML
TMP_REPOMD=$TMP_PKG_DIR/$REPOMD_XML
TMP_PRIMARY=$TMP_PKG_DIR/$PRIMARY_XML
TMP_PRIMARYGZ=${TMP_PRIMARY}.gz
Fetch $BUILD_XML_URL $TMP_BUILD
Debug "fetch $BUILD_XML_URL to $TMP_BUILD"
TARGET_XPATH="//build/buildtargets/buildtarget[@name=\"$TARGET\"]/repo[@type=\"binary\"]/text()"
Xpath_get $TARGET_XPATH $TMP_BUILD
TARGET_PATH=$XPATH_RESULT
TARGET_URL=$PKG_URL/$TARGET_PATH
REPOMD_URL=$TARGET_URL/repodata/repomd.xml
PRIMARY_XPATH='string(//*[local-name()="data"][@type="primary"]/*[local-name()="location"]/@href)'
Fetch $REPOMD_URL $TMP_REPOMD
Debug "fetch $REPOMD_URL to $TMP_REPOMD"
Xpath_get $PRIMARY_XPATH $TMP_REPOMD
PRIMARY_XML_PATH=$XPATH_RESULT
PRIMARY_URL=$TARGET_URL/$PRIMARY_XML_PATH
Fetch $PRIMARY_URL $TMP_PRIMARYGZ
Debug "fetch $PRIMARY_URL to $TMP_PRIMARYGZ"
gunzip $TMP_PRIMARYGZ
Debug "unzip $TMP_PRIMARYGZ to $TMP_PRIMARY"
}
fetch_tizen_pkgs()
{
ARCH=$1
PACKAGE_XPATH_TPL='string(//*[local-name()="metadata"]/*[local-name()="package"][*[local-name()="name"][text()="_PKG_"]][*[local-name()="arch"][text()="_ARCH_"]]/*[local-name()="location"]/@href)'
PACKAGE_CHECKSUM_XPATH_TPL='string(//*[local-name()="metadata"]/*[local-name()="package"][*[local-name()="name"][text()="_PKG_"]][*[local-name()="arch"][text()="_ARCH_"]]/*[local-name()="checksum"]/text())'
for pkg in ${@:2}
do
Inform "Fetching... $pkg"
XPATH=${PACKAGE_XPATH_TPL/_PKG_/$pkg}
XPATH=${XPATH/_ARCH_/$ARCH}
Xpath_get $XPATH $TMP_PRIMARY
PKG_PATH=$XPATH_RESULT
XPATH=${PACKAGE_CHECKSUM_XPATH_TPL/_PKG_/$pkg}
XPATH=${XPATH/_ARCH_/$ARCH}
Xpath_get $XPATH $TMP_PRIMARY
CHECKSUM=$XPATH_RESULT
PKG_URL=$TARGET_URL/$PKG_PATH
PKG_FILE=$(basename $PKG_PATH)
PKG_PATH=$TMPDIR/$PKG_FILE
Debug "Download $PKG_URL to $PKG_PATH"
Fetch $PKG_URL $PKG_PATH true
echo "$CHECKSUM $PKG_PATH" | sha256sum -c - > /dev/null
if [ $? -ne 0 ]; then
Error "Fail to fetch $PKG_URL to $PKG_PATH"
Debug "Checksum = $CHECKSUM"
exit 1
fi
done
}
Inform "Initialize arm base"
fetch_tizen_pkgs_init standard base
Inform "fetch common packages"
fetch_tizen_pkgs armv7l gcc gcc-devel-static glibc glibc-devel libicu libicu-devel libatomic linux-glibc-devel keyutils keyutils-devel libkeyutils
Inform "fetch coreclr packages"
fetch_tizen_pkgs armv7l lldb lldb-devel libgcc libstdc++ libstdc++-devel libunwind libunwind-devel lttng-ust-devel lttng-ust userspace-rcu-devel userspace-rcu
Inform "fetch corefx packages"
fetch_tizen_pkgs armv7l libcom_err libcom_err-devel zlib zlib-devel libopenssl11 libopenssl1.1-devel krb5 krb5-devel
Inform "Initialize standard unified"
fetch_tizen_pkgs_init standard unified
Inform "fetch corefx packages"
fetch_tizen_pkgs armv7l gssdp gssdp-devel tizen-release

50
eng/common/cross/armel/tizen/tizen-dotnet.ks

@ -1,50 +0,0 @@
lang en_US.UTF-8
keyboard us
timezone --utc Asia/Seoul
part / --fstype="ext4" --size=3500 --ondisk=mmcblk0 --label rootfs --fsoptions=defaults,noatime
rootpw tizen
desktop --autologinuser=root
user --name root --groups audio,video --password 'tizen'
repo --name=standard --baseurl=http://download.tizen.org/releases/milestone/tizen/unified/latest/repos/standard/packages/ --ssl_verify=no
repo --name=base --baseurl=http://download.tizen.org/releases/milestone/tizen/base/latest/repos/standard/packages/ --ssl_verify=no
%packages
tar
gzip
sed
grep
gawk
perl
binutils
findutils
util-linux
lttng-ust
userspace-rcu
procps-ng
tzdata
ca-certificates
### Core FX
libicu
libunwind
iputils
zlib
krb5
libcurl
libopenssl
%end
%post
### Update /tmp privilege
chmod 777 /tmp
####################################
%end

2
eng/common/cross/armv6/sources.list.buster

@ -1,2 +0,0 @@
deb http://raspbian.raspberrypi.org/raspbian/ buster main contrib non-free rpi
deb-src http://raspbian.raspberrypi.org/raspbian/ buster main contrib non-free rpi

8
eng/common/cross/build-rootfs.sh

@ -72,7 +72,7 @@ __AlpinePackages+=" krb5-dev"
__AlpinePackages+=" openssl-dev"
__AlpinePackages+=" zlib-dev"
__FreeBSDBase="13.4-RELEASE"
__FreeBSDBase="13.5-RELEASE"
__FreeBSDPkg="1.21.3"
__FreeBSDABI="13"
__FreeBSDPackages="libunwind"
@ -295,9 +295,7 @@ while :; do
;;
noble) # Ubuntu 24.04
__CodeName=noble
if [[ -z "$__LLDB_Package" ]]; then
__LLDB_Package="liblldb-19-dev"
fi
__LLDB_Package="liblldb-19-dev"
;;
stretch) # Debian 9
__CodeName=stretch
@ -383,7 +381,7 @@ while :; do
;;
freebsd14)
__CodeName=freebsd
__FreeBSDBase="14.2-RELEASE"
__FreeBSDBase="14.3-RELEASE"
__FreeBSDABI="14"
__SkipUnmount=1
;;

11
eng/common/cross/ppc64le/sources.list.bionic

@ -1,11 +0,0 @@
deb http://ports.ubuntu.com/ubuntu-ports/ bionic main restricted universe
deb-src http://ports.ubuntu.com/ubuntu-ports/ bionic main restricted universe
deb http://ports.ubuntu.com/ubuntu-ports/ bionic-updates main restricted universe
deb-src http://ports.ubuntu.com/ubuntu-ports/ bionic-updates main restricted universe
deb http://ports.ubuntu.com/ubuntu-ports/ bionic-backports main restricted
deb-src http://ports.ubuntu.com/ubuntu-ports/ bionic-backports main restricted
deb http://ports.ubuntu.com/ubuntu-ports/ bionic-security main restricted universe multiverse
deb-src http://ports.ubuntu.com/ubuntu-ports/ bionic-security main restricted universe multiverse

1
eng/common/cross/riscv64/sources.list.sid

@ -1 +0,0 @@
deb http://deb.debian.org/debian-ports sid main

11
eng/common/cross/s390x/sources.list.bionic

@ -1,11 +0,0 @@
deb http://ports.ubuntu.com/ubuntu-ports/ bionic main restricted universe
deb-src http://ports.ubuntu.com/ubuntu-ports/ bionic main restricted universe
deb http://ports.ubuntu.com/ubuntu-ports/ bionic-updates main restricted universe
deb-src http://ports.ubuntu.com/ubuntu-ports/ bionic-updates main restricted universe
deb http://ports.ubuntu.com/ubuntu-ports/ bionic-backports main restricted
deb-src http://ports.ubuntu.com/ubuntu-ports/ bionic-backports main restricted
deb http://ports.ubuntu.com/ubuntu-ports/ bionic-security main restricted universe multiverse
deb-src http://ports.ubuntu.com/ubuntu-ports/ bionic-security main restricted universe multiverse

11
eng/common/cross/x86/sources.list.bionic

@ -1,11 +0,0 @@
deb http://archive.ubuntu.com/ubuntu/ bionic main restricted universe
deb-src http://archive.ubuntu.com/ubuntu/ bionic main restricted universe
deb http://archive.ubuntu.com/ubuntu/ bionic-updates main restricted universe
deb-src http://archive.ubuntu.com/ubuntu/ bionic-updates main restricted universe
deb http://archive.ubuntu.com/ubuntu/ bionic-backports main restricted
deb-src http://archive.ubuntu.com/ubuntu/ bionic-backports main restricted
deb http://archive.ubuntu.com/ubuntu/ bionic-security main restricted universe multiverse
deb-src http://archive.ubuntu.com/ubuntu/ bionic-security main restricted universe multiverse

11
eng/common/cross/x86/sources.list.focal

@ -1,11 +0,0 @@
deb http://archive.ubuntu.com/ubuntu/ focal main restricted universe
deb-src http://archive.ubuntu.com/ubuntu/ focal main restricted universe
deb http://archive.ubuntu.com/ubuntu/ focal-updates main restricted universe
deb-src http://archive.ubuntu.com/ubuntu/ focal-updates main restricted universe
deb http://archive.ubuntu.com/ubuntu/ focal-backports main restricted
deb-src http://archive.ubuntu.com/ubuntu/ focal-backports main restricted
deb http://archive.ubuntu.com/ubuntu/ focal-security main restricted universe multiverse
deb-src http://archive.ubuntu.com/ubuntu/ focal-security main restricted universe multiverse

11
eng/common/cross/x86/sources.list.jammy

@ -1,11 +0,0 @@
deb http://archive.ubuntu.com/ubuntu/ jammy main restricted universe
deb-src http://archive.ubuntu.com/ubuntu/ jammy main restricted universe
deb http://archive.ubuntu.com/ubuntu/ jammy-updates main restricted universe
deb-src http://archive.ubuntu.com/ubuntu/ jammy-updates main restricted universe
deb http://archive.ubuntu.com/ubuntu/ jammy-backports main restricted
deb-src http://archive.ubuntu.com/ubuntu/ jammy-backports main restricted
deb http://archive.ubuntu.com/ubuntu/ jammy-security main restricted universe multiverse
deb-src http://archive.ubuntu.com/ubuntu/ jammy-security main restricted universe multiverse

11
eng/common/cross/x86/sources.list.trusty

@ -1,11 +0,0 @@
deb http://archive.ubuntu.com/ubuntu/ trusty main restricted universe
deb-src http://archive.ubuntu.com/ubuntu/ trusty main restricted universe
deb http://archive.ubuntu.com/ubuntu/ trusty-updates main restricted universe
deb-src http://archive.ubuntu.com/ubuntu/ trusty-updates main restricted universe
deb http://archive.ubuntu.com/ubuntu/ trusty-backports main restricted
deb-src http://archive.ubuntu.com/ubuntu/ trusty-backports main restricted
deb http://archive.ubuntu.com/ubuntu/ trusty-security main restricted universe multiverse
deb-src http://archive.ubuntu.com/ubuntu/ trusty-security main restricted universe multiverse

11
eng/common/cross/x86/sources.list.xenial

@ -1,11 +0,0 @@
deb http://archive.ubuntu.com/ubuntu/ xenial main restricted universe
deb-src http://archive.ubuntu.com/ubuntu/ xenial main restricted universe
deb http://archive.ubuntu.com/ubuntu/ xenial-updates main restricted universe
deb-src http://archive.ubuntu.com/ubuntu/ xenial-updates main restricted universe
deb http://archive.ubuntu.com/ubuntu/ xenial-backports main restricted
deb-src http://archive.ubuntu.com/ubuntu/ xenial-backports main restricted
deb http://archive.ubuntu.com/ubuntu/ xenial-security main restricted universe multiverse
deb-src http://archive.ubuntu.com/ubuntu/ xenial-security main restricted universe multiverse

35
eng/common/cross/x86/tizen-build-rootfs.sh

@ -1,35 +0,0 @@
#!/usr/bin/env bash
set -e
__X86_CrossDir=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )
__TIZEN_CROSSDIR="$__X86_CrossDir/tizen"
if [[ -z "$ROOTFS_DIR" ]]; then
echo "ROOTFS_DIR is not defined."
exit 1;
fi
TIZEN_TMP_DIR=$ROOTFS_DIR/tizen_tmp
mkdir -p $TIZEN_TMP_DIR
# Download files
echo ">>Start downloading files"
VERBOSE=1 $__X86_CrossDir/tizen-fetch.sh $TIZEN_TMP_DIR
echo "<<Finish downloading files"
echo ">>Start constructing Tizen rootfs"
TIZEN_RPM_FILES=`ls $TIZEN_TMP_DIR/*.rpm`
cd $ROOTFS_DIR
for f in $TIZEN_RPM_FILES; do
rpm2cpio $f | cpio -idm --quiet
done
echo "<<Finish constructing Tizen rootfs"
# Cleanup tmp
rm -rf $TIZEN_TMP_DIR
# Configure Tizen rootfs
echo ">>Start configuring Tizen rootfs"
ln -sfn asm-x86 ./usr/include/asm
patch -p1 < $__TIZEN_CROSSDIR/tizen.patch
echo "<<Finish configuring Tizen rootfs"

170
eng/common/cross/x86/tizen-fetch.sh

@ -1,170 +0,0 @@
#!/usr/bin/env bash
set -e
if [[ -z "${VERBOSE// }" ]] || [ "$VERBOSE" -ne "$VERBOSE" ] 2>/dev/null; then
VERBOSE=0
fi
Log()
{
if [ $VERBOSE -ge $1 ]; then
echo ${@:2}
fi
}
Inform()
{
Log 1 -e "\x1B[0;34m$@\x1B[m"
}
Debug()
{
Log 2 -e "\x1B[0;32m$@\x1B[m"
}
Error()
{
>&2 Log 0 -e "\x1B[0;31m$@\x1B[m"
}
Fetch()
{
URL=$1
FILE=$2
PROGRESS=$3
if [ $VERBOSE -ge 1 ] && [ $PROGRESS ]; then
CURL_OPT="--progress-bar"
else
CURL_OPT="--silent"
fi
curl $CURL_OPT $URL > $FILE
}
hash curl 2> /dev/null || { Error "Require 'curl' Aborting."; exit 1; }
hash xmllint 2> /dev/null || { Error "Require 'xmllint' Aborting."; exit 1; }
hash sha256sum 2> /dev/null || { Error "Require 'sha256sum' Aborting."; exit 1; }
TMPDIR=$1
if [ ! -d $TMPDIR ]; then
TMPDIR=./tizen_tmp
Debug "Create temporary directory : $TMPDIR"
mkdir -p $TMPDIR
fi
TIZEN_URL=http://download.tizen.org/snapshots/tizen
BUILD_XML=build.xml
REPOMD_XML=repomd.xml
PRIMARY_XML=primary.xml
TARGET_URL="http://__not_initialized"
Xpath_get()
{
XPATH_RESULT=''
XPATH=$1
XML_FILE=$2
RESULT=$(xmllint --xpath $XPATH $XML_FILE)
if [[ -z ${RESULT// } ]]; then
Error "Can not find target from $XML_FILE"
Debug "Xpath = $XPATH"
exit 1
fi
XPATH_RESULT=$RESULT
}
fetch_tizen_pkgs_init()
{
TARGET=$1
PROFILE=$2
Debug "Initialize TARGET=$TARGET, PROFILE=$PROFILE"
TMP_PKG_DIR=$TMPDIR/tizen_${PROFILE}_pkgs
if [ -d $TMP_PKG_DIR ]; then rm -rf $TMP_PKG_DIR; fi
mkdir -p $TMP_PKG_DIR
PKG_URL=$TIZEN_URL/$PROFILE/latest
BUILD_XML_URL=$PKG_URL/$BUILD_XML
TMP_BUILD=$TMP_PKG_DIR/$BUILD_XML
TMP_REPOMD=$TMP_PKG_DIR/$REPOMD_XML
TMP_PRIMARY=$TMP_PKG_DIR/$PRIMARY_XML
TMP_PRIMARYGZ=${TMP_PRIMARY}.gz
Fetch $BUILD_XML_URL $TMP_BUILD
Debug "fetch $BUILD_XML_URL to $TMP_BUILD"
TARGET_XPATH="//build/buildtargets/buildtarget[@name=\"$TARGET\"]/repo[@type=\"binary\"]/text()"
Xpath_get $TARGET_XPATH $TMP_BUILD
TARGET_PATH=$XPATH_RESULT
TARGET_URL=$PKG_URL/$TARGET_PATH
REPOMD_URL=$TARGET_URL/repodata/repomd.xml
PRIMARY_XPATH='string(//*[local-name()="data"][@type="primary"]/*[local-name()="location"]/@href)'
Fetch $REPOMD_URL $TMP_REPOMD
Debug "fetch $REPOMD_URL to $TMP_REPOMD"
Xpath_get $PRIMARY_XPATH $TMP_REPOMD
PRIMARY_XML_PATH=$XPATH_RESULT
PRIMARY_URL=$TARGET_URL/$PRIMARY_XML_PATH
Fetch $PRIMARY_URL $TMP_PRIMARYGZ
Debug "fetch $PRIMARY_URL to $TMP_PRIMARYGZ"
gunzip $TMP_PRIMARYGZ
Debug "unzip $TMP_PRIMARYGZ to $TMP_PRIMARY"
}
fetch_tizen_pkgs()
{
ARCH=$1
PACKAGE_XPATH_TPL='string(//*[local-name()="metadata"]/*[local-name()="package"][*[local-name()="name"][text()="_PKG_"]][*[local-name()="arch"][text()="_ARCH_"]]/*[local-name()="location"]/@href)'
PACKAGE_CHECKSUM_XPATH_TPL='string(//*[local-name()="metadata"]/*[local-name()="package"][*[local-name()="name"][text()="_PKG_"]][*[local-name()="arch"][text()="_ARCH_"]]/*[local-name()="checksum"]/text())'
for pkg in ${@:2}
do
Inform "Fetching... $pkg"
XPATH=${PACKAGE_XPATH_TPL/_PKG_/$pkg}
XPATH=${XPATH/_ARCH_/$ARCH}
Xpath_get $XPATH $TMP_PRIMARY
PKG_PATH=$XPATH_RESULT
XPATH=${PACKAGE_CHECKSUM_XPATH_TPL/_PKG_/$pkg}
XPATH=${XPATH/_ARCH_/$ARCH}
Xpath_get $XPATH $TMP_PRIMARY
CHECKSUM=$XPATH_RESULT
PKG_URL=$TARGET_URL/$PKG_PATH
PKG_FILE=$(basename $PKG_PATH)
PKG_PATH=$TMPDIR/$PKG_FILE
Debug "Download $PKG_URL to $PKG_PATH"
Fetch $PKG_URL $PKG_PATH true
echo "$CHECKSUM $PKG_PATH" | sha256sum -c - > /dev/null
if [ $? -ne 0 ]; then
Error "Fail to fetch $PKG_URL to $PKG_PATH"
Debug "Checksum = $CHECKSUM"
exit 1
fi
done
}
Inform "Initialize i686 base"
fetch_tizen_pkgs_init standard base
Inform "fetch common packages"
fetch_tizen_pkgs i686 gcc gcc-devel-static glibc glibc-devel libicu libicu-devel libatomic linux-glibc-devel keyutils keyutils-devel libkeyutils
Inform "fetch coreclr packages"
fetch_tizen_pkgs i686 lldb lldb-devel libgcc libstdc++ libstdc++-devel libunwind libunwind-devel lttng-ust-devel lttng-ust userspace-rcu-devel userspace-rcu
Inform "fetch corefx packages"
fetch_tizen_pkgs i686 libcom_err libcom_err-devel zlib zlib-devel libopenssl11 libopenssl1.1-devel krb5 krb5-devel
Inform "Initialize standard unified"
fetch_tizen_pkgs_init standard unified
Inform "fetch corefx packages"
fetch_tizen_pkgs i686 gssdp gssdp-devel tizen-release

2
eng/common/darc-init.sh

@ -5,7 +5,7 @@ darcVersion=''
versionEndpoint='https://maestro.dot.net/api/assets/darc-version?api-version=2020-02-20'
verbosity='minimal'
while [[ $# > 0 ]]; do
while [[ $# -gt 0 ]]; do
opt="$(echo "$1" | tr "[:upper:]" "[:lower:]")"
case "$opt" in
--darcversion)

774
eng/common/dotnet-install-scripts/dotnet-install.ps1

@ -1,774 +0,0 @@
#
# Copyright (c) .NET Foundation and contributors. All rights reserved.
# Licensed under the MIT license. See LICENSE file in the project root for full license information.
#
# Copied from https://dot.net/v1/dotnet-install.ps1 on 8/26/2020
<#
.SYNOPSIS
Installs dotnet cli
.DESCRIPTION
Installs dotnet cli. If dotnet installation already exists in the given directory
it will update it only if the requested version differs from the one already installed.
.PARAMETER Channel
Default: LTS
Download from the Channel specified. Possible values:
- Current - most current release
- LTS - most current supported release
- 2-part version in a format A.B - represents a specific release
examples: 2.0, 1.0
- Branch name
examples: release/2.0.0, Master
Note: The version parameter overrides the channel parameter.
.PARAMETER Version
Default: latest
Represents a build version on specific channel. Possible values:
- latest - most latest build on specific channel
- coherent - most latest coherent build on specific channel
coherent applies only to SDK downloads
- 3-part version in a format A.B.C - represents specific version of build
examples: 2.0.0-preview2-006120, 1.1.0
.PARAMETER InstallDir
Default: %LocalAppData%\Microsoft\dotnet
Path to where to install dotnet. Note that binaries will be placed directly in a given directory.
.PARAMETER Architecture
Default: <auto> - this value represents currently running OS architecture
Architecture of dotnet binaries to be installed.
Possible values are: <auto>, amd64, x64, x86, arm64, arm
.PARAMETER SharedRuntime
This parameter is obsolete and may be removed in a future version of this script.
The recommended alternative is '-Runtime dotnet'.
Installs just the shared runtime bits, not the entire SDK.
.PARAMETER Runtime
Installs just a shared runtime, not the entire SDK.
Possible values:
- dotnet - the Microsoft.NETCore.App shared runtime
- aspnetcore - the Microsoft.AspNetCore.App shared runtime
- windowsdesktop - the Microsoft.WindowsDesktop.App shared runtime
.PARAMETER DryRun
If set it will not perform installation but instead display what command line to use to consistently install
currently requested version of dotnet cli. In example if you specify version 'latest' it will display a link
with specific version so that this command can be used deterministicly in a build script.
It also displays binaries location if you prefer to install or download it yourself.
.PARAMETER NoPath
By default this script will set environment variable PATH for the current process to the binaries folder inside installation folder.
If set it will display binaries location but not set any environment variable.
.PARAMETER Verbose
Displays diagnostics information.
.PARAMETER AzureFeed
Default: https://dotnetcli.azureedge.net/dotnet
This parameter typically is not changed by the user.
It allows changing the URL for the Azure feed used by this installer.
.PARAMETER UncachedFeed
This parameter typically is not changed by the user.
It allows changing the URL for the Uncached feed used by this installer.
.PARAMETER FeedCredential
Used as a query string to append to the Azure feed.
It allows changing the URL to use non-public blob storage accounts.
.PARAMETER ProxyAddress
If set, the installer will use the proxy when making web requests
.PARAMETER ProxyUseDefaultCredentials
Default: false
Use default credentials, when using proxy address.
.PARAMETER ProxyBypassList
If set with ProxyAddress, will provide the list of comma separated urls that will bypass the proxy
.PARAMETER SkipNonVersionedFiles
Default: false
Skips installing non-versioned files if they already exist, such as dotnet.exe.
.PARAMETER NoCdn
Disable downloading from the Azure CDN, and use the uncached feed directly.
.PARAMETER JSonFile
Determines the SDK version from a user specified global.json file
Note: global.json must have a value for 'SDK:Version'
#>
[cmdletbinding()]
param(
[string]$Channel="LTS",
[string]$Version="Latest",
[string]$JSonFile,
[string]$InstallDir="<auto>",
[string]$Architecture="<auto>",
[ValidateSet("dotnet", "aspnetcore", "windowsdesktop", IgnoreCase = $false)]
[string]$Runtime,
[Obsolete("This parameter may be removed in a future version of this script. The recommended alternative is '-Runtime dotnet'.")]
[switch]$SharedRuntime,
[switch]$DryRun,
[switch]$NoPath,
[string]$AzureFeed="https://dotnetcli.azureedge.net/dotnet",
[string]$UncachedFeed="https://dotnetcli.blob.core.windows.net/dotnet",
[string]$FeedCredential,
[string]$ProxyAddress,
[switch]$ProxyUseDefaultCredentials,
[string[]]$ProxyBypassList=@(),
[switch]$SkipNonVersionedFiles,
[switch]$NoCdn
)
Set-StrictMode -Version Latest
$ErrorActionPreference="Stop"
$ProgressPreference="SilentlyContinue"
if ($NoCdn) {
$AzureFeed = $UncachedFeed
}
$BinFolderRelativePath=""
if ($SharedRuntime -and (-not $Runtime)) {
$Runtime = "dotnet"
}
# example path with regex: shared/1.0.0-beta-12345/somepath
$VersionRegEx="/\d+\.\d+[^/]+/"
$OverrideNonVersionedFiles = !$SkipNonVersionedFiles
function Say($str) {
try
{
Write-Host "dotnet-install: $str"
}
catch
{
# Some platforms cannot utilize Write-Host (Azure Functions, for instance). Fall back to Write-Output
Write-Output "dotnet-install: $str"
}
}
function Say-Verbose($str) {
try
{
Write-Verbose "dotnet-install: $str"
}
catch
{
# Some platforms cannot utilize Write-Verbose (Azure Functions, for instance). Fall back to Write-Output
Write-Output "dotnet-install: $str"
}
}
function Say-Invocation($Invocation) {
$command = $Invocation.MyCommand;
$args = (($Invocation.BoundParameters.Keys | foreach { "-$_ `"$($Invocation.BoundParameters[$_])`"" }) -join " ")
Say-Verbose "$command $args"
}
function Invoke-With-Retry([ScriptBlock]$ScriptBlock, [int]$MaxAttempts = 3, [int]$SecondsBetweenAttempts = 1) {
$Attempts = 0
while ($true) {
try {
return $ScriptBlock.Invoke()
}
catch {
$Attempts++
if ($Attempts -lt $MaxAttempts) {
Start-Sleep $SecondsBetweenAttempts
}
else {
throw
}
}
}
}
function Get-Machine-Architecture() {
Say-Invocation $MyInvocation
# On PS x86, PROCESSOR_ARCHITECTURE reports x86 even on x64 systems.
# To get the correct architecture, we need to use PROCESSOR_ARCHITEW6432.
# PS x64 doesn't define this, so we fall back to PROCESSOR_ARCHITECTURE.
# Possible values: amd64, x64, x86, arm64, arm
if( $ENV:PROCESSOR_ARCHITEW6432 -ne $null )
{
return $ENV:PROCESSOR_ARCHITEW6432
}
return $ENV:PROCESSOR_ARCHITECTURE
}
function Get-CLIArchitecture-From-Architecture([string]$Architecture) {
Say-Invocation $MyInvocation
switch ($Architecture.ToLower()) {
{ $_ -eq "<auto>" } { return Get-CLIArchitecture-From-Architecture $(Get-Machine-Architecture) }
{ ($_ -eq "amd64") -or ($_ -eq "x64") } { return "x64" }
{ $_ -eq "x86" } { return "x86" }
{ $_ -eq "arm" } { return "arm" }
{ $_ -eq "arm64" } { return "arm64" }
default { throw "Architecture not supported. If you think this is a bug, report it at https://github.com/dotnet/sdk/issues" }
}
}
# The version text returned from the feeds is a 1-line or 2-line string:
# For the SDK and the dotnet runtime (2 lines):
# Line 1: # commit_hash
# Line 2: # 4-part version
# For the aspnetcore runtime (1 line):
# Line 1: # 4-part version
function Get-Version-Info-From-Version-Text([string]$VersionText) {
Say-Invocation $MyInvocation
$Data = -split $VersionText
$VersionInfo = @{
CommitHash = $(if ($Data.Count -gt 1) { $Data[0] })
Version = $Data[-1] # last line is always the version number.
}
return $VersionInfo
}
function Load-Assembly([string] $Assembly) {
try {
Add-Type -Assembly $Assembly | Out-Null
}
catch {
# On Nano Server, Powershell Core Edition is used. Add-Type is unable to resolve base class assemblies because they are not GAC'd.
# Loading the base class assemblies is not unnecessary as the types will automatically get resolved.
}
}
function GetHTTPResponse([Uri] $Uri)
{
Invoke-With-Retry(
{
$HttpClient = $null
try {
# HttpClient is used vs Invoke-WebRequest in order to support Nano Server which doesn't support the Invoke-WebRequest cmdlet.
Load-Assembly -Assembly System.Net.Http
if(-not $ProxyAddress) {
try {
# Despite no proxy being explicitly specified, we may still be behind a default proxy
$DefaultProxy = [System.Net.WebRequest]::DefaultWebProxy;
if($DefaultProxy -and (-not $DefaultProxy.IsBypassed($Uri))) {
$ProxyAddress = $DefaultProxy.GetProxy($Uri).OriginalString
$ProxyUseDefaultCredentials = $true
}
} catch {
# Eat the exception and move forward as the above code is an attempt
# at resolving the DefaultProxy that may not have been a problem.
$ProxyAddress = $null
Say-Verbose("Exception ignored: $_.Exception.Message - moving forward...")
}
}
if($ProxyAddress) {
$HttpClientHandler = New-Object System.Net.Http.HttpClientHandler
$HttpClientHandler.Proxy = New-Object System.Net.WebProxy -Property @{
Address=$ProxyAddress;
UseDefaultCredentials=$ProxyUseDefaultCredentials;
BypassList = $ProxyBypassList;
}
$HttpClient = New-Object System.Net.Http.HttpClient -ArgumentList $HttpClientHandler
}
else {
$HttpClient = New-Object System.Net.Http.HttpClient
}
# Default timeout for HttpClient is 100s. For a 50 MB download this assumes 500 KB/s average, any less will time out
# 20 minutes allows it to work over much slower connections.
$HttpClient.Timeout = New-TimeSpan -Minutes 20
$Response = $HttpClient.GetAsync("${Uri}${FeedCredential}").Result
if (($Response -eq $null) -or (-not ($Response.IsSuccessStatusCode))) {
# The feed credential is potentially sensitive info. Do not log FeedCredential to console output.
$ErrorMsg = "Failed to download $Uri."
if ($Response -ne $null) {
$ErrorMsg += " $Response"
}
throw $ErrorMsg
}
return $Response
}
finally {
if ($HttpClient -ne $null) {
$HttpClient.Dispose()
}
}
})
}
function Get-Latest-Version-Info([string]$AzureFeed, [string]$Channel, [bool]$Coherent) {
Say-Invocation $MyInvocation
$VersionFileUrl = $null
if ($Runtime -eq "dotnet") {
$VersionFileUrl = "$UncachedFeed/Runtime/$Channel/latest.version"
}
elseif ($Runtime -eq "aspnetcore") {
$VersionFileUrl = "$UncachedFeed/aspnetcore/Runtime/$Channel/latest.version"
}
# Currently, the WindowsDesktop runtime is manufactured with the .Net core runtime
elseif ($Runtime -eq "windowsdesktop") {
$VersionFileUrl = "$UncachedFeed/Runtime/$Channel/latest.version"
}
elseif (-not $Runtime) {
if ($Coherent) {
$VersionFileUrl = "$UncachedFeed/Sdk/$Channel/latest.coherent.version"
}
else {
$VersionFileUrl = "$UncachedFeed/Sdk/$Channel/latest.version"
}
}
else {
throw "Invalid value for `$Runtime"
}
try {
$Response = GetHTTPResponse -Uri $VersionFileUrl
}
catch {
throw "Could not resolve version information."
}
$StringContent = $Response.Content.ReadAsStringAsync().Result
switch ($Response.Content.Headers.ContentType) {
{ ($_ -eq "application/octet-stream") } { $VersionText = $StringContent }
{ ($_ -eq "text/plain") } { $VersionText = $StringContent }
{ ($_ -eq "text/plain; charset=UTF-8") } { $VersionText = $StringContent }
default { throw "``$Response.Content.Headers.ContentType`` is an unknown .version file content type." }
}
$VersionInfo = Get-Version-Info-From-Version-Text $VersionText
return $VersionInfo
}
function Parse-Jsonfile-For-Version([string]$JSonFile) {
Say-Invocation $MyInvocation
If (-Not (Test-Path $JSonFile)) {
throw "Unable to find '$JSonFile'"
}
try {
$JSonContent = Get-Content($JSonFile) -Raw | ConvertFrom-Json | Select-Object -expand "sdk" -ErrorAction SilentlyContinue
}
catch {
throw "Json file unreadable: '$JSonFile'"
}
if ($JSonContent) {
try {
$JSonContent.PSObject.Properties | ForEach-Object {
$PropertyName = $_.Name
if ($PropertyName -eq "version") {
$Version = $_.Value
Say-Verbose "Version = $Version"
}
}
}
catch {
throw "Unable to parse the SDK node in '$JSonFile'"
}
}
else {
throw "Unable to find the SDK node in '$JSonFile'"
}
If ($Version -eq $null) {
throw "Unable to find the SDK:version node in '$JSonFile'"
}
return $Version
}
function Get-Specific-Version-From-Version([string]$AzureFeed, [string]$Channel, [string]$Version, [string]$JSonFile) {
Say-Invocation $MyInvocation
if (-not $JSonFile) {
switch ($Version.ToLower()) {
{ $_ -eq "latest" } {
$LatestVersionInfo = Get-Latest-Version-Info -AzureFeed $AzureFeed -Channel $Channel -Coherent $False
return $LatestVersionInfo.Version
}
{ $_ -eq "coherent" } {
$LatestVersionInfo = Get-Latest-Version-Info -AzureFeed $AzureFeed -Channel $Channel -Coherent $True
return $LatestVersionInfo.Version
}
default { return $Version }
}
}
else {
return Parse-Jsonfile-For-Version $JSonFile
}
}
function Get-Download-Link([string]$AzureFeed, [string]$SpecificVersion, [string]$CLIArchitecture) {
Say-Invocation $MyInvocation
# If anything fails in this lookup it will default to $SpecificVersion
$SpecificProductVersion = Get-Product-Version -AzureFeed $AzureFeed -SpecificVersion $SpecificVersion
if ($Runtime -eq "dotnet") {
$PayloadURL = "$AzureFeed/Runtime/$SpecificVersion/dotnet-runtime-$SpecificProductVersion-win-$CLIArchitecture.zip"
}
elseif ($Runtime -eq "aspnetcore") {
$PayloadURL = "$AzureFeed/aspnetcore/Runtime/$SpecificVersion/aspnetcore-runtime-$SpecificProductVersion-win-$CLIArchitecture.zip"
}
elseif ($Runtime -eq "windowsdesktop") {
$PayloadURL = "$AzureFeed/Runtime/$SpecificVersion/windowsdesktop-runtime-$SpecificProductVersion-win-$CLIArchitecture.zip"
}
elseif (-not $Runtime) {
$PayloadURL = "$AzureFeed/Sdk/$SpecificVersion/dotnet-sdk-$SpecificProductVersion-win-$CLIArchitecture.zip"
}
else {
throw "Invalid value for `$Runtime"
}
Say-Verbose "Constructed primary named payload URL: $PayloadURL"
return $PayloadURL, $SpecificProductVersion
}
function Get-LegacyDownload-Link([string]$AzureFeed, [string]$SpecificVersion, [string]$CLIArchitecture) {
Say-Invocation $MyInvocation
if (-not $Runtime) {
$PayloadURL = "$AzureFeed/Sdk/$SpecificVersion/dotnet-dev-win-$CLIArchitecture.$SpecificVersion.zip"
}
elseif ($Runtime -eq "dotnet") {
$PayloadURL = "$AzureFeed/Runtime/$SpecificVersion/dotnet-win-$CLIArchitecture.$SpecificVersion.zip"
}
else {
return $null
}
Say-Verbose "Constructed legacy named payload URL: $PayloadURL"
return $PayloadURL
}
function Get-Product-Version([string]$AzureFeed, [string]$SpecificVersion) {
Say-Invocation $MyInvocation
if ($Runtime -eq "dotnet") {
$ProductVersionTxtURL = "$AzureFeed/Runtime/$SpecificVersion/productVersion.txt"
}
elseif ($Runtime -eq "aspnetcore") {
$ProductVersionTxtURL = "$AzureFeed/aspnetcore/Runtime/$SpecificVersion/productVersion.txt"
}
elseif ($Runtime -eq "windowsdesktop") {
$ProductVersionTxtURL = "$AzureFeed/Runtime/$SpecificVersion/productVersion.txt"
}
elseif (-not $Runtime) {
$ProductVersionTxtURL = "$AzureFeed/Sdk/$SpecificVersion/productVersion.txt"
}
else {
throw "Invalid value specified for `$Runtime"
}
Say-Verbose "Checking for existence of $ProductVersionTxtURL"
try {
$productVersionResponse = GetHTTPResponse($productVersionTxtUrl)
if ($productVersionResponse.StatusCode -eq 200) {
$productVersion = $productVersionResponse.Content.ReadAsStringAsync().Result.Trim()
if ($productVersion -ne $SpecificVersion)
{
Say "Using alternate version $productVersion found in $ProductVersionTxtURL"
}
return $productVersion
}
else {
Say-Verbose "Got StatusCode $($productVersionResponse.StatusCode) trying to get productVersion.txt at $productVersionTxtUrl, so using default value of $SpecificVersion"
$productVersion = $SpecificVersion
}
} catch {
Say-Verbose "Could not read productVersion.txt at $productVersionTxtUrl, so using default value of $SpecificVersion"
$productVersion = $SpecificVersion
}
return $productVersion
}
function Get-User-Share-Path() {
Say-Invocation $MyInvocation
$InstallRoot = $env:DOTNET_INSTALL_DIR
if (!$InstallRoot) {
$InstallRoot = "$env:LocalAppData\Microsoft\dotnet"
}
return $InstallRoot
}
function Resolve-Installation-Path([string]$InstallDir) {
Say-Invocation $MyInvocation
if ($InstallDir -eq "<auto>") {
return Get-User-Share-Path
}
return $InstallDir
}
function Is-Dotnet-Package-Installed([string]$InstallRoot, [string]$RelativePathToPackage, [string]$SpecificVersion) {
Say-Invocation $MyInvocation
$DotnetPackagePath = Join-Path -Path $InstallRoot -ChildPath $RelativePathToPackage | Join-Path -ChildPath $SpecificVersion
Say-Verbose "Is-Dotnet-Package-Installed: DotnetPackagePath=$DotnetPackagePath"
return Test-Path $DotnetPackagePath -PathType Container
}
function Get-Absolute-Path([string]$RelativeOrAbsolutePath) {
# Too much spam
# Say-Invocation $MyInvocation
return $ExecutionContext.SessionState.Path.GetUnresolvedProviderPathFromPSPath($RelativeOrAbsolutePath)
}
function Get-Path-Prefix-With-Version($path) {
$match = [regex]::match($path, $VersionRegEx)
if ($match.Success) {
return $entry.FullName.Substring(0, $match.Index + $match.Length)
}
return $null
}
function Get-List-Of-Directories-And-Versions-To-Unpack-From-Dotnet-Package([System.IO.Compression.ZipArchive]$Zip, [string]$OutPath) {
Say-Invocation $MyInvocation
$ret = @()
foreach ($entry in $Zip.Entries) {
$dir = Get-Path-Prefix-With-Version $entry.FullName
if ($dir -ne $null) {
$path = Get-Absolute-Path $(Join-Path -Path $OutPath -ChildPath $dir)
if (-Not (Test-Path $path -PathType Container)) {
$ret += $dir
}
}
}
$ret = $ret | Sort-Object | Get-Unique
$values = ($ret | foreach { "$_" }) -join ";"
Say-Verbose "Directories to unpack: $values"
return $ret
}
# Example zip content and extraction algorithm:
# Rule: files if extracted are always being extracted to the same relative path locally
# .\
# a.exe # file does not exist locally, extract
# b.dll # file exists locally, override only if $OverrideFiles set
# aaa\ # same rules as for files
# ...
# abc\1.0.0\ # directory contains version and exists locally
# ... # do not extract content under versioned part
# abc\asd\ # same rules as for files
# ...
# def\ghi\1.0.1\ # directory contains version and does not exist locally
# ... # extract content
function Extract-Dotnet-Package([string]$ZipPath, [string]$OutPath) {
Say-Invocation $MyInvocation
Load-Assembly -Assembly System.IO.Compression.FileSystem
Set-Variable -Name Zip
try {
$Zip = [System.IO.Compression.ZipFile]::OpenRead($ZipPath)
$DirectoriesToUnpack = Get-List-Of-Directories-And-Versions-To-Unpack-From-Dotnet-Package -Zip $Zip -OutPath $OutPath
foreach ($entry in $Zip.Entries) {
$PathWithVersion = Get-Path-Prefix-With-Version $entry.FullName
if (($PathWithVersion -eq $null) -Or ($DirectoriesToUnpack -contains $PathWithVersion)) {
$DestinationPath = Get-Absolute-Path $(Join-Path -Path $OutPath -ChildPath $entry.FullName)
$DestinationDir = Split-Path -Parent $DestinationPath
$OverrideFiles=$OverrideNonVersionedFiles -Or (-Not (Test-Path $DestinationPath))
if ((-Not $DestinationPath.EndsWith("\")) -And $OverrideFiles) {
New-Item -ItemType Directory -Force -Path $DestinationDir | Out-Null
[System.IO.Compression.ZipFileExtensions]::ExtractToFile($entry, $DestinationPath, $OverrideNonVersionedFiles)
}
}
}
}
finally {
if ($Zip -ne $null) {
$Zip.Dispose()
}
}
}
function DownloadFile($Source, [string]$OutPath) {
if ($Source -notlike "http*") {
# Using System.IO.Path.GetFullPath to get the current directory
# does not work in this context - $pwd gives the current directory
if (![System.IO.Path]::IsPathRooted($Source)) {
$Source = $(Join-Path -Path $pwd -ChildPath $Source)
}
$Source = Get-Absolute-Path $Source
Say "Copying file from $Source to $OutPath"
Copy-Item $Source $OutPath
return
}
$Stream = $null
try {
$Response = GetHTTPResponse -Uri $Source
$Stream = $Response.Content.ReadAsStreamAsync().Result
$File = [System.IO.File]::Create($OutPath)
$Stream.CopyTo($File)
$File.Close()
}
finally {
if ($Stream -ne $null) {
$Stream.Dispose()
}
}
}
function Prepend-Sdk-InstallRoot-To-Path([string]$InstallRoot, [string]$BinFolderRelativePath) {
$BinPath = Get-Absolute-Path $(Join-Path -Path $InstallRoot -ChildPath $BinFolderRelativePath)
if (-Not $NoPath) {
$SuffixedBinPath = "$BinPath;"
if (-Not $env:path.Contains($SuffixedBinPath)) {
Say "Adding to current process PATH: `"$BinPath`". Note: This change will not be visible if PowerShell was run as a child process."
$env:path = $SuffixedBinPath + $env:path
} else {
Say-Verbose "Current process PATH already contains `"$BinPath`""
}
}
else {
Say "Binaries of dotnet can be found in $BinPath"
}
}
$CLIArchitecture = Get-CLIArchitecture-From-Architecture $Architecture
$SpecificVersion = Get-Specific-Version-From-Version -AzureFeed $AzureFeed -Channel $Channel -Version $Version -JSonFile $JSonFile
$DownloadLink, $EffectiveVersion = Get-Download-Link -AzureFeed $AzureFeed -SpecificVersion $SpecificVersion -CLIArchitecture $CLIArchitecture
$LegacyDownloadLink = Get-LegacyDownload-Link -AzureFeed $AzureFeed -SpecificVersion $SpecificVersion -CLIArchitecture $CLIArchitecture
$InstallRoot = Resolve-Installation-Path $InstallDir
Say-Verbose "InstallRoot: $InstallRoot"
$ScriptName = $MyInvocation.MyCommand.Name
if ($DryRun) {
Say "Payload URLs:"
Say "Primary named payload URL: $DownloadLink"
if ($LegacyDownloadLink) {
Say "Legacy named payload URL: $LegacyDownloadLink"
}
$RepeatableCommand = ".\$ScriptName -Version `"$SpecificVersion`" -InstallDir `"$InstallRoot`" -Architecture `"$CLIArchitecture`""
if ($Runtime -eq "dotnet") {
$RepeatableCommand+=" -Runtime `"dotnet`""
}
elseif ($Runtime -eq "aspnetcore") {
$RepeatableCommand+=" -Runtime `"aspnetcore`""
}
foreach ($key in $MyInvocation.BoundParameters.Keys) {
if (-not (@("Architecture","Channel","DryRun","InstallDir","Runtime","SharedRuntime","Version") -contains $key)) {
$RepeatableCommand+=" -$key `"$($MyInvocation.BoundParameters[$key])`""
}
}
Say "Repeatable invocation: $RepeatableCommand"
exit 0
}
if ($Runtime -eq "dotnet") {
$assetName = ".NET Core Runtime"
$dotnetPackageRelativePath = "shared\Microsoft.NETCore.App"
}
elseif ($Runtime -eq "aspnetcore") {
$assetName = "ASP.NET Core Runtime"
$dotnetPackageRelativePath = "shared\Microsoft.AspNetCore.App"
}
elseif ($Runtime -eq "windowsdesktop") {
$assetName = ".NET Core Windows Desktop Runtime"
$dotnetPackageRelativePath = "shared\Microsoft.WindowsDesktop.App"
}
elseif (-not $Runtime) {
$assetName = ".NET Core SDK"
$dotnetPackageRelativePath = "sdk"
}
else {
throw "Invalid value for `$Runtime"
}
if ($SpecificVersion -ne $EffectiveVersion)
{
Say "Performing installation checks for effective version: $EffectiveVersion"
$SpecificVersion = $EffectiveVersion
}
# Check if the SDK version is already installed.
$isAssetInstalled = Is-Dotnet-Package-Installed -InstallRoot $InstallRoot -RelativePathToPackage $dotnetPackageRelativePath -SpecificVersion $SpecificVersion
if ($isAssetInstalled) {
Say "$assetName version $SpecificVersion is already installed."
Prepend-Sdk-InstallRoot-To-Path -InstallRoot $InstallRoot -BinFolderRelativePath $BinFolderRelativePath
exit 0
}
New-Item -ItemType Directory -Force -Path $InstallRoot | Out-Null
$installDrive = $((Get-Item $InstallRoot).PSDrive.Name);
$diskInfo = Get-PSDrive -Name $installDrive
if ($diskInfo.Free / 1MB -le 100) {
Say "There is not enough disk space on drive ${installDrive}:"
exit 0
}
$ZipPath = [System.IO.Path]::combine([System.IO.Path]::GetTempPath(), [System.IO.Path]::GetRandomFileName())
Say-Verbose "Zip path: $ZipPath"
$DownloadFailed = $false
Say "Downloading link: $DownloadLink"
try {
DownloadFile -Source $DownloadLink -OutPath $ZipPath
}
catch {
Say "Cannot download: $DownloadLink"
if ($LegacyDownloadLink) {
$DownloadLink = $LegacyDownloadLink
$ZipPath = [System.IO.Path]::combine([System.IO.Path]::GetTempPath(), [System.IO.Path]::GetRandomFileName())
Say-Verbose "Legacy zip path: $ZipPath"
Say "Downloading legacy link: $DownloadLink"
try {
DownloadFile -Source $DownloadLink -OutPath $ZipPath
}
catch {
Say "Cannot download: $DownloadLink"
$DownloadFailed = $true
}
}
else {
$DownloadFailed = $true
}
}
if ($DownloadFailed) {
throw "Could not find/download: `"$assetName`" with version = $SpecificVersion`nRefer to: https://aka.ms/dotnet-os-lifecycle for information on .NET Core support"
}
Say "Extracting zip from $DownloadLink"
Extract-Dotnet-Package -ZipPath $ZipPath -OutPath $InstallRoot
# Check if the SDK version is installed; if not, fail the installation.
$isAssetInstalled = $false
# if the version contains "RTM" or "servicing"; check if a 'release-type' SDK version is installed.
if ($SpecificVersion -Match "rtm" -or $SpecificVersion -Match "servicing") {
$ReleaseVersion = $SpecificVersion.Split("-")[0]
Say-Verbose "Checking installation: version = $ReleaseVersion"
$isAssetInstalled = Is-Dotnet-Package-Installed -InstallRoot $InstallRoot -RelativePathToPackage $dotnetPackageRelativePath -SpecificVersion $ReleaseVersion
}
# Check if the SDK version is installed.
if (!$isAssetInstalled) {
Say-Verbose "Checking installation: version = $SpecificVersion"
$isAssetInstalled = Is-Dotnet-Package-Installed -InstallRoot $InstallRoot -RelativePathToPackage $dotnetPackageRelativePath -SpecificVersion $SpecificVersion
}
if (!$isAssetInstalled) {
throw "`"$assetName`" with version = $SpecificVersion failed to install with an unknown error."
}
Remove-Item $ZipPath
Prepend-Sdk-InstallRoot-To-Path -InstallRoot $InstallRoot -BinFolderRelativePath $BinFolderRelativePath
Say "Installation finished"
exit 0

1133
eng/common/dotnet-install-scripts/dotnet-install.sh

File diff suppressed because it is too large

2
eng/common/dotnet-install.sh

@ -18,7 +18,7 @@ architecture=''
runtime='dotnet'
runtimeSourceFeed=''
runtimeSourceFeedKey=''
while [[ $# > 0 ]]; do
while [[ $# -gt 0 ]]; do
opt="$(echo "$1" | tr "[:upper:]" "[:lower:]")"
case "$opt" in
-version|-v)

2
eng/common/dotnet.sh

@ -19,7 +19,7 @@ source $scriptroot/tools.sh
InitializeDotNetCli true # install
# Invoke acquired SDK with args if they are provided
if [[ $# > 0 ]]; then
if [[ $# -gt 0 ]]; then
__dotnetDir=${_InitializeDotNetCli}
dotnetPath=${__dotnetDir}/dotnet
${dotnetPath} "$@"

86
eng/common/generate-graph-files.ps1

@ -1,86 +0,0 @@
Param(
[Parameter(Mandatory=$true)][string] $barToken, # Token generated at https://maestro-prod.westus2.cloudapp.azure.com/Account/Tokens
[Parameter(Mandatory=$true)][string] $gitHubPat, # GitHub personal access token from https://github.com/settings/tokens (no auth scopes needed)
[Parameter(Mandatory=$true)][string] $azdoPat, # Azure Dev Ops tokens from https://dev.azure.com/dnceng/_details/security/tokens (code read scope needed)
[Parameter(Mandatory=$true)][string] $outputFolder, # Where the graphviz.txt file will be created
[string] $darcVersion, # darc's version
[string] $graphvizVersion = '2.38', # GraphViz version
[switch] $includeToolset # Whether the graph should include toolset dependencies or not. i.e. arcade, optimization. For more about
# toolset dependencies see https://github.com/dotnet/arcade/blob/master/Documentation/Darc.md#toolset-vs-product-dependencies
)
function CheckExitCode ([string]$stage)
{
$exitCode = $LASTEXITCODE
if ($exitCode -ne 0) {
Write-PipelineTelemetryError -Category 'Arcade' -Message "Something failed in stage: '$stage'. Check for errors above. Exiting now..."
ExitWithExitCode $exitCode
}
}
try {
$ErrorActionPreference = 'Stop'
. $PSScriptRoot\tools.ps1
Import-Module -Name (Join-Path $PSScriptRoot 'native\CommonLibrary.psm1')
Push-Location $PSScriptRoot
Write-Host 'Installing darc...'
. .\darc-init.ps1 -darcVersion $darcVersion
CheckExitCode 'Running darc-init'
$engCommonBaseDir = Join-Path $PSScriptRoot 'native\'
$graphvizInstallDir = CommonLibrary\Get-NativeInstallDirectory
$nativeToolBaseUri = 'https://netcorenativeassets.blob.core.windows.net/resource-packages/external'
$installBin = Join-Path $graphvizInstallDir 'bin'
Write-Host 'Installing dot...'
.\native\install-tool.ps1 -ToolName graphviz -InstallPath $installBin -BaseUri $nativeToolBaseUri -CommonLibraryDirectory $engCommonBaseDir -Version $graphvizVersion -Verbose
$darcExe = "$env:USERPROFILE\.dotnet\tools"
$darcExe = Resolve-Path "$darcExe\darc.exe"
Create-Directory $outputFolder
# Generate 3 graph descriptions:
# 1. Flat with coherency information
# 2. Graphviz (dot) file
# 3. Standard dependency graph
$graphVizFilePath = "$outputFolder\graphviz.txt"
$graphVizImageFilePath = "$outputFolder\graph.png"
$normalGraphFilePath = "$outputFolder\graph-full.txt"
$flatGraphFilePath = "$outputFolder\graph-flat.txt"
$baseOptions = @( '--github-pat', "$gitHubPat", '--azdev-pat', "$azdoPat", '--password', "$barToken" )
if ($includeToolset) {
Write-Host 'Toolsets will be included in the graph...'
$baseOptions += @( '--include-toolset' )
}
Write-Host 'Generating standard dependency graph...'
& "$darcExe" get-dependency-graph @baseOptions --output-file $normalGraphFilePath
CheckExitCode 'Generating normal dependency graph'
Write-Host 'Generating flat dependency graph and graphviz file...'
& "$darcExe" get-dependency-graph @baseOptions --flat --coherency --graphviz $graphVizFilePath --output-file $flatGraphFilePath
CheckExitCode 'Generating flat and graphviz dependency graph'
Write-Host "Generating graph image $graphVizFilePath"
$dotFilePath = Join-Path $installBin "graphviz\$graphvizVersion\release\bin\dot.exe"
& "$dotFilePath" -Tpng -o"$graphVizImageFilePath" "$graphVizFilePath"
CheckExitCode 'Generating graphviz image'
Write-Host "'$graphVizFilePath', '$flatGraphFilePath', '$normalGraphFilePath' and '$graphVizImageFilePath' created!"
}
catch {
if (!$includeToolset) {
Write-Host 'This might be a toolset repo which includes only toolset dependencies. ' -NoNewline -ForegroundColor Yellow
Write-Host 'Since -includeToolset is not set there is no graph to create. Include -includeToolset and try again...' -ForegroundColor Yellow
}
Write-Host $_.ScriptStackTrace
Write-PipelineTelemetryError -Category 'Arcade' -Message $_
ExitWithExitCode 1
} finally {
Pop-Location
}

2
eng/common/internal-feed-operations.ps1

@ -26,7 +26,7 @@ function SetupCredProvider {
$url = 'https://raw.githubusercontent.com/microsoft/artifacts-credprovider/master/helpers/installcredprovider.ps1'
Write-Host "Writing the contents of 'installcredprovider.ps1' locally..."
Invoke-WebRequest $url -OutFile installcredprovider.ps1
Invoke-WebRequest $url -UseBasicParsing -OutFile installcredprovider.ps1
Write-Host 'Installing plugin...'
.\installcredprovider.ps1 -Force

2
eng/common/internal-feed-operations.sh

@ -100,7 +100,7 @@ operation=''
authToken=''
repoName=''
while [[ $# > 0 ]]; do
while [[ $# -gt 0 ]]; do
opt="$(echo "$1" | tr "[:upper:]" "[:lower:]")"
case "$opt" in
--operation)

121
eng/common/native/find-native-compiler.sh

@ -1,121 +0,0 @@
#!/usr/bin/env bash
#
# This file locates the native compiler with the given name and version and sets the environment variables to locate it.
#
source="${BASH_SOURCE[0]}"
# resolve $SOURCE until the file is no longer a symlink
while [[ -h $source ]]; do
scriptroot="$( cd -P "$( dirname "$source" )" && pwd )"
source="$(readlink "$source")"
# if $source was a relative symlink, we need to resolve it relative to the path where the
# symlink file was located
[[ $source != /* ]] && source="$scriptroot/$source"
done
scriptroot="$( cd -P "$( dirname "$source" )" && pwd )"
if [ $# -lt 0 ]
then
echo "Usage..."
echo "find-native-compiler.sh <compiler> <compiler major version> <compiler minor version>"
echo "Specify the name of compiler (clang or gcc)."
echo "Specify the major version of compiler."
echo "Specify the minor version of compiler."
exit 1
fi
. $scriptroot/../pipeline-logging-functions.sh
compiler="$1"
cxxCompiler="$compiler++"
majorVersion="$2"
minorVersion="$3"
if [ "$compiler" = "gcc" ]; then cxxCompiler="g++"; fi
check_version_exists() {
desired_version=-1
# Set up the environment to be used for building with the desired compiler.
if command -v "$compiler-$1.$2" > /dev/null; then
desired_version="-$1.$2"
elif command -v "$compiler$1$2" > /dev/null; then
desired_version="$1$2"
elif command -v "$compiler-$1$2" > /dev/null; then
desired_version="-$1$2"
fi
echo "$desired_version"
}
if [ -z "$CLR_CC" ]; then
# Set default versions
if [ -z "$majorVersion" ]; then
# note: gcc (all versions) and clang versions higher than 6 do not have minor version in file name, if it is zero.
if [ "$compiler" = "clang" ]; then versions=( 9 8 7 6.0 5.0 4.0 3.9 3.8 3.7 3.6 3.5 )
elif [ "$compiler" = "gcc" ]; then versions=( 9 8 7 6 5 4.9 ); fi
for version in "${versions[@]}"; do
parts=(${version//./ })
desired_version="$(check_version_exists "${parts[0]}" "${parts[1]}")"
if [ "$desired_version" != "-1" ]; then majorVersion="${parts[0]}"; break; fi
done
if [ -z "$majorVersion" ]; then
if command -v "$compiler" > /dev/null; then
if [ "$(uname)" != "Darwin" ]; then
Write-PipelineTelemetryError -category "Build" -type "warning" "Specific version of $compiler not found, falling back to use the one in PATH."
fi
export CC="$(command -v "$compiler")"
export CXX="$(command -v "$cxxCompiler")"
else
Write-PipelineTelemetryError -category "Build" "No usable version of $compiler found."
exit 1
fi
else
if [ "$compiler" = "clang" ] && [ "$majorVersion" -lt 5 ]; then
if [ "$build_arch" = "arm" ] || [ "$build_arch" = "armel" ]; then
if command -v "$compiler" > /dev/null; then
Write-PipelineTelemetryError -category "Build" -type "warning" "Found clang version $majorVersion which is not supported on arm/armel architectures, falling back to use clang from PATH."
export CC="$(command -v "$compiler")"
export CXX="$(command -v "$cxxCompiler")"
else
Write-PipelineTelemetryError -category "Build" "Found clang version $majorVersion which is not supported on arm/armel architectures, and there is no clang in PATH."
exit 1
fi
fi
fi
fi
else
desired_version="$(check_version_exists "$majorVersion" "$minorVersion")"
if [ "$desired_version" = "-1" ]; then
Write-PipelineTelemetryError -category "Build" "Could not find specific version of $compiler: $majorVersion $minorVersion."
exit 1
fi
fi
if [ -z "$CC" ]; then
export CC="$(command -v "$compiler$desired_version")"
export CXX="$(command -v "$cxxCompiler$desired_version")"
if [ -z "$CXX" ]; then export CXX="$(command -v "$cxxCompiler")"; fi
fi
else
if [ ! -f "$CLR_CC" ]; then
Write-PipelineTelemetryError -category "Build" "CLR_CC is set but path '$CLR_CC' does not exist"
exit 1
fi
export CC="$CLR_CC"
export CXX="$CLR_CXX"
fi
if [ -z "$CC" ]; then
Write-PipelineTelemetryError -category "Build" "Unable to find $compiler."
exit 1
fi
export CCC_CC="$CC"
export CCC_CXX="$CXX"
export SCAN_BUILD_COMMAND="$(command -v "scan-build$desired_version")"

11
eng/common/native/install-dependencies.sh

@ -24,14 +24,16 @@ case "$os" in
apt update
apt install -y build-essential gettext locales cmake llvm clang lld lldb liblldb-dev libunwind8-dev libicu-dev liblttng-ust-dev \
libssl-dev libkrb5-dev pigz cpio
libssl-dev libkrb5-dev pigz cpio ninja-build
localedef -i en_US -c -f UTF-8 -A /usr/share/locale/locale.alias en_US.UTF-8
elif [ "$ID" = "fedora" ] || [ "$ID" = "rhel" ] || [ "$ID" = "azurelinux" ]; then
elif [ "$ID" = "fedora" ] || [ "$ID" = "rhel" ] || [ "$ID" = "azurelinux" ] || [ "$ID" = "centos" ]; then
pkg_mgr="$(command -v tdnf 2>/dev/null || command -v dnf)"
$pkg_mgr install -y cmake llvm lld lldb clang python curl libicu-devel openssl-devel krb5-devel lttng-ust-devel pigz cpio
$pkg_mgr install -y cmake llvm lld lldb clang python curl libicu-devel openssl-devel krb5-devel lttng-ust-devel pigz cpio ninja-build
elif [ "$ID" = "amzn" ]; then
dnf install -y cmake llvm lld lldb clang python libicu-devel openssl-devel krb5-devel lttng-ust-devel pigz cpio ninja-build
elif [ "$ID" = "alpine" ]; then
apk add build-base cmake bash curl clang llvm-dev lld lldb krb5-dev lttng-ust-dev icu-dev openssl-dev pigz cpio
apk add build-base cmake bash curl clang llvm-dev lld lldb krb5-dev lttng-ust-dev icu-dev openssl-dev pigz cpio ninja
else
echo "Unsupported distro. distro: $ID"
exit 1
@ -52,6 +54,7 @@ brew "openssl@3"
brew "pkgconf"
brew "python3"
brew "pigz"
brew "ninja"
EOF
;;

30
eng/common/performance/blazor_perf.proj

@ -1,30 +0,0 @@
<Project Sdk="Microsoft.DotNet.Helix.Sdk" DefaultTargets="Test">
<PropertyGroup Condition="'$(AGENT_OS)' != 'Windows_NT'">
<Python>python3</Python>
<HelixPreCommands>$(HelixPreCommands);chmod +x $HELIX_WORKITEM_PAYLOAD/SOD/SizeOnDisk</HelixPreCommands>
</PropertyGroup>
<ItemGroup>
<HelixCorrelationPayload Include="$(CorrelationPayloadDirectory)">
<PayloadDirectory>%(Identity)</PayloadDirectory>
</HelixCorrelationPayload>
</ItemGroup>
<PropertyGroup Condition="'$(AGENT_OS)' == 'Windows_NT'">
<ScenarioDirectory>%HELIX_CORRELATION_PAYLOAD%\performance\src\scenarios\</ScenarioDirectory>
<BlazorDirectory>$(ScenarioDirectory)blazor\</BlazorDirectory>
</PropertyGroup>
<PropertyGroup Condition="'$(AGENT_OS)' != 'Windows_NT'">
<ScenarioDirectory>$HELIX_CORRELATION_PAYLOAD/performance/src/scenarios/</ScenarioDirectory>
<BlazorDirectory>$(ScenarioDirectory)blazor/</BlazorDirectory>
</PropertyGroup>
<ItemGroup>
<HelixWorkItem Include="SOD - New Blazor Template - Publish">
<PayloadDirectory>$(WorkItemDirectory)</PayloadDirectory>
<PreCommands>cd $(BlazorDirectory);$(Python) pre.py publish --msbuild %27/p:_TrimmerDumpDependencies=true%27 --msbuild-static AdditionalMonoLinkerOptions=%27&quot;%24(AdditionalMonoLinkerOptions) --dump-dependencies&quot;%27 --binlog %27./traces/blazor_publish.binlog%27</PreCommands>
<Command>$(Python) test.py sod --scenario-name &quot;%(Identity)&quot;</Command>
<PostCommands>$(Python) post.py</PostCommands>
</HelixWorkItem>
</ItemGroup>
</Project>

69
eng/common/performance/crossgen_perf.proj

@ -1,69 +0,0 @@
<Project Sdk="Microsoft.DotNet.Helix.Sdk" DefaultTargets="Test">
<ItemGroup>
<HelixCorrelationPayload Include="$(CorrelationPayloadDirectory)">
<PayloadDirectory>%(Identity)</PayloadDirectory>
</HelixCorrelationPayload>
</ItemGroup>
<!--
Crossgen and Crossgen2 Scenario WorkItems
-->
<PropertyGroup Condition="'$(AGENT_OS)' == 'Windows_NT'">
<Python>py -3</Python>
<HelixPreCommands>$(HelixPreCommands)</HelixPreCommands>
<CoreRoot>%HELIX_CORRELATION_PAYLOAD%\Core_Root</CoreRoot>
<ScenarioDirectory>%HELIX_CORRELATION_PAYLOAD%\performance\src\scenarios\</ScenarioDirectory>
<CrossgenDirectory>$(ScenarioDirectory)crossgen\</CrossgenDirectory>
<Crossgen2Directory>$(ScenarioDirectory)crossgen2\</Crossgen2Directory>
</PropertyGroup>
<PropertyGroup Condition="'$(AGENT_OS)' != 'Windows_NT'">
<Python>python3</Python>
<HelixPreCommands>$(HelixPreCommands);chmod +x $HELIX_WORKITEM_PAYLOAD/startup/Startup;chmod +x $HELIX_WORKITEM_PAYLOAD/startup/perfcollect;sudo apt update</HelixPreCommands>
<CoreRoot>$HELIX_CORRELATION_PAYLOAD/Core_Root</CoreRoot>
<ScenarioDirectory>$HELIX_CORRELATION_PAYLOAD/performance/src/scenarios/</ScenarioDirectory>
<CrossgenDirectory>$(ScenarioDirectory)crossgen/</CrossgenDirectory>
<Crossgen2Directory>$(ScenarioDirectory)crossgen2/</Crossgen2Directory>
</PropertyGroup>
<ItemGroup>
<SingleAssembly Include="System.Private.Xml.dll"/>
<SingleAssembly Include="System.Linq.Expressions.dll"/>
<SingleAssembly Include="Microsoft.CodeAnalysis.VisualBasic.dll"/>
<SingleAssembly Include="Microsoft.CodeAnalysis.CSharp.dll"/>
<SingleAssembly Include="System.Private.CoreLib.dll"/>
</ItemGroup>
<ItemGroup>
<Composite Include="framework-r2r.dll.rsp"/>
</ItemGroup>
<ItemGroup>
<CrossgenWorkItem Include="@(SingleAssembly)">
<PayloadDirectory>$(WorkItemDirectory)</PayloadDirectory>
<Command>$(Python) $(CrossgenDirectory)test.py crossgen --core-root $(CoreRoot) --test-name %(Identity)</Command>
</CrossgenWorkItem>
</ItemGroup>
<ItemGroup>
<Crossgen2WorkItem Include="@(SingleAssembly)">
<PayloadDirectory>$(WorkItemDirectory)</PayloadDirectory>
<Command>$(Python) $(Crossgen2Directory)test.py crossgen2 --core-root $(CoreRoot) --single %(Identity)</Command>
</Crossgen2WorkItem>
</ItemGroup>
<ItemGroup>
<!-- Enable crossgen tests on Windows x64 and Windows x86 -->
<HelixWorkItem Include="@(CrossgenWorkItem -> 'Crossgen %(Identity)')" Condition="'$(AGENT_OS)' == 'Windows_NT'">
<Timeout>4:00</Timeout>
</HelixWorkItem>
<!-- Enable crossgen2 tests on Windows x64 and Linux x64 -->
<HelixWorkItem Include="@(Crossgen2WorkItem -> 'Crossgen2 %(Identity)')" Condition="'$(Architecture)' == 'x64'">
<Timeout>4:00</Timeout>
</HelixWorkItem>
<HelixWorkItem Include="Crossgen2 Composite Framework R2R" Condition="'$(Architecture)' == 'x64'">
<PayloadDirectory>$(WorkItemDirectory)</PayloadDirectory>
<Command>$(Python) $(Crossgen2Directory)test.py crossgen2 --core-root $(CoreRoot) --composite $(Crossgen2Directory)framework-r2r.dll.rsp</Command>
<Timeout>1:00</Timeout>
</HelixWorkItem>
</ItemGroup>
</Project>

144
eng/common/performance/microbenchmarks.proj

@ -1,144 +0,0 @@
<Project Sdk="Microsoft.DotNet.Helix.Sdk" DefaultTargets="Test">
<PropertyGroup Condition="'$(AGENT_OS)' == 'Windows_NT'">
<WorkItemCommand>%HELIX_CORRELATION_PAYLOAD%\performance\scripts\benchmarks_ci.py --csproj %HELIX_CORRELATION_PAYLOAD%\performance\$(TargetCsproj)</WorkItemCommand>
<CliArguments>--dotnet-versions %DOTNET_VERSION% --cli-source-info args --cli-branch %PERFLAB_BRANCH% --cli-commit-sha %PERFLAB_HASH% --cli-repository https://github.com/%PERFLAB_REPO% --cli-source-timestamp %PERFLAB_BUILDTIMESTAMP%</CliArguments>
<Python>py -3</Python>
<CoreRun>%HELIX_CORRELATION_PAYLOAD%\Core_Root\CoreRun.exe</CoreRun>
<BaselineCoreRun>%HELIX_CORRELATION_PAYLOAD%\Baseline_Core_Root\CoreRun.exe</BaselineCoreRun>
<HelixPreCommands>$(HelixPreCommands);call %HELIX_CORRELATION_PAYLOAD%\performance\tools\machine-setup.cmd;set PYTHONPATH=%HELIX_WORKITEM_PAYLOAD%\scripts%3B%HELIX_WORKITEM_PAYLOAD%</HelixPreCommands>
<ArtifactsDirectory>%HELIX_CORRELATION_PAYLOAD%\artifacts\BenchmarkDotNet.Artifacts</ArtifactsDirectory>
<BaselineArtifactsDirectory>%HELIX_CORRELATION_PAYLOAD%\artifacts\BenchmarkDotNet.Artifacts_Baseline</BaselineArtifactsDirectory>
<ResultsComparer>%HELIX_CORRELATION_PAYLOAD%\performance\src\tools\ResultsComparer\ResultsComparer.csproj</ResultsComparer>
<DotnetExe>%HELIX_CORRELATION_PAYLOAD%\performance\tools\dotnet\$(Architecture)\dotnet.exe</DotnetExe>
<Percent>%25%25</Percent>
<XMLResults>%HELIX_WORKITEM_ROOT%\testResults.xml</XMLResults>
</PropertyGroup>
<PropertyGroup Condition="'$(AGENT_OS)' != 'Windows_NT' and '$(RunFromPerfRepo)' == 'false'">
<BaseDirectory>$HELIX_CORRELATION_PAYLOAD</BaseDirectory>
<PerformanceDirectory>$(BaseDirectory)/performance</PerformanceDirectory>
</PropertyGroup>
<PropertyGroup Condition="'$(AGENT_OS)' != 'Windows_NT' and '$(RunFromPerfRepo)' == 'true'">
<BaseDirectory>$HELIX_WORKITEM_PAYLOAD</BaseDirectory>
<PerformanceDirectory>$(BaseDirectory)</PerformanceDirectory>
</PropertyGroup>
<PropertyGroup Condition="'$(AGENT_OS)' != 'Windows_NT'">
<WorkItemCommand>$(PerformanceDirectory)/scripts/benchmarks_ci.py --csproj $(PerformanceDirectory)/$(TargetCsproj)</WorkItemCommand>
<CliArguments>--dotnet-versions $DOTNET_VERSION --cli-source-info args --cli-branch $PERFLAB_BRANCH --cli-commit-sha $PERFLAB_HASH --cli-repository https://github.com/$PERFLAB_REPO --cli-source-timestamp $PERFLAB_BUILDTIMESTAMP</CliArguments>
<Python>python3</Python>
<CoreRun>$(BaseDirectory)/Core_Root/corerun</CoreRun>
<BaselineCoreRun>$(BaseDirectory)/Baseline_Core_Root/corerun</BaselineCoreRun>
<HelixPreCommands>$(HelixPreCommands);chmod +x $(PerformanceDirectory)/tools/machine-setup.sh;. $(PerformanceDirectory)/tools/machine-setup.sh</HelixPreCommands>
<ArtifactsDirectory>$(BaseDirectory)/artifacts/BenchmarkDotNet.Artifacts</ArtifactsDirectory>
<BaselineArtifactsDirectory>$(BaseDirectory)/artifacts/BenchmarkDotNet.Artifacts_Baseline</BaselineArtifactsDirectory>
<ResultsComparer>$(PerformanceDirectory)/src/tools/ResultsComparer/ResultsComparer.csproj</ResultsComparer>
<DotnetExe>$(PerformanceDirectory)/tools/dotnet/$(Architecture)/dotnet</DotnetExe>
<Percent>%25</Percent>
<XMLResults>$HELIX_WORKITEM_ROOT/testResults.xml</XMLResults>
</PropertyGroup>
<PropertyGroup Condition="'$(WasmDotnet)' == 'true'">
<CliArguments>$(CliArguments) --wasm</CliArguments>
</PropertyGroup>
<PropertyGroup Condition="'$(MonoDotnet)' == 'true' and '$(AGENT_OS)' == 'Windows_NT'">
<CoreRunArgument>--corerun %HELIX_CORRELATION_PAYLOAD%\dotnet-mono\shared\Microsoft.NETCore.App\6.0.0\corerun.exe</CoreRunArgument>
</PropertyGroup>
<PropertyGroup Condition="'$(MonoDotnet)' == 'true' and '$(AGENT_OS)' != 'Windows_NT'">
<CoreRunArgument>--corerun $(BaseDirectory)/dotnet-mono/shared/Microsoft.NETCore.App/6.0.0/corerun</CoreRunArgument>
</PropertyGroup>
<PropertyGroup Condition="'$(UseCoreRun)' == 'true'">
<CoreRunArgument>--corerun $(CoreRun)</CoreRunArgument>
</PropertyGroup>
<PropertyGroup Condition="'$(UseBaselineCoreRun)' == 'true'">
<BaselineCoreRunArgument>--corerun $(BaselineCoreRun)</BaselineCoreRunArgument>
</PropertyGroup>
<PropertyGroup Condition="'$(WorkItemCommand)' != ''">
<WorkItemCommand>$(Python) $(WorkItemCommand) --incremental no --architecture $(Architecture) -f $(_Framework) $(PerfLabArguments)</WorkItemCommand>
</PropertyGroup>
<PropertyGroup Condition="'$(_Framework)' != 'net461'">
<WorkItemCommand>$(WorkItemCommand) $(CliArguments)</WorkItemCommand>
</PropertyGroup>
<PropertyGroup>
<WorkItemTimeout>2:30</WorkItemTimeout>
<WorkItemTimeout Condition="'$(HelixSourcePrefix)' != 'official'">0:15</WorkItemTimeout>
</PropertyGroup>
<ItemGroup>
<HelixCorrelationPayload Include="$(CorrelationPayloadDirectory)">
<PayloadDirectory>%(Identity)</PayloadDirectory>
</HelixCorrelationPayload>
</ItemGroup>
<PropertyGroup>
<PartitionCount>30</PartitionCount>
</PropertyGroup>
<ItemGroup>
<Partition Include="$(BuildConfig).Partition0" Index="0" />
<Partition Include="$(BuildConfig).Partition1" Index="1" />
<Partition Include="$(BuildConfig).Partition2" Index="2" />
<Partition Include="$(BuildConfig).Partition3" Index="3" />
<Partition Include="$(BuildConfig).Partition4" Index="4" />
<Partition Include="$(BuildConfig).Partition5" Index="5" />
<Partition Include="$(BuildConfig).Partition6" Index="6" />
<Partition Include="$(BuildConfig).Partition7" Index="7" />
<Partition Include="$(BuildConfig).Partition8" Index="8" />
<Partition Include="$(BuildConfig).Partition9" Index="9" />
<Partition Include="$(BuildConfig).Partition10" Index="10" />
<Partition Include="$(BuildConfig).Partition11" Index="11" />
<Partition Include="$(BuildConfig).Partition12" Index="12" />
<Partition Include="$(BuildConfig).Partition13" Index="13" />
<Partition Include="$(BuildConfig).Partition14" Index="14" />
<Partition Include="$(BuildConfig).Partition15" Index="15" />
<Partition Include="$(BuildConfig).Partition16" Index="16" />
<Partition Include="$(BuildConfig).Partition17" Index="17" />
<Partition Include="$(BuildConfig).Partition18" Index="18" />
<Partition Include="$(BuildConfig).Partition19" Index="19" />
<Partition Include="$(BuildConfig).Partition20" Index="20" />
<Partition Include="$(BuildConfig).Partition21" Index="21" />
<Partition Include="$(BuildConfig).Partition22" Index="22" />
<Partition Include="$(BuildConfig).Partition23" Index="23" />
<Partition Include="$(BuildConfig).Partition24" Index="24" />
<Partition Include="$(BuildConfig).Partition25" Index="25" />
<Partition Include="$(BuildConfig).Partition26" Index="26" />
<Partition Include="$(BuildConfig).Partition27" Index="27" />
<Partition Include="$(BuildConfig).Partition28" Index="28" />
<Partition Include="$(BuildConfig).Partition29" Index="29" />
</ItemGroup>
<PropertyGroup Condition="'$(Compare)' == 'true'">
<FailOnTestFailure>false</FailOnTestFailure>
</PropertyGroup>
<!--
Partition the Microbenchmarks project, but nothing else
-->
<ItemGroup Condition="$(TargetCsproj.Contains('MicroBenchmarks.csproj'))">
<HelixWorkItem Include="@(Partition)">
<PayloadDirectory>$(WorkItemDirectory)</PayloadDirectory>
<PreCommands Condition="'$(Compare)' == 'true'">$(WorkItemCommand) --bdn-artifacts $(BaselineArtifactsDirectory) --bdn-arguments="--anyCategories $(BDNCategories) $(ExtraBenchmarkDotNetArguments) $(BaselineCoreRunArgument) --partition-count $(PartitionCount) --partition-index %(HelixWorkItem.Index)"</PreCommands>
<Command>$(WorkItemCommand) --bdn-artifacts $(ArtifactsDirectory) --bdn-arguments="--anyCategories $(BDNCategories) $(ExtraBenchmarkDotNetArguments) $(CoreRunArgument) --partition-count $(PartitionCount) --partition-index %(HelixWorkItem.Index)"</Command>
<PostCommands Condition="'$(Compare)' == 'true'">$(DotnetExe) run -f $(_Framework) -p $(ResultsComparer) --base $(BaselineArtifactsDirectory) --diff $(ArtifactsDirectory) --threshold 2$(Percent) --xml $(XMLResults);$(FinalCommand)</PostCommands>
<Timeout>$(WorkItemTimeout)</Timeout>
</HelixWorkItem>
</ItemGroup>
<ItemGroup Condition="!$(TargetCsproj.Contains('MicroBenchmarks.csproj'))">
<HelixWorkItem Include="$(BuildConfig).WorkItem">
<PayloadDirectory>$(WorkItemDirectory)</PayloadDirectory>
<PreCommands Condition="'$(Compare)' == 'true'">$(WorkItemCommand) --bdn-artifacts $(BaselineArtifactsDirectory) --bdn-arguments="--anyCategories $(BDNCategories) $(ExtraBenchmarkDotNetArguments) $(BaselineCoreRunArgument)"</PreCommands>
<Command>$(WorkItemCommand) --bdn-artifacts $(ArtifactsDirectory) --bdn-arguments="--anyCategories $(BDNCategories) $(ExtraBenchmarkDotNetArguments) $(CoreRunArgument)"</Command>
<PostCommands Condition="'$(Compare)' == 'true'">$(DotnetExe) run -f $(_Framework) -p $(ResultsComparer) --base $(BaselineArtifactsDirectory) --diff $(ArtifactsDirectory) --threshold 2$(Percent) --xml $(XMLResults)</PostCommands>
<Timeout>4:00</Timeout>
</HelixWorkItem>
</ItemGroup>
</Project>

121
eng/common/performance/perfhelixpublish.proj

@ -1,121 +0,0 @@
<Project Sdk="Microsoft.DotNet.Helix.Sdk" DefaultTargets="Test">
<PropertyGroup Condition="'$(AGENT_OS)' == 'Windows_NT'">
<WorkItemCommand>%HELIX_CORRELATION_PAYLOAD%\performance\scripts\benchmarks_ci.py --csproj %HELIX_CORRELATION_PAYLOAD%\performance\$(TargetCsproj)</WorkItemCommand>
<CliArguments>--dotnet-versions %DOTNET_VERSION% --cli-source-info args --cli-branch %PERFLAB_BRANCH% --cli-commit-sha %PERFLAB_HASH% --cli-repository https://github.com/%PERFLAB_REPO% --cli-source-timestamp %PERFLAB_BUILDTIMESTAMP%</CliArguments>
<Python>py -3</Python>
<CoreRun>%HELIX_CORRELATION_PAYLOAD%\Core_Root\CoreRun.exe</CoreRun>
<BaselineCoreRun>%HELIX_CORRELATION_PAYLOAD%\Baseline_Core_Root\CoreRun.exe</BaselineCoreRun>
<HelixPreCommands>$(HelixPreCommands);call %HELIX_CORRELATION_PAYLOAD%\performance\tools\machine-setup.cmd;set PYTHONPATH=%HELIX_WORKITEM_PAYLOAD%\scripts%3B%HELIX_WORKITEM_PAYLOAD%</HelixPreCommands>
<ArtifactsDirectory>%HELIX_CORRELATION_PAYLOAD%\artifacts\BenchmarkDotNet.Artifacts</ArtifactsDirectory>
<BaselineArtifactsDirectory>%HELIX_CORRELATION_PAYLOAD%\artifacts\BenchmarkDotNet.Artifacts_Baseline</BaselineArtifactsDirectory>
<ResultsComparer>%HELIX_CORRELATION_PAYLOAD%\performance\src\tools\ResultsComparer\ResultsComparer.csproj</ResultsComparer>
<DotnetExe>%HELIX_CORRELATION_PAYLOAD%\performance\tools\dotnet\$(Architecture)\dotnet.exe</DotnetExe>
<Percent>%25%25</Percent>
<XMLResults>%HELIX_WORKITEM_ROOT%\testResults.xml</XMLResults>
</PropertyGroup>
<PropertyGroup Condition="'$(AGENT_OS)' != 'Windows_NT' and '$(RunFromPerfRepo)' == 'false'">
<BaseDirectory>$HELIX_CORRELATION_PAYLOAD</BaseDirectory>
<PerformanceDirectory>$(BaseDirectory)/performance</PerformanceDirectory>
</PropertyGroup>
<PropertyGroup Condition="'$(AGENT_OS)' != 'Windows_NT' and '$(RunFromPerfRepo)' == 'true'">
<BaseDirectory>$HELIX_WORKITEM_PAYLOAD</BaseDirectory>
<PerformanceDirectory>$(BaseDirectory)</PerformanceDirectory>
</PropertyGroup>
<PropertyGroup Condition="'$(AGENT_OS)' != 'Windows_NT'">
<WorkItemCommand>$(PerformanceDirectory)/scripts/benchmarks_ci.py --csproj $(PerformanceDirectory)/$(TargetCsproj)</WorkItemCommand>
<CliArguments>--dotnet-versions $DOTNET_VERSION --cli-source-info args --cli-branch $PERFLAB_BRANCH --cli-commit-sha $PERFLAB_HASH --cli-repository https://github.com/$PERFLAB_REPO --cli-source-timestamp $PERFLAB_BUILDTIMESTAMP</CliArguments>
<Python>python3</Python>
<CoreRun>$(BaseDirectory)/Core_Root/corerun</CoreRun>
<BaselineCoreRun>$(BaseDirectory)/Baseline_Core_Root/corerun</BaselineCoreRun>
<HelixPreCommands>$(HelixPreCommands);chmod +x $(PerformanceDirectory)/tools/machine-setup.sh;. $(PerformanceDirectory)/tools/machine-setup.sh</HelixPreCommands>
<ArtifactsDirectory>$(BaseDirectory)/artifacts/BenchmarkDotNet.Artifacts</ArtifactsDirectory>
<BaselineArtifactsDirectory>$(BaseDirectory)/artifacts/BenchmarkDotNet.Artifacts_Baseline</BaselineArtifactsDirectory>
<ResultsComparer>$(PerformanceDirectory)/src/tools/ResultsComparer/ResultsComparer.csproj</ResultsComparer>
<DotnetExe>$(PerformanceDirectory)/tools/dotnet/$(Architecture)/dotnet</DotnetExe>
<Percent>%25</Percent>
<XMLResults>$HELIX_WORKITEM_ROOT/testResults.xml</XMLResults>
</PropertyGroup>
<PropertyGroup Condition="'$(UseCoreRun)' == 'true'">
<CoreRunArgument>--corerun $(CoreRun)</CoreRunArgument>
</PropertyGroup>
<PropertyGroup Condition="'$(UseBaselineCoreRun)' == 'true'">
<BaselineCoreRunArgument>--corerun $(BaselineCoreRun)</BaselineCoreRunArgument>
</PropertyGroup>
<PropertyGroup Condition="'$(WorkItemCommand)' != ''">
<WorkItemCommand>$(Python) $(WorkItemCommand) --incremental no --architecture $(Architecture) -f $(_Framework) $(PerfLabArguments)</WorkItemCommand>
</PropertyGroup>
<PropertyGroup Condition="'$(_Framework)' != 'net461'">
<WorkItemCommand>$(WorkItemCommand) $(CliArguments)</WorkItemCommand>
</PropertyGroup>
<ItemGroup>
<HelixCorrelationPayload Include="$(CorrelationPayloadDirectory)">
<PayloadDirectory>%(Identity)</PayloadDirectory>
</HelixCorrelationPayload>
</ItemGroup>
<PropertyGroup>
<PartitionCount>5</PartitionCount>
</PropertyGroup>
<ItemGroup>
<Partition Include="$(BuildConfig).Partition0" Index="0" />
<Partition Include="$(BuildConfig).Partition1" Index="1" />
<Partition Include="$(BuildConfig).Partition2" Index="2" />
<Partition Include="$(BuildConfig).Partition3" Index="3" />
<Partition Include="$(BuildConfig).Partition4" Index="4" />
</ItemGroup>
<PropertyGroup Condition="'$(Compare)' == 'true'">
<FailOnTestFailure>false</FailOnTestFailure>
</PropertyGroup>
<!--
Partition the Microbenchmarks project, but nothing else
-->
<ItemGroup Condition="$(TargetCsproj.Contains('MicroBenchmarks.csproj'))">
<HelixWorkItem Include="@(Partition)">
<PayloadDirectory>$(WorkItemDirectory)</PayloadDirectory>
<PreCommands Condition="'$(Compare)' == 'true'">$(WorkItemCommand) --bdn-artifacts $(BaselineArtifactsDirectory) --bdn-arguments="--anyCategories $(BDNCategories) $(ExtraBenchmarkDotNetArguments) $(BaselineCoreRunArgument) --partition-count $(PartitionCount) --partition-index %(HelixWorkItem.Index)"</PreCommands>
<Command>$(WorkItemCommand) --bdn-artifacts $(ArtifactsDirectory) --bdn-arguments="--anyCategories $(BDNCategories) $(ExtraBenchmarkDotNetArguments) $(CoreRunArgument) --partition-count $(PartitionCount) --partition-index %(HelixWorkItem.Index)"</Command>
<PostCommands Condition="'$(Compare)' == 'true'">$(DotnetExe) run -f $(_Framework) -p $(ResultsComparer) --base $(BaselineArtifactsDirectory) --diff $(ArtifactsDirectory) --threshold 2$(Percent) --xml $(XMLResults);$(FinalCommand)</PostCommands>
<Timeout>4:00</Timeout>
</HelixWorkItem>
</ItemGroup>
<ItemGroup Condition="!$(TargetCsproj.Contains('MicroBenchmarks.csproj'))">
<HelixWorkItem Include="$(BuildConfig).WorkItem">
<PayloadDirectory>$(WorkItemDirectory)</PayloadDirectory>
<PreCommands Condition="'$(Compare)' == 'true'">$(WorkItemCommand) --bdn-artifacts $(BaselineArtifactsDirectory) --bdn-arguments="--anyCategories $(BDNCategories) $(ExtraBenchmarkDotNetArguments) $(BaselineCoreRunArgument)"</PreCommands>
<Command>$(WorkItemCommand) --bdn-artifacts $(ArtifactsDirectory) --bdn-arguments="--anyCategories $(BDNCategories) $(ExtraBenchmarkDotNetArguments) $(CoreRunArgument)"</Command>
<PostCommands Condition="'$(Compare)' == 'true'">$(DotnetExe) run -f $(_Framework) -p $(ResultsComparer) --base $(BaselineArtifactsDirectory) --diff $(ArtifactsDirectory) --threshold 2$(Percent) --xml $(XMLResults)</PostCommands>
<Timeout>4:00</Timeout>
</HelixWorkItem>
</ItemGroup>
<ItemGroup Condition="'$(AGENT_OS)' == 'Windows_NT'">
<HelixWorkItem Include="Crossgen System.Private.Xml.dll">
<PayloadDirectory>$(WorkItemDirectory)\ScenarioCorrelation</PayloadDirectory>
<Command>$(Python) %HELIX_CORRELATION_PAYLOAD%\performance\src\scenarios\crossgen\test.py crossgen --test-name System.Private.Xml.dll --core-root %HELIX_CORRELATION_PAYLOAD%\Core_Root</Command>
</HelixWorkItem>
<HelixWorkItem Include="Crossgen System.Linq.Expressions.dll">
<PayloadDirectory>$(WorkItemDirectory)\ScenarioCorrelation</PayloadDirectory>
<Command>$(Python) %HELIX_CORRELATION_PAYLOAD%\performance\src\scenarios\crossgen\test.py crossgen --test-name System.Linq.Expressions.dll --core-root %HELIX_CORRELATION_PAYLOAD%\Core_Root</Command>
</HelixWorkItem>
<HelixWorkItem Include="Crossgen Microsoft.CodeAnalysis.VisualBasic.dll">
<PayloadDirectory>$(WorkItemDirectory)\ScenarioCorrelation</PayloadDirectory>
<Command>$(Python) %HELIX_CORRELATION_PAYLOAD%\performance\src\scenarios\crossgen\test.py crossgen --test-name Microsoft.CodeAnalysis.VisualBasic.dll --core-root %HELIX_CORRELATION_PAYLOAD%\Core_Root</Command>
</HelixWorkItem>
<HelixWorkItem Include="Crossgen Microsoft.CodeAnalysis.CSharp.dll">
<PayloadDirectory>$(WorkItemDirectory)\ScenarioCorrelation</PayloadDirectory>
<Command>$(Python) %HELIX_CORRELATION_PAYLOAD%\performance\src\scenarios\crossgen\test.py crossgen --test-name Microsoft.CodeAnalysis.CSharp.dll --core-root %HELIX_CORRELATION_PAYLOAD%\Core_Root</Command>
</HelixWorkItem>
</ItemGroup>
</Project>

147
eng/common/performance/performance-setup.ps1

@ -1,147 +0,0 @@
Param(
[string] $SourceDirectory=$env:BUILD_SOURCESDIRECTORY,
[string] $CoreRootDirectory,
[string] $BaselineCoreRootDirectory,
[string] $Architecture="x64",
[string] $Framework="net5.0",
[string] $CompilationMode="Tiered",
[string] $Repository=$env:BUILD_REPOSITORY_NAME,
[string] $Branch=$env:BUILD_SOURCEBRANCH,
[string] $CommitSha=$env:BUILD_SOURCEVERSION,
[string] $BuildNumber=$env:BUILD_BUILDNUMBER,
[string] $RunCategories="Libraries Runtime",
[string] $Csproj="src\benchmarks\micro\MicroBenchmarks.csproj",
[string] $Kind="micro",
[switch] $LLVM,
[switch] $MonoInterpreter,
[switch] $MonoAOT,
[switch] $Internal,
[switch] $Compare,
[string] $MonoDotnet="",
[string] $Configurations="CompilationMode=$CompilationMode RunKind=$Kind"
)
$RunFromPerformanceRepo = ($Repository -eq "dotnet/performance") -or ($Repository -eq "dotnet-performance")
$UseCoreRun = ($CoreRootDirectory -ne [string]::Empty)
$UseBaselineCoreRun = ($BaselineCoreRootDirectory -ne [string]::Empty)
$PayloadDirectory = (Join-Path $SourceDirectory "Payload")
$PerformanceDirectory = (Join-Path $PayloadDirectory "performance")
$WorkItemDirectory = (Join-Path $SourceDirectory "workitem")
$ExtraBenchmarkDotNetArguments = "--iterationCount 1 --warmupCount 0 --invocationCount 1 --unrollFactor 1 --strategy ColdStart --stopOnFirstError true"
$Creator = $env:BUILD_DEFINITIONNAME
$PerfLabArguments = ""
$HelixSourcePrefix = "pr"
$Queue = "Windows.10.Amd64.ClientRS4.DevEx.15.8.Open"
# TODO: Implement a better logic to determine if Framework is .NET Core or >= .NET 5.
if ($Framework.StartsWith("netcoreapp") -or ($Framework -eq "net5.0")) {
$Queue = "Windows.10.Amd64.ClientRS5.Open"
}
if ($Compare) {
$Queue = "Windows.10.Amd64.19H1.Tiger.Perf.Open"
$PerfLabArguments = ""
$ExtraBenchmarkDotNetArguments = ""
}
if ($Internal) {
$Queue = "Windows.10.Amd64.19H1.Tiger.Perf"
$PerfLabArguments = "--upload-to-perflab-container"
$ExtraBenchmarkDotNetArguments = ""
$Creator = ""
$HelixSourcePrefix = "official"
}
if($MonoInterpreter)
{
$ExtraBenchmarkDotNetArguments = "--category-exclusion-filter NoInterpreter"
}
if($MonoDotnet -ne "")
{
$Configurations += " LLVM=$LLVM MonoInterpreter=$MonoInterpreter MonoAOT=$MonoAOT"
if($ExtraBenchmarkDotNetArguments -eq "")
{
#FIX ME: We need to block these tests as they don't run on mono for now
$ExtraBenchmarkDotNetArguments = "--exclusion-filter *Perf_Image* *Perf_NamedPipeStream*"
}
else
{
#FIX ME: We need to block these tests as they don't run on mono for now
$ExtraBenchmarkDotNetArguments += " --exclusion-filter *Perf_Image* *Perf_NamedPipeStream*"
}
}
# FIX ME: This is a workaround until we get this from the actual pipeline
$CommonSetupArguments="--channel master --queue $Queue --build-number $BuildNumber --build-configs $Configurations --architecture $Architecture"
$SetupArguments = "--repository https://github.com/$Repository --branch $Branch --get-perf-hash --commit-sha $CommitSha $CommonSetupArguments"
#This grabs the LKG version number of dotnet and passes it to our scripts
$VersionJSON = Get-Content global.json | ConvertFrom-Json
$DotNetVersion = $VersionJSON.tools.dotnet
$SetupArguments = "--dotnet-versions $DotNetVersion $SetupArguments"
if ($RunFromPerformanceRepo) {
$SetupArguments = "--perf-hash $CommitSha $CommonSetupArguments"
robocopy $SourceDirectory $PerformanceDirectory /E /XD $PayloadDirectory $SourceDirectory\artifacts $SourceDirectory\.git
}
else {
git clone --branch master --depth 1 --quiet https://github.com/dotnet/performance $PerformanceDirectory
}
if($MonoDotnet -ne "")
{
$UsingMono = "true"
$MonoDotnetPath = (Join-Path $PayloadDirectory "dotnet-mono")
Move-Item -Path $MonoDotnet -Destination $MonoDotnetPath
}
if ($UseCoreRun) {
$NewCoreRoot = (Join-Path $PayloadDirectory "Core_Root")
Move-Item -Path $CoreRootDirectory -Destination $NewCoreRoot
}
if ($UseBaselineCoreRun) {
$NewBaselineCoreRoot = (Join-Path $PayloadDirectory "Baseline_Core_Root")
Move-Item -Path $BaselineCoreRootDirectory -Destination $NewBaselineCoreRoot
}
$DocsDir = (Join-Path $PerformanceDirectory "docs")
robocopy $DocsDir $WorkItemDirectory
# Set variables that we will need to have in future steps
$ci = $true
. "$PSScriptRoot\..\pipeline-logging-functions.ps1"
# Directories
Write-PipelineSetVariable -Name 'PayloadDirectory' -Value "$PayloadDirectory" -IsMultiJobVariable $false
Write-PipelineSetVariable -Name 'PerformanceDirectory' -Value "$PerformanceDirectory" -IsMultiJobVariable $false
Write-PipelineSetVariable -Name 'WorkItemDirectory' -Value "$WorkItemDirectory" -IsMultiJobVariable $false
# Script Arguments
Write-PipelineSetVariable -Name 'Python' -Value "py -3" -IsMultiJobVariable $false
Write-PipelineSetVariable -Name 'ExtraBenchmarkDotNetArguments' -Value "$ExtraBenchmarkDotNetArguments" -IsMultiJobVariable $false
Write-PipelineSetVariable -Name 'SetupArguments' -Value "$SetupArguments" -IsMultiJobVariable $false
Write-PipelineSetVariable -Name 'PerfLabArguments' -Value "$PerfLabArguments" -IsMultiJobVariable $false
Write-PipelineSetVariable -Name 'BDNCategories' -Value "$RunCategories" -IsMultiJobVariable $false
Write-PipelineSetVariable -Name 'TargetCsproj' -Value "$Csproj" -IsMultiJobVariable $false
Write-PipelineSetVariable -Name 'Kind' -Value "$Kind" -IsMultiJobVariable $false
Write-PipelineSetVariable -Name 'Architecture' -Value "$Architecture" -IsMultiJobVariable $false
Write-PipelineSetVariable -Name 'UseCoreRun' -Value "$UseCoreRun" -IsMultiJobVariable $false
Write-PipelineSetVariable -Name 'UseBaselineCoreRun' -Value "$UseBaselineCoreRun" -IsMultiJobVariable $false
Write-PipelineSetVariable -Name 'RunFromPerfRepo' -Value "$RunFromPerformanceRepo" -IsMultiJobVariable $false
Write-PipelineSetVariable -Name 'Compare' -Value "$Compare" -IsMultiJobVariable $false
Write-PipelineSetVariable -Name 'MonoDotnet' -Value "$UsingMono" -IsMultiJobVariable $false
# Helix Arguments
Write-PipelineSetVariable -Name 'Creator' -Value "$Creator" -IsMultiJobVariable $false
Write-PipelineSetVariable -Name 'Queue' -Value "$Queue" -IsMultiJobVariable $false
Write-PipelineSetVariable -Name 'HelixSourcePrefix' -Value "$HelixSourcePrefix" -IsMultiJobVariable $false
Write-PipelineSetVariable -Name '_BuildConfig' -Value "$Architecture.$Kind.$Framework" -IsMultiJobVariable $false
exit 0

289
eng/common/performance/performance-setup.sh

@ -1,289 +0,0 @@
#!/usr/bin/env bash
source_directory=$BUILD_SOURCESDIRECTORY
core_root_directory=
baseline_core_root_directory=
architecture=x64
framework=net5.0
compilation_mode=tiered
repository=$BUILD_REPOSITORY_NAME
branch=$BUILD_SOURCEBRANCH
commit_sha=$BUILD_SOURCEVERSION
build_number=$BUILD_BUILDNUMBER
internal=false
compare=false
mono_dotnet=
kind="micro"
llvm=false
monointerpreter=false
monoaot=false
run_categories="Libraries Runtime"
csproj="src\benchmarks\micro\MicroBenchmarks.csproj"
configurations="CompliationMode=$compilation_mode RunKind=$kind"
run_from_perf_repo=false
use_core_run=true
use_baseline_core_run=true
using_mono=false
wasm_runtime_loc=
using_wasm=false
use_latest_dotnet=false
while (($# > 0)); do
lowerI="$(echo $1 | awk '{print tolower($0)}')"
case $lowerI in
--sourcedirectory)
source_directory=$2
shift 2
;;
--corerootdirectory)
core_root_directory=$2
shift 2
;;
--baselinecorerootdirectory)
baseline_core_root_directory=$2
shift 2
;;
--architecture)
architecture=$2
shift 2
;;
--framework)
framework=$2
shift 2
;;
--compilationmode)
compilation_mode=$2
shift 2
;;
--repository)
repository=$2
shift 2
;;
--branch)
branch=$2
shift 2
;;
--commitsha)
commit_sha=$2
shift 2
;;
--buildnumber)
build_number=$2
shift 2
;;
--kind)
kind=$2
configurations="CompilationMode=$compilation_mode RunKind=$kind"
shift 2
;;
--runcategories)
run_categories=$2
shift 2
;;
--csproj)
csproj=$2
shift 2
;;
--internal)
internal=true
shift 1
;;
--llvm)
llvm=true
shift 1
;;
--monointerpreter)
monointerpreter=true
shift 1
;;
--monoaot)
monoaot=true
shift 1
;;
--monodotnet)
mono_dotnet=$2
shift 2
;;
--wasm)
wasm_runtime_loc=$2
shift 2
;;
--compare)
compare=true
shift 1
;;
--configurations)
configurations=$2
shift 2
;;
--latestdotnet)
use_latest_dotnet=true
shift 1
;;
*)
echo "Common settings:"
echo " --corerootdirectory <value> Directory where Core_Root exists, if running perf testing with --corerun"
echo " --architecture <value> Architecture of the testing being run"
echo " --configurations <value> List of key=value pairs that will be passed to perf testing infrastructure."
echo " ex: --configurations \"CompilationMode=Tiered OptimzationLevel=PGO\""
echo " --help Print help and exit"
echo ""
echo "Advanced settings:"
echo " --framework <value> The framework to run, if not running in master"
echo " --compliationmode <value> The compilation mode if not passing --configurations"
echo " --sourcedirectory <value> The directory of the sources. Defaults to env:BUILD_SOURCESDIRECTORY"
echo " --repository <value> The name of the repository in the <owner>/<repository name> format. Defaults to env:BUILD_REPOSITORY_NAME"
echo " --branch <value> The name of the branch. Defaults to env:BUILD_SOURCEBRANCH"
echo " --commitsha <value> The commit sha1 to run against. Defaults to env:BUILD_SOURCEVERSION"
echo " --buildnumber <value> The build number currently running. Defaults to env:BUILD_BUILDNUMBER"
echo " --csproj The relative path to the benchmark csproj whose tests should be run. Defaults to src\benchmarks\micro\MicroBenchmarks.csproj"
echo " --kind <value> Related to csproj. The kind of benchmarks that should be run. Defaults to micro"
echo " --runcategories <value> Related to csproj. Categories of benchmarks to run. Defaults to \"coreclr corefx\""
echo " --internal If the benchmarks are running as an official job."
echo " --monodotnet Pass the path to the mono dotnet for mono performance testing."
echo " --wasm Path to the unpacked wasm runtime pack."
echo " --latestdotnet --dotnet-versions will not be specified. --dotnet-versions defaults to LKG version in global.json "
echo ""
exit 0
;;
esac
done
if [ "$repository" == "dotnet/performance" ] || [ "$repository" == "dotnet-performance" ]; then
run_from_perf_repo=true
fi
if [ -z "$configurations" ]; then
configurations="CompilationMode=$compilation_mode"
fi
if [ -z "$core_root_directory" ]; then
use_core_run=false
fi
if [ -z "$baseline_core_root_directory" ]; then
use_baseline_core_run=false
fi
payload_directory=$source_directory/Payload
performance_directory=$payload_directory/performance
workitem_directory=$source_directory/workitem
extra_benchmark_dotnet_arguments="--iterationCount 1 --warmupCount 0 --invocationCount 1 --unrollFactor 1 --strategy ColdStart --stopOnFirstError true"
perflab_arguments=
queue=Ubuntu.1804.Amd64.Open
creator=$BUILD_DEFINITIONNAME
helix_source_prefix="pr"
if [[ "$compare" == true ]]; then
extra_benchmark_dotnet_arguments=
perflab_arguments=
# No open queues for arm64
if [[ "$architecture" = "arm64" ]]; then
echo "Compare not available for arm64"
exit 1
fi
queue=Ubuntu.1804.Amd64.Tiger.Perf.Open
fi
if [[ "$internal" == true ]]; then
perflab_arguments="--upload-to-perflab-container"
helix_source_prefix="official"
creator=
extra_benchmark_dotnet_arguments=
if [[ "$architecture" = "arm64" ]]; then
queue=Ubuntu.1804.Arm64.Perf
else
queue=Ubuntu.1804.Amd64.Tiger.Perf
fi
fi
if [[ "$mono_dotnet" != "" ]] && [[ "$monointerpreter" == "false" ]]; then
extra_benchmark_dotnet_arguments="$extra_benchmark_dotnet_arguments --category-exclusion-filter NoMono"
fi
if [[ "$wasm_runtime_loc" != "" ]]; then
configurations="CompilationMode=wasm RunKind=$kind"
extra_benchmark_dotnet_arguments="$extra_benchmark_dotnet_arguments --category-exclusion-filter NoInterpreter NoWASM NoMono"
fi
if [[ "$mono_dotnet" != "" ]] && [[ "$monointerpreter" == "true" ]]; then
configurations="$configurations LLVM=$llvm MonoInterpreter=$monointerpreter MonoAOT=$monoaot"
extra_benchmark_dotnet_arguments="$extra_benchmark_dotnet_arguments --category-exclusion-filter NoInterpreter NoMono"
fi
common_setup_arguments="--channel master --queue $queue --build-number $build_number --build-configs $configurations --architecture $architecture"
setup_arguments="--repository https://github.com/$repository --branch $branch --get-perf-hash --commit-sha $commit_sha $common_setup_arguments"
if [[ "$use_latest_dotnet" = false ]]; then
# Get the tools section from the global.json.
# This grabs the LKG version number of dotnet and passes it to our scripts
dotnet_version=`cat global.json | python3 -c 'import json,sys;obj=json.load(sys.stdin);print(obj["tools"]["dotnet"])'`
setup_arguments="--dotnet-versions $dotnet_version $setup_arguments"
fi
if [[ "$run_from_perf_repo" = true ]]; then
payload_directory=
workitem_directory=$source_directory
performance_directory=$workitem_directory
setup_arguments="--perf-hash $commit_sha $common_setup_arguments"
else
git clone --branch master --depth 1 --quiet https://github.com/dotnet/performance $performance_directory
docs_directory=$performance_directory/docs
mv $docs_directory $workitem_directory
fi
if [[ "$wasm_runtime_loc" != "" ]]; then
using_wasm=true
wasm_dotnet_path=$payload_directory/dotnet-wasm
mv $wasm_runtime_loc $wasm_dotnet_path
extra_benchmark_dotnet_arguments="$extra_benchmark_dotnet_arguments --wasmMainJS \$HELIX_CORRELATION_PAYLOAD/dotnet-wasm/runtime-test.js --wasmEngine /home/helixbot/.jsvu/v8 --customRuntimePack \$HELIX_CORRELATION_PAYLOAD/dotnet-wasm"
fi
if [[ "$mono_dotnet" != "" ]]; then
using_mono=true
mono_dotnet_path=$payload_directory/dotnet-mono
mv $mono_dotnet $mono_dotnet_path
fi
if [[ "$use_core_run" = true ]]; then
new_core_root=$payload_directory/Core_Root
mv $core_root_directory $new_core_root
fi
if [[ "$use_baseline_core_run" = true ]]; then
new_baseline_core_root=$payload_directory/Baseline_Core_Root
mv $baseline_core_root_directory $new_baseline_core_root
fi
ci=true
_script_dir=$(pwd)/eng/common
. "$_script_dir/pipeline-logging-functions.sh"
# Make sure all of our variables are available for future steps
Write-PipelineSetVariable -name "UseCoreRun" -value "$use_core_run" -is_multi_job_variable false
Write-PipelineSetVariable -name "UseBaselineCoreRun" -value "$use_baseline_core_run" -is_multi_job_variable false
Write-PipelineSetVariable -name "Architecture" -value "$architecture" -is_multi_job_variable false
Write-PipelineSetVariable -name "PayloadDirectory" -value "$payload_directory" -is_multi_job_variable false
Write-PipelineSetVariable -name "PerformanceDirectory" -value "$performance_directory" -is_multi_job_variable false
Write-PipelineSetVariable -name "WorkItemDirectory" -value "$workitem_directory" -is_multi_job_variable false
Write-PipelineSetVariable -name "Queue" -value "$queue" -is_multi_job_variable false
Write-PipelineSetVariable -name "SetupArguments" -value "$setup_arguments" -is_multi_job_variable false
Write-PipelineSetVariable -name "Python" -value "python3" -is_multi_job_variable false
Write-PipelineSetVariable -name "PerfLabArguments" -value "$perflab_arguments" -is_multi_job_variable false
Write-PipelineSetVariable -name "ExtraBenchmarkDotNetArguments" -value "$extra_benchmark_dotnet_arguments" -is_multi_job_variable false
Write-PipelineSetVariable -name "BDNCategories" -value "$run_categories" -is_multi_job_variable false
Write-PipelineSetVariable -name "TargetCsproj" -value "$csproj" -is_multi_job_variable false
Write-PipelineSetVariable -name "RunFromPerfRepo" -value "$run_from_perf_repo" -is_multi_job_variable false
Write-PipelineSetVariable -name "Creator" -value "$creator" -is_multi_job_variable false
Write-PipelineSetVariable -name "HelixSourcePrefix" -value "$helix_source_prefix" -is_multi_job_variable false
Write-PipelineSetVariable -name "Kind" -value "$kind" -is_multi_job_variable false
Write-PipelineSetVariable -name "_BuildConfig" -value "$architecture.$kind.$framework" -is_multi_job_variable false
Write-PipelineSetVariable -name "Compare" -value "$compare" -is_multi_job_variable false
Write-PipelineSetVariable -name "MonoDotnet" -value "$using_mono" -is_multi_job_variable false
Write-PipelineSetVariable -name "WasmDotnet" -value "$using_wasm" -is_multi_job_variable false

48
eng/common/post-build/add-build-to-channel.ps1

@ -1,48 +0,0 @@
param(
[Parameter(Mandatory=$true)][int] $BuildId,
[Parameter(Mandatory=$true)][int] $ChannelId,
[Parameter(Mandatory=$true)][string] $MaestroApiAccessToken,
[Parameter(Mandatory=$false)][string] $MaestroApiEndPoint = 'https://maestro-prod.westus2.cloudapp.azure.com',
[Parameter(Mandatory=$false)][string] $MaestroApiVersion = '2019-01-16'
)
try {
. $PSScriptRoot\post-build-utils.ps1
# Check that the channel we are going to promote the build to exist
$channelInfo = Get-MaestroChannel -ChannelId $ChannelId
if (!$channelInfo) {
Write-PipelineTelemetryCategory -Category 'PromoteBuild' -Message "Channel with BAR ID $ChannelId was not found in BAR!"
ExitWithExitCode 1
}
# Get info about which channel(s) the build has already been promoted to
$buildInfo = Get-MaestroBuild -BuildId $BuildId
if (!$buildInfo) {
Write-PipelineTelemetryError -Category 'PromoteBuild' -Message "Build with BAR ID $BuildId was not found in BAR!"
ExitWithExitCode 1
}
# Find whether the build is already assigned to the channel or not
if ($buildInfo.channels) {
foreach ($channel in $buildInfo.channels) {
if ($channel.Id -eq $ChannelId) {
Write-Host "The build with BAR ID $BuildId is already on channel $ChannelId!"
ExitWithExitCode 0
}
}
}
Write-Host "Promoting build '$BuildId' to channel '$ChannelId'."
Assign-BuildToChannel -BuildId $BuildId -ChannelId $ChannelId
Write-Host 'done.'
}
catch {
Write-Host $_
Write-PipelineTelemetryError -Category 'PromoteBuild' -Message "There was an error while trying to promote build '$BuildId' to channel '$ChannelId'"
ExitWithExitCode 1
}

2
eng/common/post-build/nuget-verification.ps1

@ -65,7 +65,7 @@ if ($NuGetExePath) {
Write-Host "Downloading nuget.exe from $nugetExeUrl..."
$ProgressPreference = 'SilentlyContinue'
try {
Invoke-WebRequest $nugetExeUrl -OutFile $downloadedNuGetExe
Invoke-WebRequest $nugetExeUrl -UseBasicParsing -OutFile $downloadedNuGetExe
$ProgressPreference = 'Continue'
} catch {
$ProgressPreference = 'Continue'

91
eng/common/post-build/post-build-utils.ps1

@ -1,91 +0,0 @@
# Most of the functions in this file require the variables `MaestroApiEndPoint`,
# `MaestroApiVersion` and `MaestroApiAccessToken` to be globally available.
$ErrorActionPreference = 'Stop'
Set-StrictMode -Version 2.0
# `tools.ps1` checks $ci to perform some actions. Since the post-build
# scripts don't necessarily execute in the same agent that run the
# build.ps1/sh script this variable isn't automatically set.
$ci = $true
$disableConfigureToolsetImport = $true
. $PSScriptRoot\..\tools.ps1
function Create-MaestroApiRequestHeaders([string]$ContentType = 'application/json') {
Validate-MaestroVars
$headers = New-Object 'System.Collections.Generic.Dictionary[[String],[String]]'
$headers.Add('Accept', $ContentType)
$headers.Add('Authorization',"Bearer $MaestroApiAccessToken")
return $headers
}
function Get-MaestroChannel([int]$ChannelId) {
Validate-MaestroVars
$apiHeaders = Create-MaestroApiRequestHeaders
$apiEndpoint = "$MaestroApiEndPoint/api/channels/${ChannelId}?api-version=$MaestroApiVersion"
$result = try { Invoke-WebRequest -Method Get -Uri $apiEndpoint -Headers $apiHeaders | ConvertFrom-Json } catch { Write-Host "Error: $_" }
return $result
}
function Get-MaestroBuild([int]$BuildId) {
Validate-MaestroVars
$apiHeaders = Create-MaestroApiRequestHeaders -AuthToken $MaestroApiAccessToken
$apiEndpoint = "$MaestroApiEndPoint/api/builds/${BuildId}?api-version=$MaestroApiVersion"
$result = try { return Invoke-WebRequest -Method Get -Uri $apiEndpoint -Headers $apiHeaders | ConvertFrom-Json } catch { Write-Host "Error: $_" }
return $result
}
function Get-MaestroSubscriptions([string]$SourceRepository, [int]$ChannelId) {
Validate-MaestroVars
$SourceRepository = [System.Web.HttpUtility]::UrlEncode($SourceRepository)
$apiHeaders = Create-MaestroApiRequestHeaders -AuthToken $MaestroApiAccessToken
$apiEndpoint = "$MaestroApiEndPoint/api/subscriptions?sourceRepository=$SourceRepository&channelId=$ChannelId&api-version=$MaestroApiVersion"
$result = try { Invoke-WebRequest -Method Get -Uri $apiEndpoint -Headers $apiHeaders | ConvertFrom-Json } catch { Write-Host "Error: $_" }
return $result
}
function Assign-BuildToChannel([int]$BuildId, [int]$ChannelId) {
Validate-MaestroVars
$apiHeaders = Create-MaestroApiRequestHeaders -AuthToken $MaestroApiAccessToken
$apiEndpoint = "$MaestroApiEndPoint/api/channels/${ChannelId}/builds/${BuildId}?api-version=$MaestroApiVersion"
Invoke-WebRequest -Method Post -Uri $apiEndpoint -Headers $apiHeaders | Out-Null
}
function Trigger-Subscription([string]$SubscriptionId) {
Validate-MaestroVars
$apiHeaders = Create-MaestroApiRequestHeaders -AuthToken $MaestroApiAccessToken
$apiEndpoint = "$MaestroApiEndPoint/api/subscriptions/$SubscriptionId/trigger?api-version=$MaestroApiVersion"
Invoke-WebRequest -Uri $apiEndpoint -Headers $apiHeaders -Method Post | Out-Null
}
function Validate-MaestroVars {
try {
Get-Variable MaestroApiEndPoint | Out-Null
Get-Variable MaestroApiVersion | Out-Null
Get-Variable MaestroApiAccessToken | Out-Null
if (!($MaestroApiEndPoint -Match '^http[s]?://maestro-(int|prod).westus2.cloudapp.azure.com$')) {
Write-PipelineTelemetryError -Category 'MaestroVars' -Message "MaestroApiEndPoint is not a valid Maestro URL. '$MaestroApiEndPoint'"
ExitWithExitCode 1
}
if (!($MaestroApiVersion -Match '^[0-9]{4}-[0-9]{2}-[0-9]{2}$')) {
Write-PipelineTelemetryError -Category 'MaestroVars' -Message "MaestroApiVersion does not match a version string in the format yyyy-MM-DD. '$MaestroApiVersion'"
ExitWithExitCode 1
}
}
catch {
Write-PipelineTelemetryError -Category 'MaestroVars' -Message 'Error: Variables `MaestroApiEndPoint`, `MaestroApiVersion` and `MaestroApiAccessToken` are required while using this script.'
Write-Host $_
ExitWithExitCode 1
}
}

4
eng/common/post-build/publish-using-darc.ps1

@ -7,7 +7,9 @@ param(
[Parameter(Mandatory=$false)][string] $ArtifactsPublishingAdditionalParameters,
[Parameter(Mandatory=$false)][string] $SymbolPublishingAdditionalParameters,
[Parameter(Mandatory=$false)][string] $RequireDefaultChannels,
[Parameter(Mandatory=$false)][string] $SkipAssetsPublishing
[Parameter(Mandatory=$false)][string] $SkipAssetsPublishing,
[Parameter(Mandatory=$false)][string] $runtimeSourceFeed,
[Parameter(Mandatory=$false)][string] $runtimeSourceFeedKey
)
try {

4
eng/common/post-build/redact-logs.ps1

@ -7,7 +7,9 @@ param(
# File with strings to redact - separated by newlines.
# For comments start the line with '# ' - such lines are ignored
[Parameter(Mandatory=$false)][string] $TokensFilePath,
[Parameter(ValueFromRemainingArguments=$true)][String[]]$TokensToRedact
[Parameter(ValueFromRemainingArguments=$true)][String[]]$TokensToRedact,
[Parameter(Mandatory=$false)][string] $runtimeSourceFeed,
[Parameter(Mandatory=$false)][string] $runtimeSourceFeedKey
)
try {

64
eng/common/post-build/trigger-subscriptions.ps1

@ -1,64 +0,0 @@
param(
[Parameter(Mandatory=$true)][string] $SourceRepo,
[Parameter(Mandatory=$true)][int] $ChannelId,
[Parameter(Mandatory=$true)][string] $MaestroApiAccessToken,
[Parameter(Mandatory=$false)][string] $MaestroApiEndPoint = 'https://maestro-prod.westus2.cloudapp.azure.com',
[Parameter(Mandatory=$false)][string] $MaestroApiVersion = '2019-01-16'
)
try {
. $PSScriptRoot\post-build-utils.ps1
# Get all the $SourceRepo subscriptions
$normalizedSourceRepo = $SourceRepo.Replace('dnceng@', '')
$subscriptions = Get-MaestroSubscriptions -SourceRepository $normalizedSourceRepo -ChannelId $ChannelId
if (!$subscriptions) {
Write-PipelineTelemetryError -Category 'TriggerSubscriptions' -Message "No subscriptions found for source repo '$normalizedSourceRepo' in channel '$ChannelId'"
ExitWithExitCode 0
}
$subscriptionsToTrigger = New-Object System.Collections.Generic.List[string]
$failedTriggeredSubscription = $false
# Get all enabled subscriptions that need dependency flow on 'everyBuild'
foreach ($subscription in $subscriptions) {
if ($subscription.enabled -and $subscription.policy.updateFrequency -like 'everyBuild' -and $subscription.channel.id -eq $ChannelId) {
Write-Host "Should trigger this subscription: ${$subscription.id}"
[void]$subscriptionsToTrigger.Add($subscription.id)
}
}
foreach ($subscriptionToTrigger in $subscriptionsToTrigger) {
try {
Write-Host "Triggering subscription '$subscriptionToTrigger'."
Trigger-Subscription -SubscriptionId $subscriptionToTrigger
Write-Host 'done.'
}
catch
{
Write-Host "There was an error while triggering subscription '$subscriptionToTrigger'"
Write-Host $_
Write-Host $_.ScriptStackTrace
$failedTriggeredSubscription = $true
}
}
if ($subscriptionsToTrigger.Count -eq 0) {
Write-Host "No subscription matched source repo '$normalizedSourceRepo' and channel ID '$ChannelId'."
}
elseif ($failedTriggeredSubscription) {
Write-PipelineTelemetryError -Category 'TriggerSubscriptions' -Message 'At least one subscription failed to be triggered...'
ExitWithExitCode 1
}
else {
Write-Host 'All subscriptions were triggered successfully!'
}
}
catch {
Write-Host $_.ScriptStackTrace
Write-PipelineTelemetryError -Category 'TriggerSubscriptions' -Message $_
ExitWithExitCode 1
}

4
eng/common/sdk-task.ps1

@ -9,6 +9,8 @@ Param(
[switch][Alias('nobl')]$excludeCIBinaryLog,
[switch]$noWarnAsError,
[switch] $help,
[string] $runtimeSourceFeed = '',
[string] $runtimeSourceFeedKey = '',
[Parameter(ValueFromRemainingArguments=$true)][String[]]$properties
)
@ -68,7 +70,7 @@ try {
$GlobalJson.tools | Add-Member -Name "vs" -Value (ConvertFrom-Json "{ `"version`": `"16.5`" }") -MemberType NoteProperty
}
if( -not ($GlobalJson.tools.PSObject.Properties.Name -match "xcopy-msbuild" )) {
$GlobalJson.tools | Add-Member -Name "xcopy-msbuild" -Value "17.13.0" -MemberType NoteProperty
$GlobalJson.tools | Add-Member -Name "xcopy-msbuild" -Value "18.0.0" -MemberType NoteProperty
}
if ($GlobalJson.tools."xcopy-msbuild".Trim() -ine "none") {
$xcopyMSBuildToolsFolder = InitializeXCopyMSBuild $GlobalJson.tools."xcopy-msbuild" -install $true

69
eng/common/sdl/push-gdn.ps1

@ -1,69 +0,0 @@
Param(
[string] $Repository,
[string] $BranchName='master',
[string] $GdnFolder,
[string] $AzureDevOpsAccessToken,
[string] $PushReason
)
$ErrorActionPreference = 'Stop'
Set-StrictMode -Version 2.0
$disableConfigureToolsetImport = $true
$LASTEXITCODE = 0
try {
# `tools.ps1` checks $ci to perform some actions. Since the SDL
# scripts don't necessarily execute in the same agent that run the
# build.ps1/sh script this variable isn't automatically set.
$ci = $true
. $PSScriptRoot\..\tools.ps1
# We create the temp directory where we'll store the sdl-config repository
$sdlDir = Join-Path $env:TEMP 'sdl'
if (Test-Path $sdlDir) {
Remove-Item -Force -Recurse $sdlDir
}
Write-Host "git clone https://dnceng:`$AzureDevOpsAccessToken@dev.azure.com/dnceng/internal/_git/sdl-tool-cfg $sdlDir"
git clone https://dnceng:$AzureDevOpsAccessToken@dev.azure.com/dnceng/internal/_git/sdl-tool-cfg $sdlDir
if ($LASTEXITCODE -ne 0) {
Write-PipelineTelemetryError -Force -Category 'Sdl' -Message "Git clone failed with exit code $LASTEXITCODE."
ExitWithExitCode $LASTEXITCODE
}
# We copy the .gdn folder from our local run into the git repository so it can be committed
$sdlRepositoryFolder = Join-Path (Join-Path (Join-Path $sdlDir $Repository) $BranchName) '.gdn'
if (Get-Command Robocopy) {
Robocopy /S $GdnFolder $sdlRepositoryFolder
} else {
rsync -r $GdnFolder $sdlRepositoryFolder
}
# cd to the sdl-config directory so we can run git there
Push-Location $sdlDir
# git add . --> git commit --> git push
Write-Host 'git add .'
git add .
if ($LASTEXITCODE -ne 0) {
Write-PipelineTelemetryError -Force -Category 'Sdl' -Message "Git add failed with exit code $LASTEXITCODE."
ExitWithExitCode $LASTEXITCODE
}
Write-Host "git -c user.email=`"dn-bot@microsoft.com`" -c user.name=`"Dotnet Bot`" commit -m `"$PushReason for $Repository/$BranchName`""
git -c user.email="dn-bot@microsoft.com" -c user.name="Dotnet Bot" commit -m "$PushReason for $Repository/$BranchName"
if ($LASTEXITCODE -ne 0) {
Write-PipelineTelemetryError -Force -Category 'Sdl' -Message "Git commit failed with exit code $LASTEXITCODE."
ExitWithExitCode $LASTEXITCODE
}
Write-Host 'git push'
git push
if ($LASTEXITCODE -ne 0) {
Write-PipelineTelemetryError -Force -Category 'Sdl' -Message "Git push failed with exit code $LASTEXITCODE."
ExitWithExitCode $LASTEXITCODE
}
# Return to the original directory
Pop-Location
}
catch {
Write-Host $_.ScriptStackTrace
Write-PipelineTelemetryError -Category 'Sdl' -Message $_
ExitWithExitCode 1
}

139
eng/common/templates/job/execute-sdl.yml

@ -1,139 +0,0 @@
parameters:
enable: 'false' # Whether the SDL validation job should execute or not
overrideParameters: '' # Optional: to override values for parameters.
additionalParameters: '' # Optional: parameters that need user specific values eg: '-SourceToolsList @("abc","def") -ArtifactToolsList @("ghi","jkl")'
# Optional: if specified, restore and use this version of Guardian instead of the default.
overrideGuardianVersion: ''
# Optional: if true, publish the '.gdn' folder as a pipeline artifact. This can help with in-depth
# diagnosis of problems with specific tool configurations.
publishGuardianDirectoryToPipeline: false
# The script to run to execute all SDL tools. Use this if you want to use a script to define SDL
# parameters rather than relying on YAML. It may be better to use a local script, because you can
# reproduce results locally without piecing together a command based on the YAML.
executeAllSdlToolsScript: 'eng/common/sdl/execute-all-sdl-tools.ps1'
# There is some sort of bug (has been reported) in Azure DevOps where if this parameter is named
# 'continueOnError', the parameter value is not correctly picked up.
# This can also be remedied by the caller (post-build.yml) if it does not use a nested parameter
sdlContinueOnError: false # optional: determines whether to continue the build if the step errors;
# optional: determines if build artifacts should be downloaded.
downloadArtifacts: true
# optional: determines if this job should search the directory of downloaded artifacts for
# 'tar.gz' and 'zip' archive files and extract them before running SDL validation tasks.
extractArchiveArtifacts: false
dependsOn: '' # Optional: dependencies of the job
artifactNames: '' # Optional: patterns supplied to DownloadBuildArtifacts
# Usage:
# artifactNames:
# - 'BlobArtifacts'
# - 'Artifacts_Windows_NT_Release'
# Optional: download a list of pipeline artifacts. 'downloadArtifacts' controls build artifacts,
# not pipeline artifacts, so doesn't affect the use of this parameter.
pipelineArtifactNames: []
jobs:
- job: Run_SDL
dependsOn: ${{ parameters.dependsOn }}
displayName: Run SDL tool
condition: and(succeededOrFailed(), eq( ${{ parameters.enable }}, 'true'))
variables:
- group: DotNet-VSTS-Bot
- name: AzDOProjectName
value: ${{ parameters.AzDOProjectName }}
- name: AzDOPipelineId
value: ${{ parameters.AzDOPipelineId }}
- name: AzDOBuildId
value: ${{ parameters.AzDOBuildId }}
- template: /eng/common/templates/variables/sdl-variables.yml
- name: GuardianVersion
value: ${{ coalesce(parameters.overrideGuardianVersion, '$(DefaultGuardianVersion)') }}
- template: /eng/common/templates/variables/pool-providers.yml
pool:
# We don't use the collection uri here because it might vary (.visualstudio.com vs. dev.azure.com)
${{ if eq(variables['System.TeamProject'], 'DevDiv') }}:
name: VSEngSS-MicroBuild2022-1ES
demands: Cmd
# If it's not devdiv, it's dnceng
${{ if ne(variables['System.TeamProject'], 'DevDiv') }}:
name: $(DncEngInternalBuildPool)
demands: ImageOverride -equals windows.vs2019.amd64
steps:
- checkout: self
clean: true
# If the template caller didn't provide an AzDO parameter, set them all up as Maestro vars.
- ${{ if not(and(parameters.AzDOProjectName, parameters.AzDOPipelineId, parameters.AzDOBuildId)) }}:
- template: /eng/common/templates/post-build/setup-maestro-vars.yml
- ${{ if ne(parameters.downloadArtifacts, 'false')}}:
- ${{ if ne(parameters.artifactNames, '') }}:
- ${{ each artifactName in parameters.artifactNames }}:
- task: DownloadBuildArtifacts@0
displayName: Download Build Artifacts
inputs:
buildType: specific
buildVersionToDownload: specific
project: $(AzDOProjectName)
pipeline: $(AzDOPipelineId)
buildId: $(AzDOBuildId)
artifactName: ${{ artifactName }}
downloadPath: $(Build.ArtifactStagingDirectory)\artifacts
checkDownloadedFiles: true
- ${{ if eq(parameters.artifactNames, '') }}:
- task: DownloadBuildArtifacts@0
displayName: Download Build Artifacts
inputs:
buildType: specific
buildVersionToDownload: specific
project: $(AzDOProjectName)
pipeline: $(AzDOPipelineId)
buildId: $(AzDOBuildId)
downloadType: specific files
itemPattern: "**"
downloadPath: $(Build.ArtifactStagingDirectory)\artifacts
checkDownloadedFiles: true
- ${{ each artifactName in parameters.pipelineArtifactNames }}:
- task: DownloadPipelineArtifact@2
displayName: Download Pipeline Artifacts
inputs:
buildType: specific
buildVersionToDownload: specific
project: $(AzDOProjectName)
pipeline: $(AzDOPipelineId)
buildId: $(AzDOBuildId)
artifactName: ${{ artifactName }}
downloadPath: $(Build.ArtifactStagingDirectory)\artifacts
checkDownloadedFiles: true
- powershell: eng/common/sdl/trim-assets-version.ps1
-InputPath $(Build.ArtifactStagingDirectory)\artifacts
displayName: Trim the version from the NuGet packages
continueOnError: ${{ parameters.sdlContinueOnError }}
- powershell: eng/common/sdl/extract-artifact-packages.ps1
-InputPath $(Build.ArtifactStagingDirectory)\artifacts\BlobArtifacts
-ExtractPath $(Build.ArtifactStagingDirectory)\artifacts\BlobArtifacts
displayName: Extract Blob Artifacts
continueOnError: ${{ parameters.sdlContinueOnError }}
- powershell: eng/common/sdl/extract-artifact-packages.ps1
-InputPath $(Build.ArtifactStagingDirectory)\artifacts\PackageArtifacts
-ExtractPath $(Build.ArtifactStagingDirectory)\artifacts\PackageArtifacts
displayName: Extract Package Artifacts
continueOnError: ${{ parameters.sdlContinueOnError }}
- ${{ if ne(parameters.extractArchiveArtifacts, 'false') }}:
- powershell: eng/common/sdl/extract-artifact-archives.ps1
-InputPath $(Build.ArtifactStagingDirectory)\artifacts
-ExtractPath $(Build.ArtifactStagingDirectory)\artifacts
displayName: Extract Archive Artifacts
continueOnError: ${{ parameters.sdlContinueOnError }}
- template: /eng/common/templates/steps/execute-sdl.yml
parameters:
overrideGuardianVersion: ${{ parameters.overrideGuardianVersion }}
executeAllSdlToolsScript: ${{ parameters.executeAllSdlToolsScript }}
overrideParameters: ${{ parameters.overrideParameters }}
additionalParameters: ${{ parameters.additionalParameters }}
publishGuardianDirectoryToPipeline: ${{ parameters.publishGuardianDirectoryToPipeline }}
sdlContinueOnError: ${{ parameters.sdlContinueOnError }}

48
eng/common/templates/job/generate-graph-files.yml

@ -1,48 +0,0 @@
parameters:
# Optional: dependencies of the job
dependsOn: ''
# Optional: A defined YAML pool - https://docs.microsoft.com/en-us/azure/devops/pipelines/yaml-schema?view=vsts&tabs=schema#pool
pool: {}
# Optional: Include toolset dependencies in the generated graph files
includeToolset: false
jobs:
- job: Generate_Graph_Files
dependsOn: ${{ parameters.dependsOn }}
displayName: Generate Graph Files
pool: ${{ parameters.pool }}
variables:
# Publish-Build-Assets provides: MaestroAccessToken, BotAccount-dotnet-maestro-bot-PAT
# DotNet-AllOrgs-Darc-Pats provides: dn-bot-devdiv-dnceng-rw-code-pat
- group: Publish-Build-Assets
- group: DotNet-AllOrgs-Darc-Pats
- name: _GraphArguments
value: -gitHubPat $(BotAccount-dotnet-maestro-bot-PAT)
-azdoPat $(dn-bot-devdiv-dnceng-rw-code-pat)
-barToken $(MaestroAccessToken)
-outputFolder '$(Build.StagingDirectory)/GraphFiles/'
- ${{ if ne(parameters.includeToolset, 'false') }}:
- name: _GraphArguments
value: ${{ variables._GraphArguments }} -includeToolset
steps:
- task: PowerShell@2
displayName: Generate Graph Files
inputs:
filePath: eng\common\generate-graph-files.ps1
arguments: $(_GraphArguments)
continueOnError: true
- task: PublishBuildArtifacts@1
displayName: Publish Graph to Artifacts
inputs:
PathtoPublish: '$(Build.StagingDirectory)/GraphFiles'
PublishLocation: Container
ArtifactName: GraphFiles
continueOnError: true
condition: always()

95
eng/common/templates/job/performance.yml

@ -1,95 +0,0 @@
parameters:
steps: [] # optional -- any additional steps that need to happen before pulling down the performance repo and sending the performance benchmarks to helix (ie building your repo)
variables: [] # optional -- list of additional variables to send to the template
jobName: '' # required -- job name
displayName: '' # optional -- display name for the job. Will use jobName if not passed
pool: '' # required -- name of the Build pool
container: '' # required -- name of the container
osGroup: '' # required -- operating system for the job
extraSetupParameters: '' # optional -- extra arguments to pass to the setup script
frameworks: ['netcoreapp3.0'] # optional -- list of frameworks to run against
continueOnError: 'false' # optional -- determines whether to continue the build if the step errors
dependsOn: '' # optional -- dependencies of the job
timeoutInMinutes: 320 # optional -- timeout for the job
enableTelemetry: false # optional -- enable for telemetry
jobs:
- template: ../jobs/jobs.yml
parameters:
dependsOn: ${{ parameters.dependsOn }}
enableTelemetry: ${{ parameters.enableTelemetry }}
enablePublishBuildArtifacts: true
continueOnError: ${{ parameters.continueOnError }}
jobs:
- job: '${{ parameters.jobName }}'
${{ if ne(parameters.displayName, '') }}:
displayName: '${{ parameters.displayName }}'
${{ if eq(parameters.displayName, '') }}:
displayName: '${{ parameters.jobName }}'
timeoutInMinutes: ${{ parameters.timeoutInMinutes }}
variables:
- ${{ each variable in parameters.variables }}:
- ${{ if ne(variable.name, '') }}:
- name: ${{ variable.name }}
value: ${{ variable.value }}
- ${{ if ne(variable.group, '') }}:
- group: ${{ variable.group }}
- IsInternal: ''
- HelixApiAccessToken: ''
- HelixPreCommand: ''
- ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}:
- ${{ if eq( parameters.osGroup, 'Windows_NT') }}:
- HelixPreCommand: 'set "PERFLAB_UPLOAD_TOKEN=$(PerfCommandUploadToken)"'
- IsInternal: -Internal
- ${{ if ne(parameters.osGroup, 'Windows_NT') }}:
- HelixPreCommand: 'export PERFLAB_UPLOAD_TOKEN="$(PerfCommandUploadTokenLinux)"'
- IsInternal: --internal
- group: DotNet-HelixApi-Access
- group: dotnet-benchview
workspace:
clean: all
pool:
${{ parameters.pool }}
container: ${{ parameters.container }}
strategy:
matrix:
${{ each framework in parameters.frameworks }}:
${{ framework }}:
_Framework: ${{ framework }}
steps:
- checkout: self
clean: true
# Run all of the steps to setup repo
- ${{ each step in parameters.steps }}:
- ${{ step }}
- powershell: $(Build.SourcesDirectory)\eng\common\performance\performance-setup.ps1 $(IsInternal) -Framework $(_Framework) ${{ parameters.extraSetupParameters }}
displayName: Performance Setup (Windows)
condition: and(succeeded(), eq(variables['Agent.Os'], 'Windows_NT'))
continueOnError: ${{ parameters.continueOnError }}
- script: $(Build.SourcesDirectory)/eng/common/performance/performance-setup.sh $(IsInternal) --framework $(_Framework) ${{ parameters.extraSetupParameters }}
displayName: Performance Setup (Unix)
condition: and(succeeded(), ne(variables['Agent.Os'], 'Windows_NT'))
continueOnError: ${{ parameters.continueOnError }}
- script: $(Python) $(PerformanceDirectory)/scripts/ci_setup.py $(SetupArguments)
displayName: Run ci setup script
# Run perf testing in helix
- template: /eng/common/templates/steps/perf-send-to-helix.yml
parameters:
HelixSource: '$(HelixSourcePrefix)/$(Build.Repository.Name)/$(Build.SourceBranch)' # sources must start with pr/, official/, prodcon/, or agent/
HelixType: 'test/performance/$(Kind)/$(_Framework)/$(Architecture)'
HelixAccessToken: $(HelixApiAccessToken)
HelixTargetQueues: $(Queue)
HelixPreCommands: $(HelixPreCommand)
Creator: $(Creator)
WorkItemTimeout: 4:00 # 4 hours
WorkItemDirectory: '$(WorkItemDirectory)' # WorkItemDirectory can not be empty, so we send it some docs to keep it happy
CorrelationPayloadDirectory: '$(PayloadDirectory)' # it gets checked out to a folder with shorter path than WorkItemDirectory so we can avoid file name too long exceptions

130
eng/common/templates/phases/base.yml

@ -1,130 +0,0 @@
parameters:
# Optional: Clean sources before building
clean: true
# Optional: Git fetch depth
fetchDepth: ''
# Optional: name of the phase (not specifying phase name may cause name collisions)
name: ''
# Optional: display name of the phase
displayName: ''
# Optional: condition for the job to run
condition: ''
# Optional: dependencies of the phase
dependsOn: ''
# Required: A defined YAML queue
queue: {}
# Required: build steps
steps: []
# Optional: variables
variables: {}
# Optional: should run as a public build even in the internal project
# if 'true', the build won't run any of the internal only steps, even if it is running in non-public projects.
runAsPublic: false
## Telemetry variables
# Optional: enable sending telemetry
# if 'true', these "variables" must be specified in the variables object or as part of the queue matrix
# _HelixBuildConfig - differentiate between Debug, Release, other
# _HelixSource - Example: build/product
# _HelixType - Example: official/dotnet/arcade/$(Build.SourceBranch)
enableTelemetry: false
# Optional: Enable installing Microbuild plugin
# if 'true', these "variables" must be specified in the variables object or as part of the queue matrix
# _TeamName - the name of your team
# _SignType - 'test' or 'real'
enableMicrobuild: false
# Internal resources (telemetry, microbuild) can only be accessed from non-public projects,
# and some (Microbuild) should only be applied to non-PR cases for internal builds.
phases:
- phase: ${{ parameters.name }}
${{ if ne(parameters.displayName, '') }}:
displayName: ${{ parameters.displayName }}
${{ if ne(parameters.condition, '') }}:
condition: ${{ parameters.condition }}
${{ if ne(parameters.dependsOn, '') }}:
dependsOn: ${{ parameters.dependsOn }}
queue: ${{ parameters.queue }}
${{ if ne(parameters.variables, '') }}:
variables:
${{ insert }}: ${{ parameters.variables }}
steps:
- checkout: self
clean: ${{ parameters.clean }}
${{ if ne(parameters.fetchDepth, '') }}:
fetchDepth: ${{ parameters.fetchDepth }}
- ${{ if eq(parameters.enableTelemetry, 'true') }}:
- template: /eng/common/templates/steps/telemetry-start.yml
parameters:
buildConfig: $(_HelixBuildConfig)
helixSource: $(_HelixSource)
helixType: $(_HelixType)
runAsPublic: ${{ parameters.runAsPublic }}
- ${{ if eq(parameters.enableMicrobuild, 'true') }}:
# Internal only resource, and Microbuild signing shouldn't be applied to PRs.
- ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}:
- task: MicroBuildSigningPlugin@2
displayName: Install MicroBuild plugin
inputs:
signType: $(_SignType)
zipSources: false
feedSource: https://dnceng.pkgs.visualstudio.com/_packaging/MicroBuildToolset/nuget/v3/index.json
env:
TeamName: $(_TeamName)
continueOnError: false
condition: and(succeeded(), in(variables['_SignType'], 'real', 'test'), eq(variables['Agent.Os'], 'Windows_NT'))
# Run provided build steps
- ${{ parameters.steps }}
- ${{ if eq(parameters.enableMicrobuild, 'true') }}:
# Internal only resources
- ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}:
- task: MicroBuildCleanup@1
displayName: Execute Microbuild cleanup tasks
condition: and(always(), in(variables['_SignType'], 'real', 'test'), eq(variables['Agent.Os'], 'Windows_NT'))
env:
TeamName: $(_TeamName)
- ${{ if eq(parameters.enableTelemetry, 'true') }}:
- template: /eng/common/templates/steps/telemetry-end.yml
parameters:
helixSource: $(_HelixSource)
helixType: $(_HelixType)
- ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}:
- task: CopyFiles@2
displayName: Gather Asset Manifests
inputs:
SourceFolder: '$(Build.SourcesDirectory)/artifacts/log/$(_BuildConfig)/AssetManifest'
TargetFolder: '$(Build.StagingDirectory)/AssetManifests'
continueOnError: false
condition: and(succeeded(), eq(variables['_DotNetPublishToBlobFeed'], 'true'))
- task: PublishBuildArtifacts@1
displayName: Push Asset Manifests
inputs:
PathtoPublish: '$(Build.StagingDirectory)/AssetManifests'
PublishLocation: Container
ArtifactName: AssetManifests
continueOnError: false
condition: and(succeeded(), eq(variables['_DotNetPublishToBlobFeed'], 'true'))

52
eng/common/templates/phases/publish-build-assets.yml

@ -1,52 +0,0 @@
parameters:
dependsOn: ''
queue: {}
configuration: 'Debug'
condition: succeeded()
continueOnError: false
runAsPublic: false
publishUsingPipelines: false
phases:
- phase: Asset_Registry_Publish
displayName: Publish to Build Asset Registry
dependsOn: ${{ parameters.dependsOn }}
queue: ${{ parameters.queue }}
variables:
_BuildConfig: ${{ parameters.configuration }}
steps:
- ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}:
- task: DownloadBuildArtifacts@0
displayName: Download artifact
inputs:
artifactName: AssetManifests
downloadPath: '$(Build.StagingDirectory)/Download'
checkDownloadedFiles: true
condition: ${{ parameters.condition }}
continueOnError: ${{ parameters.continueOnError }}
- task: AzureKeyVault@1
inputs:
azureSubscription: 'DotNet-Engineering-Services_KeyVault'
KeyVaultName: EngKeyVault
SecretsFilter: 'MaestroAccessToken'
condition: ${{ parameters.condition }}
continueOnError: ${{ parameters.continueOnError }}
- task: PowerShell@2
displayName: Publish Build Assets
inputs:
filePath: eng\common\sdk-task.ps1
arguments: -task PublishBuildAssets -restore -msbuildEngine dotnet
/p:ManifestsPath='$(Build.StagingDirectory)/Download/AssetManifests'
/p:BuildAssetRegistryToken=$(MaestroAccessToken)
/p:MaestroApiEndpoint=https://maestro-prod.westus2.cloudapp.azure.com
/p:PublishUsingPipelines=${{ parameters.publishUsingPipelines }}
/p:Configuration=$(_BuildConfig)
condition: ${{ parameters.condition }}
continueOnError: ${{ parameters.continueOnError }}
- task: PublishBuildArtifacts@1
displayName: Publish Logs to VSTS
inputs:
PathtoPublish: '$(Build.SourcesDirectory)/artifacts/log/$(_BuildConfig)'
PublishLocation: Container
ArtifactName: $(Agent.Os)_Asset_Registry_Publish
continueOnError: true
condition: always()

190
eng/common/templates/post-build/channels/generic-internal-channel.yml

@ -1,190 +0,0 @@
parameters:
BARBuildId: ''
PromoteToChannelIds: ''
artifactsPublishingAdditionalParameters: ''
dependsOn:
- Validate
publishInstallersAndChecksums: true
symbolPublishingAdditionalParameters: ''
stageName: ''
channelName: ''
channelId: ''
transportFeed: ''
shippingFeed: ''
symbolsFeed: ''
stages:
- stage: ${{ parameters.stageName }}
dependsOn: ${{ parameters.dependsOn }}
variables:
- template: ../common-variables.yml
displayName: ${{ parameters.channelName }} Publishing
jobs:
- template: ../setup-maestro-vars.yml
parameters:
BARBuildId: ${{ parameters.BARBuildId }}
PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }}
- job: publish_symbols
displayName: Symbol Publishing
dependsOn: setupMaestroVars
condition: contains(dependencies.setupMaestroVars.outputs['setReleaseVars.TargetChannels'], format('[{0}]', ${{ parameters.channelId }} ))
variables:
- group: DotNet-Symbol-Server-Pats
- name: AzDOProjectName
value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOProjectName'] ]
- name: AzDOPipelineId
value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOPipelineId'] ]
- name: AzDOBuildId
value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOBuildId'] ]
pool:
vmImage: 'windows-2019'
steps:
- script: echo "##vso[task.logissue type=warning]Going forward, v2 Arcade publishing is no longer supported. Please read https://github.com/dotnet/arcade/blob/main/Documentation/CorePackages/Publishing.md for details, then contact dnceng if you have further questions."
displayName: Warn about v2 Arcade Publishing Usage
# This is necessary whenever we want to publish/restore to an AzDO private feed
- task: NuGetAuthenticate@0
displayName: 'Authenticate to AzDO Feeds'
- task: DownloadBuildArtifacts@0
displayName: Download Build Assets
continueOnError: true
inputs:
buildType: specific
buildVersionToDownload: specific
project: $(AzDOProjectName)
pipeline: $(AzDOPipelineId)
buildId: $(AzDOBuildId)
downloadType: 'specific'
itemPattern: |
PdbArtifacts/**
BlobArtifacts/**
downloadPath: '$(Build.ArtifactStagingDirectory)'
checkDownloadedFiles: true
# This is necessary whenever we want to publish/restore to an AzDO private feed
# Since sdk-task.ps1 tries to restore packages we need to do this authentication here
# otherwise it'll complain about accessing a private feed.
- task: NuGetAuthenticate@0
displayName: 'Authenticate to AzDO Feeds'
- task: PowerShell@2
displayName: Enable cross-org publishing
inputs:
filePath: eng\common\enable-cross-org-publishing.ps1
arguments: -token $(dn-bot-dnceng-artifact-feeds-rw)
- task: PowerShell@2
displayName: Publish
inputs:
filePath: eng\common\sdk-task.ps1
arguments: -task PublishToSymbolServers -restore -msbuildEngine dotnet
/p:DotNetSymbolServerTokenMsdl=$(microsoft-symbol-server-pat)
/p:DotNetSymbolServerTokenSymWeb=$(symweb-symbol-server-pat)
/p:PDBArtifactsDirectory='$(Build.ArtifactStagingDirectory)/PDBArtifacts/'
/p:BlobBasePath='$(Build.ArtifactStagingDirectory)/BlobArtifacts/'
/p:SymbolPublishingExclusionsFile='$(Build.SourcesDirectory)/eng/SymbolPublishingExclusionsFile.txt'
/p:Configuration=Release
/p:PublishToMSDL=false
${{ parameters.symbolPublishingAdditionalParameters }}
- template: ../../steps/publish-logs.yml
parameters:
StageLabel: '${{ parameters.stageName }}'
JobLabel: 'SymbolPublishing'
- job: publish_assets
displayName: Publish Assets
dependsOn: setupMaestroVars
timeoutInMinutes: 120
variables:
- name: BARBuildId
value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.BARBuildId'] ]
- name: IsStableBuild
value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.IsStableBuild'] ]
- name: AzDOProjectName
value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOProjectName'] ]
- name: AzDOPipelineId
value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOPipelineId'] ]
- name: AzDOBuildId
value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOBuildId'] ]
condition: contains(dependencies.setupMaestroVars.outputs['setReleaseVars.TargetChannels'], format('[{0}]', ${{ parameters.channelId }} ))
pool:
vmImage: 'windows-2019'
steps:
- script: echo "##vso[task.logissue type=warning]Going forward, v2 Arcade publishing is no longer supported. Please read https://github.com/dotnet/arcade/blob/main/Documentation/CorePackages/Publishing.md for details, then contact dnceng if you have further questions."
displayName: Warn about v2 Arcade Publishing Usage
- task: DownloadBuildArtifacts@0
displayName: Download Build Assets
continueOnError: true
inputs:
buildType: specific
buildVersionToDownload: specific
project: $(AzDOProjectName)
pipeline: $(AzDOPipelineId)
buildId: $(AzDOBuildId)
downloadType: 'specific'
itemPattern: |
PackageArtifacts/**
BlobArtifacts/**
AssetManifests/**
downloadPath: '$(Build.ArtifactStagingDirectory)'
checkDownloadedFiles: true
- task: NuGetToolInstaller@1
displayName: 'Install NuGet.exe'
# This is necessary whenever we want to publish/restore to an AzDO private feed
- task: NuGetAuthenticate@0
displayName: 'Authenticate to AzDO Feeds'
- task: PowerShell@2
displayName: Enable cross-org publishing
inputs:
filePath: eng\common\enable-cross-org-publishing.ps1
arguments: -token $(dn-bot-dnceng-artifact-feeds-rw)
- task: PowerShell@2
displayName: Publish Assets
inputs:
filePath: eng\common\sdk-task.ps1
arguments: -task PublishArtifactsInManifest -restore -msbuildEngine dotnet
/p:PublishingInfraVersion=2
/p:IsStableBuild=$(IsStableBuild)
/p:IsInternalBuild=$(IsInternalBuild)
/p:RepositoryName=$(Build.Repository.Name)
/p:CommitSha=$(Build.SourceVersion)
/p:NugetPath=$(NuGetExeToolPath)
/p:AzdoTargetFeedPAT='$(dn-bot-dnceng-universal-packages-rw)'
/p:AzureStorageTargetFeedPAT='$(dotnetfeed-storage-access-key-1)'
/p:BARBuildId=$(BARBuildId)
/p:MaestroApiEndpoint='$(MaestroApiEndPoint)'
/p:BuildAssetRegistryToken='$(MaestroApiAccessToken)'
/p:ManifestsBasePath='$(Build.ArtifactStagingDirectory)/AssetManifests/'
/p:BlobBasePath='$(Build.ArtifactStagingDirectory)/BlobArtifacts/'
/p:PackageBasePath='$(Build.ArtifactStagingDirectory)/PackageArtifacts/'
/p:Configuration=Release
/p:PublishInstallersAndChecksums=${{ parameters.publishInstallersAndChecksums }}
/p:ChecksumsTargetStaticFeed=$(InternalChecksumsBlobFeedUrl)
/p:ChecksumsAzureAccountKey=$(InternalChecksumsBlobFeedKey)
/p:InstallersTargetStaticFeed=$(InternalInstallersBlobFeedUrl)
/p:InstallersAzureAccountKey=$(InternalInstallersBlobFeedKey)
/p:AzureDevOpsStaticShippingFeed='${{ parameters.shippingFeed }}'
/p:AzureDevOpsStaticShippingFeedKey='$(dn-bot-dnceng-artifact-feeds-rw)'
/p:AzureDevOpsStaticTransportFeed='${{ parameters.transportFeed }}'
/p:AzureDevOpsStaticTransportFeedKey='$(dn-bot-dnceng-artifact-feeds-rw)'
/p:AzureDevOpsStaticSymbolsFeed='${{ parameters.symbolsFeed }}'
/p:AzureDevOpsStaticSymbolsFeedKey='$(dn-bot-dnceng-artifact-feeds-rw)'
/p:PublishToMSDL=false
${{ parameters.artifactsPublishingAdditionalParameters }}
- template: ../../steps/publish-logs.yml
parameters:
StageLabel: '${{ parameters.stageName }}'
JobLabel: 'AssetsPublishing'
- template: ../../steps/add-build-to-channel.yml
parameters:
ChannelId: ${{ parameters.channelId }}

192
eng/common/templates/post-build/channels/generic-public-channel.yml

@ -1,192 +0,0 @@
parameters:
BARBuildId: ''
PromoteToChannelIds: ''
artifactsPublishingAdditionalParameters: ''
dependsOn:
- Validate
publishInstallersAndChecksums: true
symbolPublishingAdditionalParameters: ''
stageName: ''
channelName: ''
channelId: ''
transportFeed: ''
shippingFeed: ''
symbolsFeed: ''
# If the channel name is empty, no links will be generated
akaMSChannelName: ''
stages:
- stage: ${{ parameters.stageName }}
dependsOn: ${{ parameters.dependsOn }}
variables:
- template: ../common-variables.yml
displayName: ${{ parameters.channelName }} Publishing
jobs:
- template: ../setup-maestro-vars.yml
parameters:
BARBuildId: ${{ parameters.BARBuildId }}
PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }}
- job: publish_symbols
displayName: Symbol Publishing
dependsOn: setupMaestroVars
condition: contains(dependencies.setupMaestroVars.outputs['setReleaseVars.TargetChannels'], format('[{0}]', ${{ parameters.channelId }} ))
variables:
- group: DotNet-Symbol-Server-Pats
- name: AzDOProjectName
value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOProjectName'] ]
- name: AzDOPipelineId
value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOPipelineId'] ]
- name: AzDOBuildId
value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOBuildId'] ]
pool:
vmImage: 'windows-2019'
steps:
- script: echo "##vso[task.logissue type=warning]Going forward, v2 Arcade publishing is no longer supported. Please read https://github.com/dotnet/arcade/blob/main/Documentation/CorePackages/Publishing.md for details, then contact dnceng if you have further questions."
displayName: Warn about v2 Arcade Publishing Usage
- task: DownloadBuildArtifacts@0
displayName: Download Build Assets
continueOnError: true
inputs:
buildType: specific
buildVersionToDownload: specific
project: $(AzDOProjectName)
pipeline: $(AzDOPipelineId)
buildId: $(AzDOBuildId)
downloadType: 'specific'
itemPattern: |
PdbArtifacts/**
BlobArtifacts/**
downloadPath: '$(Build.ArtifactStagingDirectory)'
checkDownloadedFiles: true
# This is necessary whenever we want to publish/restore to an AzDO private feed
# Since sdk-task.ps1 tries to restore packages we need to do this authentication here
# otherwise it'll complain about accessing a private feed.
- task: NuGetAuthenticate@0
displayName: 'Authenticate to AzDO Feeds'
- task: PowerShell@2
displayName: Enable cross-org publishing
inputs:
filePath: eng\common\enable-cross-org-publishing.ps1
arguments: -token $(dn-bot-dnceng-artifact-feeds-rw)
- task: PowerShell@2
displayName: Publish
inputs:
filePath: eng\common\sdk-task.ps1
arguments: -task PublishToSymbolServers -restore -msbuildEngine dotnet
/p:DotNetSymbolServerTokenMsdl=$(microsoft-symbol-server-pat)
/p:DotNetSymbolServerTokenSymWeb=$(symweb-symbol-server-pat)
/p:PDBArtifactsDirectory='$(Build.ArtifactStagingDirectory)/PDBArtifacts/'
/p:BlobBasePath='$(Build.ArtifactStagingDirectory)/BlobArtifacts/'
/p:SymbolPublishingExclusionsFile='$(Build.SourcesDirectory)/eng/SymbolPublishingExclusionsFile.txt'
/p:Configuration=Release
${{ parameters.symbolPublishingAdditionalParameters }}
- template: ../../steps/publish-logs.yml
parameters:
StageLabel: '${{ parameters.stageName }}'
JobLabel: 'SymbolPublishing'
- job: publish_assets
displayName: Publish Assets
dependsOn: setupMaestroVars
timeoutInMinutes: 120
variables:
- name: BARBuildId
value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.BARBuildId'] ]
- name: IsStableBuild
value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.IsStableBuild'] ]
- name: AzDOProjectName
value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOProjectName'] ]
- name: AzDOPipelineId
value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOPipelineId'] ]
- name: AzDOBuildId
value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOBuildId'] ]
- name: ArtifactsCategory
value: ${{ coalesce(variables._DotNetArtifactsCategory, '.NETCore') }}
condition: contains(dependencies.setupMaestroVars.outputs['setReleaseVars.TargetChannels'], format('[{0}]', ${{ parameters.channelId }} ))
pool:
vmImage: 'windows-2019'
steps:
- script: echo "##vso[task.logissue type=warning]Going forward, v2 Arcade publishing is no longer supported. Please read https://github.com/dotnet/arcade/blob/main/Documentation/CorePackages/Publishing.md for details, then contact dnceng if you have further questions."
displayName: Warn about v2 Arcade Publishing Usage
- task: DownloadBuildArtifacts@0
displayName: Download Build Assets
continueOnError: true
inputs:
buildType: specific
buildVersionToDownload: specific
project: $(AzDOProjectName)
pipeline: $(AzDOPipelineId)
buildId: $(AzDOBuildId)
downloadType: 'specific'
itemPattern: |
PackageArtifacts/**
BlobArtifacts/**
AssetManifests/**
downloadPath: '$(Build.ArtifactStagingDirectory)'
checkDownloadedFiles: true
- task: NuGetToolInstaller@1
displayName: 'Install NuGet.exe'
# This is necessary whenever we want to publish/restore to an AzDO private feed
- task: NuGetAuthenticate@0
displayName: 'Authenticate to AzDO Feeds'
- task: PowerShell@2
displayName: Enable cross-org publishing
inputs:
filePath: eng\common\enable-cross-org-publishing.ps1
arguments: -token $(dn-bot-dnceng-artifact-feeds-rw)
- task: PowerShell@2
displayName: Publish Assets
inputs:
filePath: eng\common\sdk-task.ps1
arguments: -task PublishArtifactsInManifest -restore -msbuildEngine dotnet
/p:PublishingInfraVersion=2
/p:ArtifactsCategory=$(ArtifactsCategory)
/p:IsStableBuild=$(IsStableBuild)
/p:IsInternalBuild=$(IsInternalBuild)
/p:RepositoryName=$(Build.Repository.Name)
/p:CommitSha=$(Build.SourceVersion)
/p:NugetPath=$(NuGetExeToolPath)
/p:AzdoTargetFeedPAT='$(dn-bot-dnceng-universal-packages-rw)'
/p:AzureStorageTargetFeedPAT='$(dotnetfeed-storage-access-key-1)'
/p:BARBuildId=$(BARBuildId)
/p:MaestroApiEndpoint='$(MaestroApiEndPoint)'
/p:BuildAssetRegistryToken='$(MaestroApiAccessToken)'
/p:ManifestsBasePath='$(Build.ArtifactStagingDirectory)/AssetManifests/'
/p:BlobBasePath='$(Build.ArtifactStagingDirectory)/BlobArtifacts/'
/p:PackageBasePath='$(Build.ArtifactStagingDirectory)/PackageArtifacts/'
/p:Configuration=Release
/p:PublishInstallersAndChecksums=${{ parameters.publishInstallersAndChecksums }}
/p:InstallersTargetStaticFeed=$(InstallersBlobFeedUrl)
/p:InstallersAzureAccountKey=$(dotnetcli-storage-key)
/p:ChecksumsTargetStaticFeed=$(ChecksumsBlobFeedUrl)
/p:ChecksumsAzureAccountKey=$(dotnetclichecksums-storage-key)
/p:AzureDevOpsStaticShippingFeed='${{ parameters.shippingFeed }}'
/p:AzureDevOpsStaticShippingFeedKey='$(dn-bot-dnceng-artifact-feeds-rw)'
/p:AzureDevOpsStaticTransportFeed='${{ parameters.transportFeed }}'
/p:AzureDevOpsStaticTransportFeedKey='$(dn-bot-dnceng-artifact-feeds-rw)'
/p:AzureDevOpsStaticSymbolsFeed='${{ parameters.symbolsFeed }}'
/p:AzureDevOpsStaticSymbolsFeedKey='$(dn-bot-dnceng-artifact-feeds-rw)'
/p:LatestLinkShortUrlPrefix=dotnet/'${{ parameters.akaMSChannelName }}'
/p:AkaMSClientId=$(akams-client-id)
/p:AkaMSClientSecret=$(akams-client-secret)
${{ parameters.artifactsPublishingAdditionalParameters }}
- template: ../../steps/publish-logs.yml
parameters:
StageLabel: '${{ parameters.stageName }}'
JobLabel: 'AssetsPublishing'
- template: ../../steps/add-build-to-channel.yml
parameters:
ChannelId: ${{ parameters.channelId }}

13
eng/common/templates/post-build/trigger-subscription.yml

@ -1,13 +0,0 @@
parameters:
ChannelId: 0
steps:
- task: PowerShell@2
displayName: Triggering subscriptions
inputs:
filePath: $(Build.SourcesDirectory)/eng/common/post-build/trigger-subscriptions.ps1
arguments: -SourceRepo $(Build.Repository.Uri)
-ChannelId ${{ parameters.ChannelId }}
-MaestroApiAccessToken $(MaestroAccessToken)
-MaestroApiEndPoint $(MaestroApiEndPoint)
-MaestroApiVersion $(MaestroApiVersion)

13
eng/common/templates/steps/add-build-to-channel.yml

@ -1,13 +0,0 @@
parameters:
ChannelId: 0
steps:
- task: PowerShell@2
displayName: Add Build to Channel
inputs:
filePath: $(Build.SourcesDirectory)/eng/common/post-build/add-build-to-channel.ps1
arguments: -BuildId $(BARBuildId)
-ChannelId ${{ parameters.ChannelId }}
-MaestroApiAccessToken $(MaestroApiAccessToken)
-MaestroApiEndPoint $(MaestroApiEndPoint)
-MaestroApiVersion $(MaestroApiVersion)

12
eng/common/templates/steps/build-reason.yml

@ -1,12 +0,0 @@
# build-reason.yml
# Description: runs steps if build.reason condition is valid. conditions is a string of valid build reasons
# to include steps (',' separated).
parameters:
conditions: ''
steps: []
steps:
- ${{ if and( not(startsWith(parameters.conditions, 'not')), contains(parameters.conditions, variables['build.reason'])) }}:
- ${{ parameters.steps }}
- ${{ if and( startsWith(parameters.conditions, 'not'), not(contains(parameters.conditions, variables['build.reason']))) }}:
- ${{ parameters.steps }}

32
eng/common/templates/steps/execute-codeql.yml

@ -1,32 +0,0 @@
parameters:
# Language that should be analyzed. Defaults to csharp
language: csharp
# Build Commands
buildCommands: ''
overrideParameters: '' # Optional: to override values for parameters.
additionalParameters: '' # Optional: parameters that need user specific values eg: '-SourceToolsList @("abc","def") -ArtifactToolsList @("ghi","jkl")'
# Optional: if specified, restore and use this version of Guardian instead of the default.
overrideGuardianVersion: ''
# Optional: if true, publish the '.gdn' folder as a pipeline artifact. This can help with in-depth
# diagnosis of problems with specific tool configurations.
publishGuardianDirectoryToPipeline: false
# The script to run to execute all SDL tools. Use this if you want to use a script to define SDL
# parameters rather than relying on YAML. It may be better to use a local script, because you can
# reproduce results locally without piecing together a command based on the YAML.
executeAllSdlToolsScript: 'eng/common/sdl/execute-all-sdl-tools.ps1'
# There is some sort of bug (has been reported) in Azure DevOps where if this parameter is named
# 'continueOnError', the parameter value is not correctly picked up.
# This can also be remedied by the caller (post-build.yml) if it does not use a nested parameter
# optional: determines whether to continue the build if the step errors;
sdlContinueOnError: false
steps:
- template: /eng/common/templates/steps/execute-sdl.yml
parameters:
overrideGuardianVersion: ${{ parameters.overrideGuardianVersion }}
executeAllSdlToolsScript: ${{ parameters.executeAllSdlToolsScript }}
overrideParameters: ${{ parameters.overrideParameters }}
additionalParameters: '${{ parameters.additionalParameters }}
-CodeQLAdditionalRunConfigParams @("BuildCommands < ${{ parameters.buildCommands }}", "Language < ${{ parameters.language }}")'
publishGuardianDirectoryToPipeline: ${{ parameters.publishGuardianDirectoryToPipeline }}
sdlContinueOnError: ${{ parameters.sdlContinueOnError }}

88
eng/common/templates/steps/execute-sdl.yml

@ -1,88 +0,0 @@
parameters:
overrideGuardianVersion: ''
executeAllSdlToolsScript: ''
overrideParameters: ''
additionalParameters: ''
publishGuardianDirectoryToPipeline: false
sdlContinueOnError: false
condition: ''
steps:
- task: NuGetAuthenticate@1
inputs:
nuGetServiceConnections: GuardianConnect
- task: NuGetToolInstaller@1
displayName: 'Install NuGet.exe'
- ${{ if ne(parameters.overrideGuardianVersion, '') }}:
- pwsh: |
Set-Location -Path $(Build.SourcesDirectory)\eng\common\sdl
. .\sdl.ps1
$guardianCliLocation = Install-Gdn -Path $(Build.SourcesDirectory)\.artifacts -Version ${{ parameters.overrideGuardianVersion }}
Write-Host "##vso[task.setvariable variable=GuardianCliLocation]$guardianCliLocation"
displayName: Install Guardian (Overridden)
- ${{ if eq(parameters.overrideGuardianVersion, '') }}:
- pwsh: |
Set-Location -Path $(Build.SourcesDirectory)\eng\common\sdl
. .\sdl.ps1
$guardianCliLocation = Install-Gdn -Path $(Build.SourcesDirectory)\.artifacts
Write-Host "##vso[task.setvariable variable=GuardianCliLocation]$guardianCliLocation"
displayName: Install Guardian
- ${{ if ne(parameters.overrideParameters, '') }}:
- powershell: ${{ parameters.executeAllSdlToolsScript }} ${{ parameters.overrideParameters }}
displayName: Execute SDL (Overridden)
continueOnError: ${{ parameters.sdlContinueOnError }}
condition: ${{ parameters.condition }}
- ${{ if eq(parameters.overrideParameters, '') }}:
- powershell: ${{ parameters.executeAllSdlToolsScript }}
-GuardianCliLocation $(GuardianCliLocation)
-NugetPackageDirectory $(Build.SourcesDirectory)\.packages
-AzureDevOpsAccessToken $(dn-bot-dotnet-build-rw-code-rw)
${{ parameters.additionalParameters }}
displayName: Execute SDL
continueOnError: ${{ parameters.sdlContinueOnError }}
condition: ${{ parameters.condition }}
- ${{ if ne(parameters.publishGuardianDirectoryToPipeline, 'false') }}:
# We want to publish the Guardian results and configuration for easy diagnosis. However, the
# '.gdn' dir is a mix of configuration, results, extracted dependencies, and Guardian default
# tooling files. Some of these files are large and aren't useful during an investigation, so
# exclude them by simply deleting them before publishing. (As of writing, there is no documented
# way to selectively exclude a dir from the pipeline artifact publish task.)
- task: DeleteFiles@1
displayName: Delete Guardian dependencies to avoid uploading
inputs:
SourceFolder: $(Agent.BuildDirectory)/.gdn
Contents: |
c
i
condition: succeededOrFailed()
- publish: $(Agent.BuildDirectory)/.gdn
artifact: GuardianConfiguration
displayName: Publish GuardianConfiguration
condition: succeededOrFailed()
# Publish the SARIF files in a container named CodeAnalysisLogs to enable integration
# with the "SARIF SAST Scans Tab" Azure DevOps extension
- task: CopyFiles@2
displayName: Copy SARIF files
inputs:
flattenFolders: true
sourceFolder: $(Agent.BuildDirectory)/.gdn/rc/
contents: '**/*.sarif'
targetFolder: $(Build.SourcesDirectory)/CodeAnalysisLogs
condition: succeededOrFailed()
# Use PublishBuildArtifacts because the SARIF extension only checks this case
# see microsoft/sarif-azuredevops-extension#4
- task: PublishBuildArtifacts@1
displayName: Publish SARIF files to CodeAnalysisLogs container
inputs:
pathToPublish: $(Build.SourcesDirectory)/CodeAnalysisLogs
artifactName: CodeAnalysisLogs
condition: succeededOrFailed()

50
eng/common/templates/steps/perf-send-to-helix.yml

@ -1,50 +0,0 @@
# Please remember to update the documentation if you make changes to these parameters!
parameters:
ProjectFile: '' # required -- project file that specifies the helix workitems
HelixSource: 'pr/default' # required -- sources must start with pr/, official/, prodcon/, or agent/
HelixType: 'tests/default/' # required -- Helix telemetry which identifies what type of data this is; should include "test" for clarity and must end in '/'
HelixBuild: $(Build.BuildNumber) # required -- the build number Helix will use to identify this -- automatically set to the AzDO build number
HelixTargetQueues: '' # required -- semicolon delimited list of Helix queues to test on; see https://helix.dot.net/ for a list of queues
HelixAccessToken: '' # required -- access token to make Helix API requests; should be provided by the appropriate variable group
HelixPreCommands: '' # optional -- commands to run before Helix work item execution
HelixPostCommands: '' # optional -- commands to run after Helix work item execution
WorkItemDirectory: '' # optional -- a payload directory to zip up and send to Helix; requires WorkItemCommand; incompatible with XUnitProjects
CorrelationPayloadDirectory: '' # optional -- a directory to zip up and send to Helix as a correlation payload
IncludeDotNetCli: false # optional -- true will download a version of the .NET CLI onto the Helix machine as a correlation payload; requires DotNetCliPackageType and DotNetCliVersion
DotNetCliPackageType: '' # optional -- either 'sdk', 'runtime' or 'aspnetcore-runtime'; determines whether the sdk or runtime will be sent to Helix; see https://raw.githubusercontent.com/dotnet/core/master/release-notes/releases.json
DotNetCliVersion: '' # optional -- version of the CLI to send to Helix; based on this: https://raw.githubusercontent.com/dotnet/core/master/release-notes/releases.json
EnableXUnitReporter: false # optional -- true enables XUnit result reporting to Mission Control
WaitForWorkItemCompletion: true # optional -- true will make the task wait until work items have been completed and fail the build if work items fail. False is "fire and forget."
Creator: '' # optional -- if the build is external, use this to specify who is sending the job
DisplayNamePrefix: 'Send job to Helix' # optional -- rename the beginning of the displayName of the steps in AzDO
condition: succeeded() # optional -- condition for step to execute; defaults to succeeded()
continueOnError: false # optional -- determines whether to continue the build if the step errors; defaults to false
osGroup: '' # required -- operating system for the job
steps:
- template: /eng/pipelines/common/templates/runtimes/send-to-helix-inner-step.yml
parameters:
osGroup: ${{ parameters.osGroup }}
sendParams: $(Build.SourcesDirectory)/eng/common/performance/${{ parameters.ProjectFile }} /restore /t:Test /bl:$(Build.SourcesDirectory)/artifacts/log/$(_BuildConfig)/SendToHelix.binlog
displayName: ${{ parameters.DisplayNamePrefix }}
condition: ${{ parameters.condition }}
continueOnError: ${{ parameters.continueOnError }}
environment:
BuildConfig: $(_BuildConfig)
HelixSource: ${{ parameters.HelixSource }}
HelixType: ${{ parameters.HelixType }}
HelixBuild: ${{ parameters.HelixBuild }}
HelixTargetQueues: ${{ parameters.HelixTargetQueues }}
HelixAccessToken: ${{ parameters.HelixAccessToken }}
HelixPreCommands: ${{ parameters.HelixPreCommands }}
HelixPostCommands: ${{ parameters.HelixPostCommands }}
WorkItemDirectory: ${{ parameters.WorkItemDirectory }}
CorrelationPayloadDirectory: ${{ parameters.CorrelationPayloadDirectory }}
IncludeDotNetCli: ${{ parameters.IncludeDotNetCli }}
DotNetCliPackageType: ${{ parameters.DotNetCliPackageType }}
DotNetCliVersion: ${{ parameters.DotNetCliVersion }}
EnableXUnitReporter: ${{ parameters.EnableXUnitReporter }}
WaitForWorkItemCompletion: ${{ parameters.WaitForWorkItemCompletion }}
Creator: ${{ parameters.Creator }}
SYSTEM_ACCESSTOKEN: $(System.AccessToken)

7
eng/common/templates/steps/run-on-unix.yml

@ -1,7 +0,0 @@
parameters:
agentOs: ''
steps: []
steps:
- ${{ if ne(parameters.agentOs, 'Windows_NT') }}:
- ${{ parameters.steps }}

7
eng/common/templates/steps/run-on-windows.yml

@ -1,7 +0,0 @@
parameters:
agentOs: ''
steps: []
steps:
- ${{ if eq(parameters.agentOs, 'Windows_NT') }}:
- ${{ parameters.steps }}

33
eng/common/templates/steps/run-script-ifequalelse.yml

@ -1,33 +0,0 @@
parameters:
# if parameter1 equals parameter 2, run 'ifScript' command, else run 'elsescript' command
parameter1: ''
parameter2: ''
ifScript: ''
elseScript: ''
# name of script step
name: Script
# display name of script step
displayName: If-Equal-Else Script
# environment
env: {}
# conditional expression for step execution
condition: ''
steps:
- ${{ if and(ne(parameters.ifScript, ''), eq(parameters.parameter1, parameters.parameter2)) }}:
- script: ${{ parameters.ifScript }}
name: ${{ parameters.name }}
displayName: ${{ parameters.displayName }}
env: ${{ parameters.env }}
condition: ${{ parameters.condition }}
- ${{ if and(ne(parameters.elseScript, ''), ne(parameters.parameter1, parameters.parameter2)) }}:
- script: ${{ parameters.elseScript }}
name: ${{ parameters.name }}
displayName: ${{ parameters.displayName }}
env: ${{ parameters.env }}
condition: ${{ parameters.condition }}

102
eng/common/templates/steps/telemetry-end.yml

@ -1,102 +0,0 @@
parameters:
maxRetries: 5
retryDelay: 10 # in seconds
steps:
- bash: |
if [ "$AGENT_JOBSTATUS" = "Succeeded" ] || [ "$AGENT_JOBSTATUS" = "PartiallySucceeded" ]; then
errorCount=0
else
errorCount=1
fi
warningCount=0
curlStatus=1
retryCount=0
# retry loop to harden against spotty telemetry connections
# we don't retry successes and 4xx client errors
until [[ $curlStatus -eq 0 || ( $curlStatus -ge 400 && $curlStatus -le 499 ) || $retryCount -ge $MaxRetries ]]
do
if [ $retryCount -gt 0 ]; then
echo "Failed to send telemetry to Helix; waiting $RetryDelay seconds before retrying..."
sleep $RetryDelay
fi
# create a temporary file for curl output
res=`mktemp`
curlResult=`
curl --verbose --output $res --write-out "%{http_code}"\
-H 'Content-Type: application/json' \
-H "X-Helix-Job-Token: $Helix_JobToken" \
-H 'Content-Length: 0' \
-X POST -G "https://helix.dot.net/api/2018-03-14/telemetry/job/build/$Helix_WorkItemId/finish" \
--data-urlencode "errorCount=$errorCount" \
--data-urlencode "warningCount=$warningCount"`
curlStatus=$?
if [ $curlStatus -eq 0 ]; then
if [ $curlResult -gt 299 ] || [ $curlResult -lt 200 ]; then
curlStatus=$curlResult
fi
fi
let retryCount++
done
if [ $curlStatus -ne 0 ]; then
echo "Failed to Send Build Finish information after $retryCount retries"
vstsLogOutput="vso[task.logissue type=error;sourcepath=templates/steps/telemetry-end.yml;code=1;]Failed to Send Build Finish information: $curlStatus"
echo "##$vstsLogOutput"
exit 1
fi
displayName: Send Unix Build End Telemetry
env:
# defined via VSTS variables in start-job.sh
Helix_JobToken: $(Helix_JobToken)
Helix_WorkItemId: $(Helix_WorkItemId)
MaxRetries: ${{ parameters.maxRetries }}
RetryDelay: ${{ parameters.retryDelay }}
condition: and(always(), ne(variables['Agent.Os'], 'Windows_NT'))
- powershell: |
if (($env:Agent_JobStatus -eq 'Succeeded') -or ($env:Agent_JobStatus -eq 'PartiallySucceeded')) {
$ErrorCount = 0
} else {
$ErrorCount = 1
}
$WarningCount = 0
# Basic retry loop to harden against server flakiness
$retryCount = 0
while ($retryCount -lt $env:MaxRetries) {
try {
Invoke-RestMethod -Uri "https://helix.dot.net/api/2018-03-14/telemetry/job/build/$env:Helix_WorkItemId/finish?errorCount=$ErrorCount&warningCount=$WarningCount" -Method Post -ContentType "application/json" -Body "" `
-Headers @{ 'X-Helix-Job-Token'=$env:Helix_JobToken }
break
}
catch {
$statusCode = $_.Exception.Response.StatusCode.value__
if ($statusCode -ge 400 -and $statusCode -le 499) {
Write-Host "##vso[task.logissue]error Failed to send telemetry to Helix (status code $statusCode); not retrying (4xx client error)"
Write-Host "##vso[task.logissue]error ", $_.Exception.GetType().FullName, $_.Exception.Message
exit 1
}
Write-Host "Failed to send telemetry to Helix (status code $statusCode); waiting $env:RetryDelay seconds before retrying..."
$retryCount++
sleep $env:RetryDelay
continue
}
}
if ($retryCount -ge $env:MaxRetries) {
Write-Host "##vso[task.logissue]error Failed to send telemetry to Helix after $retryCount retries."
exit 1
}
displayName: Send Windows Build End Telemetry
env:
# defined via VSTS variables in start-job.ps1
Helix_JobToken: $(Helix_JobToken)
Helix_WorkItemId: $(Helix_WorkItemId)
MaxRetries: ${{ parameters.maxRetries }}
RetryDelay: ${{ parameters.retryDelay }}
condition: and(always(),eq(variables['Agent.Os'], 'Windows_NT'))

241
eng/common/templates/steps/telemetry-start.yml

@ -1,241 +0,0 @@
parameters:
helixSource: 'undefined_defaulted_in_telemetry.yml'
helixType: 'undefined_defaulted_in_telemetry.yml'
buildConfig: ''
runAsPublic: false
maxRetries: 5
retryDelay: 10 # in seconds
steps:
- ${{ if and(eq(parameters.runAsPublic, 'false'), not(eq(variables['System.TeamProject'], 'public'))) }}:
- task: AzureKeyVault@1
inputs:
azureSubscription: 'HelixProd_KeyVault'
KeyVaultName: HelixProdKV
SecretsFilter: 'HelixApiAccessToken'
condition: always()
- bash: |
# create a temporary file
jobInfo=`mktemp`
# write job info content to temporary file
cat > $jobInfo <<JobListStuff
{
"QueueId": "$QueueId",
"Source": "$Source",
"Type": "$Type",
"Build": "$Build",
"Attempt": "$Attempt",
"Properties": {
"operatingSystem": "$OperatingSystem",
"configuration": "$Configuration"
}
}
JobListStuff
cat $jobInfo
# create a temporary file for curl output
res=`mktemp`
accessTokenParameter="?access_token=$HelixApiAccessToken"
curlStatus=1
retryCount=0
# retry loop to harden against spotty telemetry connections
# we don't retry successes and 4xx client errors
until [[ $curlStatus -eq 0 || ( $curlStatus -ge 400 && $curlStatus -le 499 ) || $retryCount -ge $MaxRetries ]]
do
if [ $retryCount -gt 0 ]; then
echo "Failed to send telemetry to Helix; waiting $RetryDelay seconds before retrying..."
sleep $RetryDelay
fi
curlResult=`
cat $jobInfo |\
curl --trace - --verbose --output $res --write-out "%{http_code}" \
-H 'Content-Type: application/json' \
-X POST "https://helix.dot.net/api/2018-03-14/telemetry/job$accessTokenParameter" -d @-`
curlStatus=$?
if [ $curlStatus -eq 0 ]; then
if [ $curlResult -gt 299 ] || [ $curlResult -lt 200 ]; then
curlStatus=$curlResult
fi
fi
let retryCount++
done
curlResult=`cat $res`
# validate status of curl command
if [ $curlStatus -ne 0 ]; then
echo "Failed To Send Job Start information after $retryCount retries"
# We have to append the ## vso prefix or vso will pick up the command when it dumps the inline script into the shell
vstsLogOutput="vso[task.logissue type=error;sourcepath=telemetry/start-job.sh;code=1;]Failed to Send Job Start information: $curlStatus"
echo "##$vstsLogOutput"
exit 1
fi
# Set the Helix_JobToken variable
export Helix_JobToken=`echo $curlResult | xargs echo` # Strip Quotes
echo "##vso[task.setvariable variable=Helix_JobToken;issecret=true;]$Helix_JobToken"
displayName: Send Unix Job Start Telemetry
env:
HelixApiAccessToken: $(HelixApiAccessToken)
Source: ${{ parameters.helixSource }}
Type: ${{ parameters.helixType }}
Build: $(Build.BuildNumber)
QueueId: $(Agent.Os)
Attempt: 1
OperatingSystem: $(Agent.Os)
Configuration: ${{ parameters.buildConfig }}
MaxRetries: ${{ parameters.maxRetries }}
RetryDelay: ${{ parameters.retryDelay }}
condition: and(always(), ne(variables['Agent.Os'], 'Windows_NT'))
- bash: |
curlStatus=1
retryCount=0
# retry loop to harden against spotty telemetry connections
# we don't retry successes and 4xx client errors
until [[ $curlStatus -eq 0 || ( $curlStatus -ge 400 && $curlStatus -le 499 ) || $retryCount -ge $MaxRetries ]]
do
if [ $retryCount -gt 0 ]; then
echo "Failed to send telemetry to Helix; waiting $RetryDelay seconds before retrying..."
sleep $RetryDelay
fi
res=`mktemp`
curlResult=`
curl --verbose --output $res --write-out "%{http_code}"\
-H 'Content-Type: application/json' \
-H "X-Helix-Job-Token: $Helix_JobToken" \
-H 'Content-Length: 0' \
-X POST -G "https://helix.dot.net/api/2018-03-14/telemetry/job/build" \
--data-urlencode "buildUri=$BuildUri"`
curlStatus=$?
if [ $curlStatus -eq 0 ]; then
if [ $curlResult -gt 299 ] || [ $curlResult -lt 200 ]; then
curlStatus=$curlResult
fi
fi
curlResult=`cat $res`
let retryCount++
done
# validate status of curl command
if [ $curlStatus -ne 0 ]; then
echo "Failed to Send Build Start information after $retryCount retries"
vstsLogOutput="vso[task.logissue type=error;sourcepath=telemetry/build/start.sh;code=1;]Failed to Send Build Start information: $curlStatus"
echo "##$vstsLogOutput"
exit 1
fi
export Helix_WorkItemId=`echo $curlResult | xargs echo` # Strip Quotes
echo "##vso[task.setvariable variable=Helix_WorkItemId]$Helix_WorkItemId"
displayName: Send Unix Build Start Telemetry
env:
BuildUri: $(System.TaskDefinitionsUri)$(System.TeamProject)/_build/index?buildId=$(Build.BuildId)&_a=summary
Helix_JobToken: $(Helix_JobToken)
MaxRetries: ${{ parameters.maxRetries }}
RetryDelay: ${{ parameters.retryDelay }}
condition: and(always(), ne(variables['Agent.Os'], 'Windows_NT'))
- powershell: |
$jobInfo = [pscustomobject]@{
QueueId=$env:QueueId;
Source=$env:Source;
Type=$env:Type;
Build=$env:Build;
Attempt=$env:Attempt;
Properties=[pscustomobject]@{ operatingSystem=$env:OperatingSystem; configuration=$env:Configuration };
}
$jobInfoJson = $jobInfo | ConvertTo-Json
if ($env:HelixApiAccessToken) {
$accessTokenParameter="?access_token=$($env:HelixApiAccessToken)"
}
Write-Host "Job Info: $jobInfoJson"
# Basic retry loop to harden against server flakiness
$retryCount = 0
while ($retryCount -lt $env:MaxRetries) {
try {
$jobToken = Invoke-RestMethod -Uri "https://helix.dot.net/api/2018-03-14/telemetry/job$($accessTokenParameter)" -Method Post -ContentType "application/json" -Body $jobInfoJson
break
}
catch {
$statusCode = $_.Exception.Response.StatusCode.value__
if ($statusCode -ge 400 -and $statusCode -le 499) {
Write-Host "##vso[task.logissue]error Failed to send telemetry to Helix (status code $statusCode); not retrying (4xx client error)"
Write-Host "##vso[task.logissue]error ", $_.Exception.GetType().FullName, $_.Exception.Message
exit 1
}
Write-Host "Failed to send telemetry to Helix (status code $statusCode); waiting $env:RetryDelay seconds before retrying..."
$retryCount++
sleep $env:RetryDelay
continue
}
}
if ($retryCount -ge $env:MaxRetries) {
Write-Host "##vso[task.logissue]error Failed to send telemetry to Helix after $retryCount retries."
exit 1
}
$env:Helix_JobToken = $jobToken
Write-Host "##vso[task.setvariable variable=Helix_JobToken;issecret=true;]$env:Helix_JobToken"
env:
HelixApiAccessToken: $(HelixApiAccessToken)
Source: ${{ parameters.helixSource }}
Type: ${{ parameters.helixType }}
Build: $(Build.BuildNumber)
QueueId: $(Agent.Os)
Attempt: 1
OperatingSystem: $(Agent.Os)
Configuration: ${{ parameters.buildConfig }}
MaxRetries: ${{ parameters.maxRetries }}
RetryDelay: ${{ parameters.retryDelay }}
condition: and(always(), eq(variables['Agent.Os'], 'Windows_NT'))
displayName: Send Windows Job Start Telemetry
- powershell: |
# Basic retry loop to harden against server flakiness
$retryCount = 0
while ($retryCount -lt $env:MaxRetries) {
try {
$workItemId = Invoke-RestMethod -Uri "https://helix.dot.net/api/2018-03-14/telemetry/job/build?buildUri=$([Net.WebUtility]::UrlEncode($env:BuildUri))" -Method Post -ContentType "application/json" -Body "" `
-Headers @{ 'X-Helix-Job-Token'=$env:Helix_JobToken }
break
}
catch {
$statusCode = $_.Exception.Response.StatusCode.value__
if ($statusCode -ge 400 -and $statusCode -le 499) {
Write-Host "##vso[task.logissue]error Failed to send telemetry to Helix (status code $statusCode); not retrying (4xx client error)"
Write-Host "##vso[task.logissue]error ", $_.Exception.GetType().FullName, $_.Exception.Message
exit 1
}
Write-Host "Failed to send telemetry to Helix (status code $statusCode); waiting $env:RetryDelay seconds before retrying..."
$retryCount++
sleep $env:RetryDelay
continue
}
}
if ($retryCount -ge $env:MaxRetries) {
Write-Host "##vso[task.logissue]error Failed to send telemetry to Helix after $retryCount retries."
exit 1
}
$env:Helix_WorkItemId = $workItemId
Write-Host "##vso[task.setvariable variable=Helix_WorkItemId]$env:Helix_WorkItemId"
displayName: Send Windows Build Start Telemetry
env:
BuildUri: $(System.TaskDefinitionsUri)$(System.TeamProject)/_build/index?buildId=$(Build.BuildId)&_a=summary
Helix_JobToken: $(Helix_JobToken)
MaxRetries: ${{ parameters.maxRetries }}
RetryDelay: ${{ parameters.retryDelay }}
condition: and(always(), eq(variables['Agent.Os'], 'Windows_NT'))

2
eng/common/templates/variables/pool-providers.yml

@ -23,7 +23,7 @@
#
# pool:
# name: $(DncEngInternalBuildPool)
# demands: ImageOverride -equals windows.vs2019.amd64
# demands: ImageOverride -equals windows.vs2022.amd64
variables:
- ${{ if eq(variables['System.TeamProject'], 'internal') }}:
- template: /eng/common/templates-official/variables/pool-providers.yml

Some files were not shown because too many files changed in this diff

Loading…
Cancel
Save