From 02ddfad24582fc8ad439071a5551ff96e3bbb8fe Mon Sep 17 00:00:00 2001 From: Max Katz Date: Fri, 22 Dec 2023 01:44:07 -0800 Subject: [PATCH] Implement helper OutputApiDiff target (#13818) --- .gitignore | 1 + .nuke/build.schema.json | 2 + Avalonia.sln | 13 ++ ...{ApiDiffValidation.cs => ApiDiffHelper.cs} | 150 +++++++++++++++--- nukebuild/Build.cs | 16 +- nukebuild/_build.csproj | 3 + 6 files changed, 160 insertions(+), 25 deletions(-) rename nukebuild/{ApiDiffValidation.cs => ApiDiffHelper.cs} (54%) diff --git a/.gitignore b/.gitignore index ee778ed4e2..ddbe436d3a 100644 --- a/.gitignore +++ b/.gitignore @@ -217,3 +217,4 @@ node_modules src/Browser/Avalonia.Browser.Blazor/webapp/package-lock.json src/Browser/Avalonia.Browser.Blazor/wwwroot src/Browser/Avalonia.Browser/wwwroot +api/diff diff --git a/.nuke/build.schema.json b/.nuke/build.schema.json index b802589fc7..a21636409b 100644 --- a/.nuke/build.schema.json +++ b/.nuke/build.schema.json @@ -83,6 +83,7 @@ "CreateIntermediateNugetPackages", "CreateNugetPackages", "GenerateCppHeaders", + "OutputApiDiff", "Package", "RunCoreLibsTests", "RunHtmlPreviewerTests", @@ -117,6 +118,7 @@ "CreateIntermediateNugetPackages", "CreateNugetPackages", "GenerateCppHeaders", + "OutputApiDiff", "Package", "RunCoreLibsTests", "RunHtmlPreviewerTests", diff --git a/Avalonia.sln b/Avalonia.sln index 22ac15e6ee..9c78c109a5 100644 --- a/Avalonia.sln +++ b/Avalonia.sln @@ -236,6 +236,19 @@ EndProject Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Solution Items", "Solution Items", "{176582E8-46AF-416A-85C1-13A5C6744497}" ProjectSection(SolutionItems) = preProject .editorconfig = .editorconfig + azure-pipelines.yml = azure-pipelines.yml + azure-pipelines-integrationtests.yml = azure-pipelines-integrationtests.yml + CODE_OF_CONDUCT.md = CODE_OF_CONDUCT.md + CONTRIBUTING.md = CONTRIBUTING.md + Directory.Build.props = Directory.Build.props + Directory.Build.targets = Directory.Build.targets + dirs.proj = dirs.proj + global.json = global.json + licence.md = licence.md + NOTICE.md = NOTICE.md + NuGet.Config = NuGet.Config + readme.md = readme.md + Settings.StyleCop = Settings.StyleCop EndProjectSection EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Avalonia.Controls.ItemsRepeater", "src\Avalonia.Controls.ItemsRepeater\Avalonia.Controls.ItemsRepeater.csproj", "{EE0F0DD4-A70D-472B-BD5D-B7D32D0E9386}" diff --git a/nukebuild/ApiDiffValidation.cs b/nukebuild/ApiDiffHelper.cs similarity index 54% rename from nukebuild/ApiDiffValidation.cs rename to nukebuild/ApiDiffHelper.cs index 3c58215d26..946477d287 100644 --- a/nukebuild/ApiDiffValidation.cs +++ b/nukebuild/ApiDiffHelper.cs @@ -1,5 +1,6 @@ using System; using System.Collections.Generic; +using System.Diagnostics; using System.IO; using System.IO.Compression; using System.Linq; @@ -10,9 +11,95 @@ using System.Threading.Tasks; using Nuke.Common.Tooling; using static Serilog.Log; -public static class ApiDiffValidation +public static class ApiDiffHelper { - private static readonly HttpClient s_httpClient = new(); + static readonly HttpClient s_httpClient = new(); + + public static async Task GetDiff( + Tool apiDiffTool, string outputFolder, + string packagePath, string baselineVersion) + { + await using var baselineStream = await DownloadBaselinePackage(packagePath, baselineVersion); + if (baselineStream == null) + return; + + if (!Directory.Exists(outputFolder)) + { + Directory.CreateDirectory(outputFolder!); + } + + using (var target = new ZipArchive(File.Open(packagePath, FileMode.Open, FileAccess.Read), ZipArchiveMode.Read)) + using (var baseline = new ZipArchive(baselineStream, ZipArchiveMode.Read)) + using (Helpers.UseTempDir(out var tempFolder)) + { + var targetDlls = GetDlls(target); + var baselineDlls = GetDlls(baseline); + + var pairs = new List<(string baseline, string target)>(); + + var packageId = GetPackageId(packagePath); + + // Don't use Path.Combine with these left and right tool parameters. + // Microsoft.DotNet.ApiCompat.Tool is stupid and treats '/' and '\' as different assemblies in suppression files. + // So, always use Unix '/' + foreach (var baselineDll in baselineDlls) + { + var baselineDllPath = await ExtractDll("baseline", baselineDll, tempFolder); + + var targetTfm = baselineDll.target; + if (s_tfmRedirects.FirstOrDefault(t => baselineDll.target.StartsWith(t.oldTfm)).newTfm is {} newTfm) + { + targetTfm = newTfm; + } + + var targetDll = targetDlls.FirstOrDefault(e => + e.target.StartsWith(targetTfm) && e.entry.Name == baselineDll.entry.Name); + if (targetDll?.entry is null) + { + throw new InvalidOperationException($"Some assemblies are missing in the new package {packageId}: {baselineDll.entry.Name} for {baselineDll.target}"); + } + + var targetDllPath = await ExtractDll("target", targetDll, tempFolder); + + pairs.Add((baselineDllPath, targetDllPath)); + } + + await Task.WhenAll(pairs.Select(p => Task.Run(() => + { + var baselineApi = p.baseline + ".api.cs"; + var targetApi = p.target + ".api.cs"; + var resultDiff = p.target + ".api.diff.cs"; + + GenerateApiListing(apiDiffTool, p.baseline, baselineApi, tempFolder); + GenerateApiListing(apiDiffTool, p.target, targetApi, tempFolder); + + var args = $"""-c core.autocrlf=false diff --no-index --minimal """; + args += """--ignore-matching-lines="^\[assembly: System.Reflection.AssemblyVersionAttribute" """; + args += $""" --output {resultDiff} {baselineApi} {targetApi}"""; + + using (var gitProcess = new Process()) + { + gitProcess.StartInfo = new ProcessStartInfo + { + CreateNoWindow = true, + RedirectStandardError = false, + RedirectStandardOutput = false, + FileName = "git", + Arguments = args, + WorkingDirectory = tempFolder + }; + gitProcess.Start(); + gitProcess.WaitForExit(); + } + + var resultFile = new FileInfo(Path.Combine(tempFolder, resultDiff)); + if (resultFile.Length > 0) + { + resultFile.CopyTo(Path.Combine(outputFolder, Path.GetFileName(resultDiff)), true); + } + }))); + } + } private static readonly (string oldTfm, string newTfm)[] s_tfmRedirects = new[] { @@ -25,12 +112,6 @@ public static class ApiDiffValidation Tool apiCompatTool, string packagePath, string baselineVersion, string suppressionFilesFolder, bool updateSuppressionFile) { - if (baselineVersion is null) - { - throw new InvalidOperationException( - "Build \"api-baseline\" parameter must be set when running Nuke CreatePackages"); - } - if (!Directory.Exists(suppressionFilesFolder)) { Directory.CreateDirectory(suppressionFilesFolder!); @@ -58,13 +139,7 @@ public static class ApiDiffValidation // So, always use Unix '/' foreach (var baselineDll in baselineDlls) { - var baselineDllPath = $"baseline/{baselineDll.target}/{baselineDll.entry.Name}"; - var baselineDllRealPath = Path.Combine(tempFolder, baselineDllPath); - Directory.CreateDirectory(Path.GetDirectoryName(baselineDllRealPath)!); - await using (var baselineDllFile = File.Create(baselineDllRealPath)) - { - await baselineDll.entry.Open().CopyToAsync(baselineDllFile); - } + var baselineDllPath = await ExtractDll("baseline", baselineDll, tempFolder); var targetTfm = baselineDll.target; if (s_tfmRedirects.FirstOrDefault(t => baselineDll.target.StartsWith(t.oldTfm)).newTfm is {} newTfm) @@ -79,13 +154,7 @@ public static class ApiDiffValidation throw new InvalidOperationException($"Some assemblies are missing in the new package {packageId}: {baselineDll.entry.Name} for {baselineDll.target}"); } - var targetDllPath = $"target/{targetDll.target}/{targetDll.entry.Name}"; - var targetDllRealPath = Path.Combine(tempFolder, targetDllPath); - Directory.CreateDirectory(Path.GetDirectoryName(targetDllRealPath)!); - await using (var targetDllFile = File.Create(targetDllRealPath)) - { - await targetDll.entry.Open().CopyToAsync(targetDllFile); - } + var targetDllPath = await ExtractDll("target", targetDll, tempFolder); left.Add(baselineDllPath); right.Add(targetDllPath); @@ -116,7 +185,9 @@ public static class ApiDiffValidation } } - private static IReadOnlyCollection<(string target, ZipArchiveEntry entry)> GetDlls(ZipArchive archive) + record DllEntry(string target, ZipArchiveEntry entry); + + static IReadOnlyCollection GetDlls(ZipArchive archive) { return archive.Entries .Where(e => Path.GetExtension(e.FullName) == ".dll" @@ -130,12 +201,18 @@ public static class ApiDiffValidation ) .GroupBy(e => (e.target, e.entry.Name)) .Select(g => g.MaxBy(e => e.isRef)) - .Select(e => (e.target, e.entry)) + .Select(e => new DllEntry(e.target, e.entry)) .ToArray(); } static async Task DownloadBaselinePackage(string packagePath, string baselineVersion) { + if (baselineVersion is null) + { + throw new InvalidOperationException( + "Build \"api-baseline\" parameter must be set when running Nuke CreatePackages"); + } + /* Gets package name from versions like: Avalonia.0.10.0-preview1 @@ -167,6 +244,31 @@ public static class ApiDiffValidation } } + static async Task ExtractDll(string basePath, DllEntry dllEntry, string targetFolder) + { + var dllPath = $"{basePath}/{dllEntry.target}/{dllEntry.entry.Name}"; + var dllRealPath = Path.Combine(targetFolder, dllPath); + Directory.CreateDirectory(Path.GetDirectoryName(dllRealPath)!); + await using (var dllFile = File.Create(dllRealPath)) + { + await dllEntry.entry.Open().CopyToAsync(dllFile); + } + + return dllPath; + } + + static void GenerateApiListing(Tool apiDiffTool, string inputFile, string outputFile, string workingDif) + { + var args = $""" --assembly={inputFile} --output-path={outputFile} --include-assembly-attributes=true"""; + var result = apiDiffTool(args, workingDif) + .Where(t => t.Type == OutputType.Err).ToArray(); + if (result.Any()) + { + throw new AggregateException($"GetApi tool failed task has failed", + result.Select(r => new Exception(r.Text))); + } + } + static string GetPackageId(string packagePath) { return Regex.Replace( diff --git a/nukebuild/Build.cs b/nukebuild/Build.cs index 8b5cc4292a..871af047cd 100644 --- a/nukebuild/Build.cs +++ b/nukebuild/Build.cs @@ -18,6 +18,7 @@ using static Nuke.Common.Tools.Xunit.XunitTasks; using static Nuke.Common.Tools.VSWhere.VSWhereTasks; using static Serilog.Log; using MicroCom.CodeGenerator; +using Nuke.Common.IO; /* Before editing this file, install support plugin for your IDE, @@ -33,6 +34,9 @@ partial class Build : NukeBuild [PackageExecutable("Microsoft.DotNet.ApiCompat.Tool", "Microsoft.DotNet.ApiCompat.Tool.dll", Framework = "net6.0")] Tool ApiCompatTool; + + [PackageExecutable("Microsoft.DotNet.GenAPI.Tool", "Microsoft.DotNet.GenAPI.Tool.dll", Framework = "net8.0")] + Tool ApiGenTool; protected override void OnBuildInitialized() { @@ -283,11 +287,21 @@ partial class Build : NukeBuild .Executes(async () => { await Task.WhenAll( - Directory.GetFiles(Parameters.NugetRoot, "*.nupkg").Select(nugetPackage => ApiDiffValidation.ValidatePackage( + Directory.GetFiles(Parameters.NugetRoot, "*.nupkg").Select(nugetPackage => ApiDiffHelper.ValidatePackage( ApiCompatTool, nugetPackage, Parameters.ApiValidationBaseline, Parameters.ApiValidationSuppressionFiles, Parameters.UpdateApiValidationSuppression))); }); + Target OutputApiDiff => _ => _ + .DependsOn(CreateNugetPackages) + .Executes(async () => + { + await Task.WhenAll( + Directory.GetFiles(Parameters.NugetRoot, "*.nupkg").Select(nugetPackage => ApiDiffHelper.GetDiff( + ApiGenTool, RootDirectory / "api" / "diff", + nugetPackage, Parameters.ApiValidationBaseline))); + }); + Target RunTests => _ => _ .DependsOn(RunCoreLibsTests) .DependsOn(RunRenderTests) diff --git a/nukebuild/_build.csproj b/nukebuild/_build.csproj index 1286102347..8a1d7069b8 100644 --- a/nukebuild/_build.csproj +++ b/nukebuild/_build.csproj @@ -7,6 +7,8 @@ $(NoWarn);CS0649;CS0169;SYSLIB0011 1 net7.0 + + https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet8-transport/nuget/v3/index.json @@ -24,6 +26,7 @@ +