Use MSBuild in process to grovel project information

This replaces the (dirty) technique of target-based injection with
hosting MSBuild in process.

This is a speedup of about 800ms per project for every project past the first
one. Given that Tye is for microservices, hopefully you have more than one
project.

src/Microsoft.Tye.Core/Microsoft.Tye.Core.csproj

@@ -12,16 +12,23 @@
 
   <ItemGroup>
     <PackageReference Include="KubernetesClient" Version="1.6.11" />
-    <PackageReference Include="Microsoft.Build" Version="16.5.0" />
+    <!--
+      The Microsoft.Build.Locator package takes care of dynamically loading these assemblies
+      at runtime. We don't need/want to ship them, just to have them as references.
+    -->
+    <PackageReference Include="Microsoft.Build" Version="16.5.0" ExcludeAssets="runtime" />
+    <PackageReference Include="Microsoft.Build.Locator" Version="1.2.6" />
+    <!-- Hoisted to avoid a conflict with Microsoft.Build -->
+    <PackageReference Include="Microsoft.Win32.Registry" Version="4.7.0" />
     <PackageReference Include="semver" Version="2.0.6" />
     <PackageReference Include="System.CommandLine" Version="2.0.0-beta1.20071.2" />
     <PackageReference Include="System.Commandline.Rendering" Version="0.3.0-alpha.20070.2" />
+    <!-- Hoisted to avoid a conflict with Microsoft.Build -->
+    <PackageReference Include="System.Security.Principal.Windows" Version="4.7.0" />
     <PackageReference Include="YamlDotNet" Version="8.0.0" />
   </ItemGroup>
 
   <ItemGroup>
-    <Content Include="build\**" CopyToOutputDirectory="PreserveNewest" />
-    <Content Include="buildMultiTargeting\**" CopyToOutputDirectory="PreserveNewest" />
     <Content Include="Templates\**" CopyToOutputDirectory="PreserveNewest" />
   </ItemGroup>
 
@@ -30,9 +37,5 @@
     <Compile Include="..\shared\TempDirectory.cs" Link="TempDirectory.cs" />
     <Compile Include="..\shared\TempFile.cs" Link="TempFile.cs" />
   </ItemGroup>
-  
-  <ItemGroup>
-    <EmbeddedResource Include="Resources\Imports.targets" />
-  </ItemGroup>
 
 </Project>

src/Microsoft.Tye.Core/OutputContext.cs

@@ -101,11 +101,21 @@ namespace Microsoft.Tye
             WriteLine(Verbosity.Info, message);
         }
 
+        public void WriteInfo(string message)
+        {
+            Write(Verbosity.Info, message);
+        }
+
         public void WriteInfoLine(string message)
         {
             WriteLine(Verbosity.Info, message);
         }
 
+        public void WriteDebug(string message)
+        {
+            Write(Verbosity.Debug, message);
+        }
+
         public void WriteDebugLine(string message)
         {
             WriteLine(Verbosity.Debug, message);

src/Microsoft.Tye.Core/ProjectReader.cs

@@ -3,17 +3,26 @@
 // See the LICENSE file in the project root for more information.
 
 using System;
-using System.CommandLine.Invocation;
+using System.Diagnostics;
 using System.IO;
 using System.Linq;
+using System.Reflection;
+using System.Runtime.CompilerServices;
+using System.Runtime.Loader;
 using System.Threading.Tasks;
+using Microsoft.Build.Definition;
+using Microsoft.Build.Execution;
+using Microsoft.Build.Framework;
+using Microsoft.Build.Locator;
+using Microsoft.Build.Logging;
 using Semver;
 
 namespace Microsoft.Tye
 {
     public static class ProjectReader
     {
-        public static async Task ReadProjectDetailsAsync(OutputContext output, FileInfo projectFile, Project project)
+        private static bool registered;
+        public static Task ReadProjectDetailsAsync(OutputContext output, FileInfo projectFile, Project project)
         {
             if (output is null)
             {
@@ -30,94 +39,136 @@ namespace Microsoft.Tye
                 throw new ArgumentNullException(nameof(project));
             }
 
+            EnsureMSBuildRegistered(projectFile);
+
             using (var step = output.BeginStep("Reading Project Details..."))
             {
-                await EvaluateMSBuildAsync(output, projectFile, project);
+                EvaluateProject(output, projectFile, project);
 
                 if (!SemVersion.TryParse(project.Version, out var version))
                 {
-                    output.WriteInfoLine($"No version or invalid version 'application.Version' found, using default.");
+                    output.WriteInfoLine($"No version or invalid version '{project.Version}' found, using default.");
                     version = new SemVersion(0, 1, 0);
                     project.Version = version.ToString();
                 }
 
                 step.MarkComplete();
             }
+
+            return Task.CompletedTask;
+        }
+
+        private static void EnsureMSBuildRegistered(FileInfo projectFile)
+        {
+            if (!registered)
+            {
+                // It says VisualStudio - but we'll just use .NET SDK
+                var instances = MSBuildLocator.QueryVisualStudioInstances(new VisualStudioInstanceQueryOptions()
+                {
+                    DiscoveryTypes = DiscoveryType.DotNetSdk,
+
+                    // Using the project as the working directory. We're making the assumption that
+                    // all of the projects want to use the same SDK version. This library is going
+                    // load a single version of the SDK's assemblies into our process, so we can't
+                    // use supprt SDKs at once without getting really tricky.
+                    //
+                    // The .NET SDK-based discovery uses `dotnet --info` and returns the SDK
+                    // in use for the directory.
+                    //
+                    // https://github.com/microsoft/MSBuildLocator/blob/master/src/MSBuildLocator/MSBuildLocator.cs#L320
+                    WorkingDirectory = projectFile.DirectoryName,
+                });
+
+                var instance = instances.SingleOrDefault();
+                if (instance == null)
+                {
+                    throw new CommandException("Failed to find dotnet. Make sure the .NET SDK is installed and on the PATH.");
+                }
+
+                MSBuildLocator.RegisterInstance(instance);
+                registered = true;
+            }
         }
 
-        private static async Task EvaluateMSBuildAsync(OutputContext output, FileInfo projectFile, Project project)
+        // Do not load MSBuild types before using EnsureMSBuildRegistered.
+        [MethodImpl(MethodImplOptions.NoInlining)]
+        private static void EvaluateProject(OutputContext output, FileInfo projectFile, Project project)
         {
+            var sw = Stopwatch.StartNew();
+
+            ProjectInstance projectInstance;
             try
             {
-                output.WriteDebugLine("Installing msbuild targets.");
+                output.WriteDebugLine($"Loading project '{projectFile.FullName}'.");
-                TargetInstaller.Install(projectFile.FullName);
+                projectInstance = ProjectInstance.FromFile(projectFile.FullName, new ProjectOptions());
-                output.WriteDebugLine("Installed msbuild targets.");
+                output.WriteDebugLine($"Loaded project '{projectFile.FullName}'.");
             }
-            catch (Exception ex)
+            catch
             {
-                throw new CommandException("Failed to install targets.", ex);
+                throw new CommandException($"Failed to load project: '{projectFile.FullName}'.");
             }
 
-            var outputFilePath = Path.GetTempFileName();
+            // Currently we only log at debug level.
+            var logger = new ConsoleLogger(
+                verbosity: LoggerVerbosity.Normal,
+                write: message => output.WriteDebug(message),
+                colorSet: null,
+                colorReset: null);
 
             try
             {
-                var capture = output.Capture();
+                AssemblyLoadContext.Default.Resolving += ResolveAssembly;
-                var programRoot = Path.GetDirectoryName(typeof(Program).Assembly.Location);
+                var result = projectInstance.Build(
-
+                    targets: new[] { "Restore", "ResolveReferences", "ResolvePackageDependenciesDesignTime", "PrepareResources", "GetAssemblyAttributes", },
-                var restore = string.Empty;
+                    loggers: new[] { logger, });
-                if (!File.Exists(Path.Combine(projectFile.DirectoryName, "obj", "project.assets.json")))
+
-                {
+                // If the build fails, we're not really blocked from doing our work.
-                    restore = "/restore";
+                // For now we just log the output to debug. There are errors that occur during
-                }
+                // running these targets we don't really care as long as we get the data.
-
+            }
-                output.WriteDebugLine("Running 'dotnet msbuild'.");
+            finally
-                var msbuildCommand = $"msbuild {restore} /t:EvaluateTyeProjectInfo \"/p:TyeTargetLocation={programRoot}\" \"/p:TyeOutputFilePath={outputFilePath}\"";
+            {
-
+                AssemblyLoadContext.Default.Resolving -= ResolveAssembly;
-                output.WriteCommandLine("dotnet", msbuildCommand);
+            }
-                var exitCode = await Process.ExecuteAsync(
-                    $"dotnet",
-                    msbuildCommand,
-                    workingDir: projectFile.DirectoryName,
-                    stdOut: capture.StdOut,
-                    stdErr: capture.StdErr);
-
-                output.WriteDebugLine($"Done running 'dotnet msbuild' exit code: {exitCode}");
-                if (exitCode != 0)
-                {
-                    throw new CommandException("'dotnet msbuild' failed.");
-                }
 
-                var lines = await File.ReadAllLinesAsync(outputFilePath);
+            // Reading both InformationalVersion and Version is more resilant in the face of build failures.
-                for (var i = 0; i < lines.Length; i++)
+            var version = projectInstance.GetProperty("InformationalVersion")?.EvaluatedValue ?? projectInstance.GetProperty("Version").EvaluatedValue;
+            project.Version = version;
+            output.WriteDebugLine($"Found application version: {version}");
+
+            var targetFramework = projectInstance.GetPropertyValue("TargetFramework");
+            project.TargetFramework = targetFramework;
+            output.WriteDebugLine($"Found target framework: {targetFramework}");
+
+            var sharedFrameworks = projectInstance.GetItems("FrameworkReference").Select(i => i.EvaluatedInclude).ToList();
+            project.Frameworks.AddRange(sharedFrameworks.Select(s => new Framework(s)));
+            output.WriteDebugLine($"Found shared frameworks: {string.Join(", ", sharedFrameworks)}");
+
+            output.WriteDebugLine($"Evaluation Took: {sw.Elapsed.TotalMilliseconds}ms");
+
+            // The Microsoft.Build.Locator doesn't handle the loading of other assemblies
+            // that are shipped with MSBuild (ex NuGet).
+            //
+            // This means that the set of assemblies that need special handling depends on the targets
+            // that we run :(
+            //
+            // This is workaround for this limitation based on the targets we need to run
+            // to resolve references and versions.
+            //
+            // See: https://github.com/microsoft/MSBuildLocator/issues/86
+            Assembly? ResolveAssembly(AssemblyLoadContext context, AssemblyName assemblyName)
+            {
+                if (assemblyName.Name is object && assemblyName.Name.StartsWith("NuGet."))
                 {
-                    var line = lines[i];
+                    var msbuildDirectory = Environment.GetEnvironmentVariable("MSBuildExtensionsPath")!;
-                    if (line.StartsWith("version="))
+                    var assemblyFilePath = Path.Combine(msbuildDirectory, assemblyName.Name + ".dll");
+                    if (File.Exists(assemblyFilePath))
                     {
-                        project.Version = line.Substring("version=".Length).Trim();
+                        return context.LoadFromAssemblyPath(assemblyFilePath);
-                        output.WriteDebugLine($"Found application version: {line}");
-                        continue;
-                    }
-
-                    if (line.StartsWith("tfm"))
-                    {
-                        project.TargetFramework = line.Substring("tfm=".Length).Trim();
-                        output.WriteDebugLine($"Found target framework: {line}");
-                        continue;
-                    }
-
-                    if (line.StartsWith("frameworks="))
-                    {
-                        var right = line.Substring("frameworks=".Length).Trim();
-                        project.Frameworks.AddRange(right.Split(",").Select(s => new Framework(s)));
-                        output.WriteDebugLine($"Found shared frameworks: {line}");
-                        continue;
                     }
                 }
-            }
+
-            finally
+                return default;
-            {
-                File.Delete(outputFilePath);
             }
         }
     }

src/Microsoft.Tye.Core/Resources/Imports.targets

@@ -1,8 +0,0 @@
-<Project ToolsVersion="14.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
-  <ImportGroup Condition=" '$(TargetFramework)' == '' AND '$(ExcludeRestorePackageImports)' != 'true' ">
-    <Import Project="$(TyeTargetLocation)\buildMultiTargeting\Tye.targets" Condition="'$(TyeTargetLocation)'!=''" />
-  </ImportGroup>
-  <ImportGroup Condition=" '$(TargetFramework)' != '' AND '$(ExcludeRestorePackageImports)' != 'true' ">
-    <Import Project="$(TyeTargetLocation)\build\Tye.targets" Condition="'$(TyeTargetLocation)'!=''" />
-  </ImportGroup>
-</Project>

src/Microsoft.Tye.Core/TargetInstaller.cs

@@ -1,48 +0,0 @@
-// Licensed to the .NET Foundation under one or more agreements.
-// The .NET Foundation licenses this file to you under the MIT license.
-// See the LICENSE file in the project root for more information.
-
-using System;
-using System.IO;
-using System.Reflection;
-
-namespace Microsoft.Tye
-{
-    public static class TargetInstaller
-    {
-        public static void Install(string projectFilePath)
-        {
-            if (projectFilePath is null)
-            {
-                throw new ArgumentNullException(nameof(projectFilePath));
-            }
-
-            var projectDirectory = Path.GetDirectoryName(projectFilePath);
-            var intermediateDirectory = Path.Combine(projectDirectory!, "obj");
-
-            Directory.CreateDirectory(intermediateDirectory);
-
-            var fileName = $"{Path.GetFileName(projectFilePath)}.Tye.targets";
-            var targetFilePath = Path.Combine(intermediateDirectory, fileName);
-
-            if (File.Exists(targetFilePath))
-            {
-                return;
-            }
-
-            var toolType = typeof(TargetInstaller);
-            var toolAssembly = toolType.GetTypeInfo().Assembly;
-            var toolImportTargetsResourceName = $"Tye.Resources.Imports.targets";
-
-            using var stream = toolAssembly.GetManifestResourceStream(toolImportTargetsResourceName);
-            if (stream == null)
-            {
-                throw new CommandException("Failed to find resource. Valid names: " + string.Join(", ", toolAssembly.GetManifestResourceNames()));
-            }
-
-            var targetBytes = new byte[stream.Length];
-            stream.Read(targetBytes, 0, targetBytes.Length);
-            File.WriteAllBytes(targetFilePath, targetBytes);
-        }
-    }
-}

src/Microsoft.Tye.Core/build/Tye.targets

@@ -1,28 +0,0 @@
-<Project xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
-
-  <PropertyGroup>
-    <EvaluateTyeProjectInfoDependsOn>
-      $(EvaluateTyeProjectInfoDependsOn);
-      ResolveReferences;
-      ResolvePackageDependenciesDesignTime;
-      PrepareResources;
-      GetAssemblyAttributes;
-    </EvaluateTyeProjectInfoDependsOn>
-  </PropertyGroup>
-
-  <Target Name="EvaluateTyeProjectInfo" DependsOnTargets="$(EvaluateTyeProjectInfoDependsOn)">
-    <Error Message="TyeOutputFilePath was not specified." Condition="'$(TyeOutputFilePath)'==''" />
-
-    <PropertyGroup>
-      <_Frameworks>@(FrameworkReference, ',')</_Frameworks>
-    </PropertyGroup>
-    <ItemGroup>
-     <_Lines Include="version=$(InformationalVersion)" />
-     <_Lines Include="tfm=$(TargetFramework)" />
-     <_Lines Include="frameworks=$(_Frameworks)" />
-    </ItemGroup>
-
-    <WriteLinesToFile File="$(TyeOutputFilePath)" Lines="@(_Lines)" Overwrite="true" />
-  </Target>
-
-</Project>

src/Microsoft.Tye.Core/buildMultiTargeting/Tye.targets

@@ -1,20 +0,0 @@
-<Project xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
-  <UsingTask TaskName="GetNearestTargetFramework" AssemblyFile="$(MicrosoftNETBuildTasksAssembly)" />
-
-  <Target Name="EvaluateTyeProjectInfo" DependsOnTargets="$(EvaluateTyeProjectInfoDependsOn)">
-    <GetNearestTargetFramework 
-      ReferringTargetFramework="netcoreapp3.0"
-      PossibleTargetFrameworks="$(TargetFrameworks)"
-      ProjectFilePath="$(MSBuildProjectFullPath)"
-      ContinueOnError="WarnAndContinue">
-      <Output PropertyName="NearestTargetFramework" TaskParameter="NearestTargetFramework" />
-    </GetNearestTargetFramework>
-
-    <Error Text="Tye requires targeting netcoreapp3.0 and newer." Condition=" '$(NearestTargetFramework)' == '' " />
-
-    <MSBuild 
-      Projects="$(MSBuildProjectFile)"
-      Targets="EvaluateTyeProjectInfo"
-      Properties="TargetFramework=$(NearestTargetFramework);OutputFile=$(OutputFile)" />
-  </Target>
-</Project>