From 7395203978339d53f549eceb203dc9670246c248 Mon Sep 17 00:00:00 2001 From: David Fowler Date: Thu, 7 May 2020 13:28:23 -0700 Subject: [PATCH] Fixes for docker scenarios (#444) --- .../ContainerServiceBuilder.cs | 2 + src/Microsoft.Tye.Hosting/DockerRunner.cs | 68 ++++------------ .../Model/DockerRunInfo.cs | 2 + .../Model/EnvironmentVariable.cs | 6 ++ .../Model/ProjectRunInfo.cs | 3 +- src/Microsoft.Tye.Hosting/ProcessRunner.cs | 9 ++- .../TransformProjectsIntoContainers.cs | 78 ++++++++++++++++++- src/tye/ApplicationBuilderExtensions.cs | 5 +- test/E2ETest/TyeRunTests.cs | 15 +++- 9 files changed, 126 insertions(+), 62 deletions(-) diff --git a/src/Microsoft.Tye.Core/ContainerServiceBuilder.cs b/src/Microsoft.Tye.Core/ContainerServiceBuilder.cs index f6aec50f..df6b19b5 100644 --- a/src/Microsoft.Tye.Core/ContainerServiceBuilder.cs +++ b/src/Microsoft.Tye.Core/ContainerServiceBuilder.cs @@ -16,6 +16,8 @@ namespace Microsoft.Tye public string Image { get; set; } + public bool IsAspNet { get; set; } + public string? Args { get; set; } public string? DockerFile { get; set; } diff --git a/src/Microsoft.Tye.Hosting/DockerRunner.cs b/src/Microsoft.Tye.Hosting/DockerRunner.cs index 5020399b..0006ed26 100644 --- a/src/Microsoft.Tye.Hosting/DockerRunner.cs +++ b/src/Microsoft.Tye.Hosting/DockerRunner.cs @@ -229,15 +229,6 @@ namespace Microsoft.Tye.Hosting hostname = addresses[0].ToString(); } - // This is .NET specific - var userSecretStore = GetUserSecretsPathFromSecrets(); - - if (!string.IsNullOrEmpty(userSecretStore)) - { - // Map the user secrets on this drive to user secrets - docker.VolumeMappings.Add(new DockerVolume(source: userSecretStore, name: null, target: "/root/.microsoft/usersecrets:ro")); - } - var dockerInfo = new DockerInformation(new Task[service.Description.Replicas]); async Task RunDockerContainer(IEnumerable<(int ExternalPort, int Port, int? ContainerPort, string? Protocol)> ports) @@ -250,15 +241,7 @@ namespace Microsoft.Tye.Hosting service.ReplicaEvents.OnNext(new ReplicaEvent(ReplicaState.Added, status)); - var environment = new Dictionary - { - // Default to development environment - ["DOTNET_ENVIRONMENT"] = "Development", - ["ASPNETCORE_ENVIRONMENT"] = "Development", - // Remove the color codes from the console output - ["DOTNET_LOGGING__CONSOLE__DISABLECOLORS"] = "true", - ["ASPNETCORE_LOGGING__CONSOLE__DISABLECOLORS"] = "true" - }; + var environment = new Dictionary(); var portString = ""; @@ -271,16 +254,19 @@ namespace Microsoft.Tye.Hosting // 1. Tell the docker container what port to bind to portString = docker.Private ? "" : string.Join(" ", ports.Select(p => $"-p {p.Port}:{p.ContainerPort ?? p.Port}")); - // 2. Configure ASP.NET Core to bind to those same ports - environment["ASPNETCORE_URLS"] = string.Join(";", ports.Select(p => $"{p.Protocol ?? "http"}://*:{p.ContainerPort ?? p.Port}")); - - // Set the HTTPS port for the redirect middleware - foreach (var p in ports) + if (docker.IsAspNet) { - if (string.Equals(p.Protocol, "https", StringComparison.OrdinalIgnoreCase)) + // 2. Configure ASP.NET Core to bind to those same ports + environment["ASPNETCORE_URLS"] = string.Join(";", ports.Select(p => $"{p.Protocol ?? "http"}://*:{p.ContainerPort ?? p.Port}")); + + // Set the HTTPS port for the redirect middleware + foreach (var p in ports) { - // We need to set the redirect URL to the exposed port so the redirect works cleanly - environment["HTTPS_PORT"] = p.ExternalPort.ToString(); + if (string.Equals(p.Protocol, "https", StringComparison.OrdinalIgnoreCase)) + { + // We need to set the redirect URL to the exposed port so the redirect works cleanly + environment["HTTPS_PORT"] = p.ExternalPort.ToString(); + } } } @@ -385,6 +371,8 @@ namespace Microsoft.Tye.Hosting _logger.LogInformation("Collecting docker logs for {ContainerName}.", replica); + var backOff = TimeSpan.FromSeconds(5); + while (!dockerInfo.StoppingTokenSource.Token.IsCancellationRequested) { var logsRes = await ProcessUtil.RunAsync("docker", $"logs -f {containerId}", @@ -403,13 +391,15 @@ namespace Microsoft.Tye.Hosting try { // Avoid spamming logs if restarts are happening - await Task.Delay(5000, dockerInfo.StoppingTokenSource.Token); + await Task.Delay(backOff, dockerInfo.StoppingTokenSource.Token); } catch (OperationCanceledException) { break; } } + + backOff *= 2; } _logger.LogInformation("docker logs collection for {ContainerName} complete with exit code {ExitCode}", replica, result.ExitCode); @@ -528,30 +518,6 @@ namespace Microsoft.Tye.Hosting return Task.CompletedTask; } - private static string? GetUserSecretsPathFromSecrets() - { - // This is the logic used to determine the user secrets path - // See https://github.com/dotnet/extensions/blob/64140f90157fec1bfd8aeafdffe8f30308ccdf41/src/Configuration/Config.UserSecrets/src/PathHelper.cs#L27 - const string userSecretsFallbackDir = "DOTNET_USER_SECRETS_FALLBACK_DIR"; - - // For backwards compat, this checks env vars first before using Env.GetFolderPath - var appData = Environment.GetEnvironmentVariable("APPDATA"); - var root = appData // On Windows it goes to %APPDATA%\Microsoft\UserSecrets\ - ?? Environment.GetEnvironmentVariable("HOME") // On Mac/Linux it goes to ~/.microsoft/usersecrets/ - ?? Environment.GetFolderPath(Environment.SpecialFolder.ApplicationData) - ?? Environment.GetFolderPath(Environment.SpecialFolder.UserProfile) - ?? Environment.GetEnvironmentVariable(userSecretsFallbackDir); // this fallback is an escape hatch if everything else fails - - if (string.IsNullOrEmpty(root)) - { - return null; - } - - return !string.IsNullOrEmpty(appData) - ? Path.Combine(root, "Microsoft", "UserSecrets") - : Path.Combine(root, ".microsoft", "usersecrets"); - } - private class DockerInformation { public DockerInformation(Task[] tasks) diff --git a/src/Microsoft.Tye.Hosting/Model/DockerRunInfo.cs b/src/Microsoft.Tye.Hosting/Model/DockerRunInfo.cs index 696177cc..7ea1d170 100644 --- a/src/Microsoft.Tye.Hosting/Model/DockerRunInfo.cs +++ b/src/Microsoft.Tye.Hosting/Model/DockerRunInfo.cs @@ -2,6 +2,7 @@ // The .NET Foundation licenses this file to you under the MIT license. // See the LICENSE file in the project root for more information. +using System; using System.Collections.Generic; using System.IO; @@ -16,6 +17,7 @@ namespace Microsoft.Tye.Hosting.Model } public bool Private { get; set; } + public bool IsAspNet { get; set; } public string? NetworkAlias { get; set; } diff --git a/src/Microsoft.Tye.Hosting/Model/EnvironmentVariable.cs b/src/Microsoft.Tye.Hosting/Model/EnvironmentVariable.cs index 2cb4ab32..7d2ac528 100644 --- a/src/Microsoft.Tye.Hosting/Model/EnvironmentVariable.cs +++ b/src/Microsoft.Tye.Hosting/Model/EnvironmentVariable.cs @@ -11,6 +11,12 @@ namespace Microsoft.Tye.Hosting.Model Name = name; } + public EnvironmentVariable(string name, string? value) + { + Name = name; + Value = value; + } + public string Name { get; } public string? Value { get; set; } diff --git a/src/Microsoft.Tye.Hosting/Model/ProjectRunInfo.cs b/src/Microsoft.Tye.Hosting/Model/ProjectRunInfo.cs index 6a09eda0..59d054ab 100644 --- a/src/Microsoft.Tye.Hosting/Model/ProjectRunInfo.cs +++ b/src/Microsoft.Tye.Hosting/Model/ProjectRunInfo.cs @@ -24,6 +24,7 @@ namespace Microsoft.Tye.Hosting.Model RunCommand = project.RunCommand; RunArguments = project.RunArguments; PublishOutputPath = project.PublishDir; + IsAspNet = project.IsAspNet; } public Dictionary BuildProperties { get; } = new Dictionary(); @@ -34,7 +35,7 @@ namespace Microsoft.Tye.Hosting.Model public string TargetFrameworkName { get; set; } = default!; public string TargetFrameworkVersion { get; set; } = default!; public string TargetFramework { get; } - + public bool IsAspNet { get; } public string Version { get; } public string AssemblyName { get; } diff --git a/src/Microsoft.Tye.Hosting/ProcessRunner.cs b/src/Microsoft.Tye.Hosting/ProcessRunner.cs index 302fb433..30eb38ab 100644 --- a/src/Microsoft.Tye.Hosting/ProcessRunner.cs +++ b/src/Microsoft.Tye.Hosting/ProcessRunner.cs @@ -230,6 +230,8 @@ namespace Microsoft.Tye.Hosting environment["PORT"] = string.Join(";", ports.Select(p => $"{p.Port}")); } + var backOff = TimeSpan.FromSeconds(5); + while (!processInfo.StoppedTokenSource.IsCancellationRequested) { var replica = serviceName + "_" + Guid.NewGuid().ToString().Substring(0, 10).ToLower(); @@ -280,6 +282,9 @@ namespace Microsoft.Tye.Hosting _logger.LogInformation("{ServiceName} running on process id {PID}", replica, pid); } + // Reset the backoff + backOff = TimeSpan.FromSeconds(5); + status.Pid = pid; WriteReplicaToStore(pid.ToString()); @@ -301,7 +306,7 @@ namespace Microsoft.Tye.Hosting try { - await Task.Delay(5000, processInfo.StoppedTokenSource.Token); + await Task.Delay(backOff, processInfo.StoppedTokenSource.Token); } catch (OperationCanceledException) { @@ -309,6 +314,8 @@ namespace Microsoft.Tye.Hosting } } + backOff *= 2; + service.Restarts++; if (status.ExitCode != null) diff --git a/src/Microsoft.Tye.Hosting/TransformProjectsIntoContainers.cs b/src/Microsoft.Tye.Hosting/TransformProjectsIntoContainers.cs index d62f8764..c4ccc9b1 100644 --- a/src/Microsoft.Tye.Hosting/TransformProjectsIntoContainers.cs +++ b/src/Microsoft.Tye.Hosting/TransformProjectsIntoContainers.cs @@ -2,22 +2,25 @@ // The .NET Foundation licenses this file to you under the MIT license. // See the LICENSE file in the project root for more information. +using System; using System.Collections.Generic; +using System.IO; +using System.Linq; using System.Threading.Tasks; using Microsoft.Extensions.Logging; using Microsoft.Tye.Hosting.Model; namespace Microsoft.Tye.Hosting { - using System.Linq; - public class TransformProjectsIntoContainers : IApplicationProcessor { private readonly ILogger _logger; + private Lazy _certificateDirectory; public TransformProjectsIntoContainers(ILogger logger) { _logger = logger; + _certificateDirectory = new Lazy(() => TempDirectory.Create()); } public Task StartAsync(Application application) @@ -72,7 +75,8 @@ namespace Microsoft.Tye.Hosting var outputFileName = project.AssemblyName + ".dll"; var dockerRunInfo = new DockerRunInfo(containerImage, $"dotnet {outputFileName} {project.Args}") { - WorkingDirectory = "/app" + WorkingDirectory = "/app", + IsAspNet = project.IsAspNet }; dockerRunInfo.VolumeMappings.Add(new DockerVolume(source: project.PublishOutputPath, name: null, target: "/app")); @@ -80,18 +84,84 @@ namespace Microsoft.Tye.Hosting // Make volume mapping works when running as a container dockerRunInfo.VolumeMappings.AddRange(project.VolumeMappings); + // This is .NET specific + var userSecretStore = GetUserSecretsPathFromSecrets(); + + if (!string.IsNullOrEmpty(userSecretStore)) + { + // Map the user secrets on this drive to user secrets + dockerRunInfo.VolumeMappings.Add(new DockerVolume(source: userSecretStore, name: null, target: "/root/.microsoft/usersecrets:ro")); + } + + // Default to development environment + serviceDescription.Configuration.Add(new EnvironmentVariable("DOTNET_ENVIRONMENT", "Development")); + + // Remove the color codes from the console output + serviceDescription.Configuration.Add(new EnvironmentVariable("DOTNET_LOGGING__CONSOLE__DISABLECOLORS", "true")); + + if (project.IsAspNet) + { + serviceDescription.Configuration.Add(new EnvironmentVariable("ASPNETCORE_ENVIRONMENT", "Development")); + serviceDescription.Configuration.Add(new EnvironmentVariable("ASPNETCORE_LOGGING__CONSOLE__DISABLECOLORS", "true")); + } + + // If we have an https binding then export the dev cert and mount the volume into the container + if (serviceDescription.Bindings.Any(b => string.Equals(b.Protocol, "https", StringComparison.OrdinalIgnoreCase))) + { + // We export the developer certificate from this machine + var certPassword = Guid.NewGuid().ToString(); + var certificateDirectory = _certificateDirectory.Value; + var certificateFilePath = Path.Combine(certificateDirectory.DirectoryPath, project.AssemblyName + ".pfx"); + await ProcessUtil.RunAsync("dotnet", $"dev-certs https -ep {certificateFilePath} -p {certPassword}"); + serviceDescription.Configuration.Add(new EnvironmentVariable("Kestrel__Certificates__Development__Password", certPassword)); + + // Certificate Path: https://github.com/dotnet/aspnetcore/blob/a9d702624a02ad4ebf593d9bf9c1c69f5702a6f5/src/Servers/Kestrel/Core/src/KestrelConfigurationLoader.cs#L419 + dockerRunInfo.VolumeMappings.Add(new DockerVolume(source: certificateDirectory.DirectoryPath, name: null, target: "/root/.aspnet/https:ro")); + } + // Change the project into a container info serviceDescription.RunInfo = dockerRunInfo; } private static string DetermineContainerImage(ProjectRunInfo project) { - return $"mcr.microsoft.com/dotnet/core/sdk:{project.TargetFrameworkVersion}"; + var baseImage = project.IsAspNet ? "mcr.microsoft.com/dotnet/core/aspnet" : "mcr.microsoft.com/dotnet/core/runtime"; + + return $"{baseImage}:{project.TargetFrameworkVersion}"; } public Task StopAsync(Application application) { + if (_certificateDirectory.IsValueCreated) + { + _certificateDirectory.Value.Dispose(); + } + return Task.CompletedTask; } + + private static string? GetUserSecretsPathFromSecrets() + { + // This is the logic used to determine the user secrets path + // See https://github.com/dotnet/extensions/blob/64140f90157fec1bfd8aeafdffe8f30308ccdf41/src/Configuration/Config.UserSecrets/src/PathHelper.cs#L27 + const string userSecretsFallbackDir = "DOTNET_USER_SECRETS_FALLBACK_DIR"; + + // For backwards compat, this checks env vars first before using Env.GetFolderPath + var appData = Environment.GetEnvironmentVariable("APPDATA"); + var root = appData // On Windows it goes to %APPDATA%\Microsoft\UserSecrets\ + ?? Environment.GetEnvironmentVariable("HOME") // On Mac/Linux it goes to ~/.microsoft/usersecrets/ + ?? Environment.GetFolderPath(Environment.SpecialFolder.ApplicationData) + ?? Environment.GetFolderPath(Environment.SpecialFolder.UserProfile) + ?? Environment.GetEnvironmentVariable(userSecretsFallbackDir); // this fallback is an escape hatch if everything else fails + + if (string.IsNullOrEmpty(root)) + { + return null; + } + + return !string.IsNullOrEmpty(appData) + ? Path.Combine(root, "Microsoft", "UserSecrets") + : Path.Combine(root, ".microsoft", "usersecrets"); + } } } diff --git a/src/tye/ApplicationBuilderExtensions.cs b/src/tye/ApplicationBuilderExtensions.cs index 65243230..53aaaf3a 100644 --- a/src/tye/ApplicationBuilderExtensions.cs +++ b/src/tye/ApplicationBuilderExtensions.cs @@ -45,7 +45,10 @@ namespace Microsoft.Tye } else if (service is ContainerServiceBuilder container) { - var dockerRunInfo = new DockerRunInfo(container.Image, container.Args); + var dockerRunInfo = new DockerRunInfo(container.Image, container.Args) + { + IsAspNet = container.IsAspNet + }; if (!string.IsNullOrEmpty(container.DockerFile)) { diff --git a/test/E2ETest/TyeRunTests.cs b/test/E2ETest/TyeRunTests.cs index e52a6435..60bd1e51 100644 --- a/test/E2ETest/TyeRunTests.cs +++ b/test/E2ETest/TyeRunTests.cs @@ -165,10 +165,13 @@ namespace E2ETest application.Services.Remove(project); var outputFileName = project.AssemblyName + ".dll"; - var container = new ContainerServiceBuilder(project.Name, $"mcr.microsoft.com/dotnet/core/sdk:{project.TargetFrameworkVersion}"); + var container = new ContainerServiceBuilder(project.Name, $"mcr.microsoft.com/dotnet/core/aspnet:{project.TargetFrameworkVersion}") + { + IsAspNet = true + }; container.Volumes.Add(new VolumeBuilder(project.PublishDir, name: null, target: "/app")); container.Args = $"dotnet /app/{outputFileName} {project.Args}"; - container.Bindings.AddRange(project.Bindings); + container.Bindings.AddRange(project.Bindings.Where(b => b.Protocol != "https")); await ProcessUtil.RunAsync("dotnet", $"publish \"{project.ProjectFile.FullName}\" /nologo", outputDataReceived: _sink.WriteLine, errorDataReceived: _sink.WriteLine); application.Services.Add(container); @@ -209,11 +212,15 @@ namespace E2ETest application.Services.Remove(project); var outputFileName = project.AssemblyName + ".dll"; - var container = new ContainerServiceBuilder(project.Name, $"mcr.microsoft.com/dotnet/core/sdk:{project.TargetFrameworkVersion}"); + var container = new ContainerServiceBuilder(project.Name, $"mcr.microsoft.com/dotnet/core/aspnet:{project.TargetFrameworkVersion}") + { + IsAspNet = true + }; container.Dependencies.UnionWith(project.Dependencies); container.Volumes.Add(new VolumeBuilder(project.PublishDir, name: null, target: "/app")); container.Args = $"dotnet /app/{outputFileName} {project.Args}"; - container.Bindings.AddRange(project.Bindings); + // We're not setting up the dev cert here + container.Bindings.AddRange(project.Bindings.Where(b => b.Protocol != "https")); await ProcessUtil.RunAsync("dotnet", $"publish \"{project.ProjectFile.FullName}\" /nologo", outputDataReceived: _sink.WriteLine, errorDataReceived: _sink.WriteLine); application.Services.Add(container);