From d2be7d504f3743bcef212a7b8aa0c38a25d14d8f Mon Sep 17 00:00:00 2001 From: David Fowler Date: Thu, 2 Apr 2020 19:20:28 -0700 Subject: [PATCH] Added container proxy for talking to the host (#292) * Added container proxy for talking to the host - This change introduces a container proxy which makes it possible for docker containers can talk to host services using container networking. These proxies will not show up in the dashboard as they are "infrastructure" containers. - Added Private and NetworkAlias to DockerRunInfo but did not expose these configuration. - All container communication is done using host names. - Fix host shutdown again - Bind to all interfaces on linux --- .../ApplicationA/ApplicationA.csproj | 7 + samples/nginx-ingress/ApplicationA/Program.cs | 26 ++++ .../Properties/launchSettings.json | 27 ++++ samples/nginx-ingress/ApplicationA/Startup.cs | 40 ++++++ .../ApplicationA/appsettings.Development.json | 9 ++ .../ApplicationA/appsettings.json | 10 ++ .../ApplicationB/ApplicationB.csproj | 7 + samples/nginx-ingress/ApplicationB/Program.cs | 26 ++++ .../Properties/launchSettings.json | 27 ++++ samples/nginx-ingress/ApplicationB/Startup.cs | 40 ++++++ .../ApplicationB/appsettings.Development.json | 9 ++ .../ApplicationB/appsettings.json | 10 ++ samples/nginx-ingress/nginx-with-ingress.sln | 48 +++++++ samples/nginx-ingress/nginx.conf | 25 ++++ samples/nginx-ingress/tye.yaml | 16 +++ .../AggregateApplicationProcessor.cs | 21 ++- src/Microsoft.Tye.Hosting/DockerRunner.cs | 92 ++++++++++-- .../Microsoft.Tye.Hosting.csproj | 1 + .../Model/Application.cs | 6 +- .../Model/DockerRunInfo.cs | 4 + src/Microsoft.Tye.Hosting/PortAssigner.cs | 24 ++-- src/Microsoft.Tye.Hosting/ProxyService.cs | 8 +- src/Microsoft.Tye.Hosting/TyeHost.cs | 3 +- .../Microsoft.Tye.Proxy.csproj | 22 +++ src/Microsoft.Tye.Proxy/Program.cs | 135 ++++++++++++++++++ test/E2ETest/TyeRunTests.cs | 50 ++++++- tye.sln | 27 +++- 27 files changed, 682 insertions(+), 38 deletions(-) create mode 100644 samples/nginx-ingress/ApplicationA/ApplicationA.csproj create mode 100644 samples/nginx-ingress/ApplicationA/Program.cs create mode 100644 samples/nginx-ingress/ApplicationA/Properties/launchSettings.json create mode 100644 samples/nginx-ingress/ApplicationA/Startup.cs create mode 100644 samples/nginx-ingress/ApplicationA/appsettings.Development.json create mode 100644 samples/nginx-ingress/ApplicationA/appsettings.json create mode 100644 samples/nginx-ingress/ApplicationB/ApplicationB.csproj create mode 100644 samples/nginx-ingress/ApplicationB/Program.cs create mode 100644 samples/nginx-ingress/ApplicationB/Properties/launchSettings.json create mode 100644 samples/nginx-ingress/ApplicationB/Startup.cs create mode 100644 samples/nginx-ingress/ApplicationB/appsettings.Development.json create mode 100644 samples/nginx-ingress/ApplicationB/appsettings.json create mode 100644 samples/nginx-ingress/nginx-with-ingress.sln create mode 100644 samples/nginx-ingress/nginx.conf create mode 100644 samples/nginx-ingress/tye.yaml create mode 100644 src/Microsoft.Tye.Proxy/Microsoft.Tye.Proxy.csproj create mode 100644 src/Microsoft.Tye.Proxy/Program.cs diff --git a/samples/nginx-ingress/ApplicationA/ApplicationA.csproj b/samples/nginx-ingress/ApplicationA/ApplicationA.csproj new file mode 100644 index 00000000..92605c5a --- /dev/null +++ b/samples/nginx-ingress/ApplicationA/ApplicationA.csproj @@ -0,0 +1,7 @@ + + + + netcoreapp3.1 + + + diff --git a/samples/nginx-ingress/ApplicationA/Program.cs b/samples/nginx-ingress/ApplicationA/Program.cs new file mode 100644 index 00000000..56492c75 --- /dev/null +++ b/samples/nginx-ingress/ApplicationA/Program.cs @@ -0,0 +1,26 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; +using Microsoft.AspNetCore.Hosting; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Logging; + +namespace ApplicationA +{ + public class Program + { + public static void Main(string[] args) + { + CreateHostBuilder(args).Build().Run(); + } + + public static IHostBuilder CreateHostBuilder(string[] args) => + Host.CreateDefaultBuilder(args) + .ConfigureWebHostDefaults(webBuilder => + { + webBuilder.UseStartup(); + }); + } +} diff --git a/samples/nginx-ingress/ApplicationA/Properties/launchSettings.json b/samples/nginx-ingress/ApplicationA/Properties/launchSettings.json new file mode 100644 index 00000000..767a1d0b --- /dev/null +++ b/samples/nginx-ingress/ApplicationA/Properties/launchSettings.json @@ -0,0 +1,27 @@ +{ + "iisSettings": { + "windowsAuthentication": false, + "anonymousAuthentication": true, + "iisExpress": { + "applicationUrl": "http://localhost:2755", + "sslPort": 44369 + } + }, + "profiles": { + "IIS Express": { + "commandName": "IISExpress", + "launchBrowser": true, + "environmentVariables": { + "ASPNETCORE_ENVIRONMENT": "Development" + } + }, + "ApplicationA": { + "commandName": "Project", + "launchBrowser": true, + "applicationUrl": "https://localhost:5001;http://localhost:5000", + "environmentVariables": { + "ASPNETCORE_ENVIRONMENT": "Development" + } + } + } +} diff --git a/samples/nginx-ingress/ApplicationA/Startup.cs b/samples/nginx-ingress/ApplicationA/Startup.cs new file mode 100644 index 00000000..257fd799 --- /dev/null +++ b/samples/nginx-ingress/ApplicationA/Startup.cs @@ -0,0 +1,40 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; +using Microsoft.AspNetCore.Builder; +using Microsoft.AspNetCore.Hosting; +using Microsoft.AspNetCore.Http; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Hosting; + +namespace ApplicationA +{ + public class Startup + { + // This method gets called by the runtime. Use this method to add services to the container. + // For more information on how to configure your application, visit https://go.microsoft.com/fwlink/?LinkID=398940 + public void ConfigureServices(IServiceCollection services) + { + } + + // This method gets called by the runtime. Use this method to configure the HTTP request pipeline. + public void Configure(IApplicationBuilder app, IWebHostEnvironment env) + { + if (env.IsDevelopment()) + { + app.UseDeveloperExceptionPage(); + } + + app.UseRouting(); + + app.UseEndpoints(endpoints => + { + endpoints.MapGet("/", async context => + { + await context.Response.WriteAsync("Hello from Application A " + Environment.GetEnvironmentVariable("APP_INSTANCE") ?? Environment.GetEnvironmentVariable("HOSTNAME")); + }); + }); + } + } +} diff --git a/samples/nginx-ingress/ApplicationA/appsettings.Development.json b/samples/nginx-ingress/ApplicationA/appsettings.Development.json new file mode 100644 index 00000000..8983e0fc --- /dev/null +++ b/samples/nginx-ingress/ApplicationA/appsettings.Development.json @@ -0,0 +1,9 @@ +{ + "Logging": { + "LogLevel": { + "Default": "Information", + "Microsoft": "Warning", + "Microsoft.Hosting.Lifetime": "Information" + } + } +} diff --git a/samples/nginx-ingress/ApplicationA/appsettings.json b/samples/nginx-ingress/ApplicationA/appsettings.json new file mode 100644 index 00000000..d9d9a9bf --- /dev/null +++ b/samples/nginx-ingress/ApplicationA/appsettings.json @@ -0,0 +1,10 @@ +{ + "Logging": { + "LogLevel": { + "Default": "Information", + "Microsoft": "Warning", + "Microsoft.Hosting.Lifetime": "Information" + } + }, + "AllowedHosts": "*" +} diff --git a/samples/nginx-ingress/ApplicationB/ApplicationB.csproj b/samples/nginx-ingress/ApplicationB/ApplicationB.csproj new file mode 100644 index 00000000..92605c5a --- /dev/null +++ b/samples/nginx-ingress/ApplicationB/ApplicationB.csproj @@ -0,0 +1,7 @@ + + + + netcoreapp3.1 + + + diff --git a/samples/nginx-ingress/ApplicationB/Program.cs b/samples/nginx-ingress/ApplicationB/Program.cs new file mode 100644 index 00000000..8b3c38af --- /dev/null +++ b/samples/nginx-ingress/ApplicationB/Program.cs @@ -0,0 +1,26 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; +using Microsoft.AspNetCore.Hosting; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Logging; + +namespace ApplicationB +{ + public class Program + { + public static void Main(string[] args) + { + CreateHostBuilder(args).Build().Run(); + } + + public static IHostBuilder CreateHostBuilder(string[] args) => + Host.CreateDefaultBuilder(args) + .ConfigureWebHostDefaults(webBuilder => + { + webBuilder.UseStartup(); + }); + } +} diff --git a/samples/nginx-ingress/ApplicationB/Properties/launchSettings.json b/samples/nginx-ingress/ApplicationB/Properties/launchSettings.json new file mode 100644 index 00000000..2b1eba0f --- /dev/null +++ b/samples/nginx-ingress/ApplicationB/Properties/launchSettings.json @@ -0,0 +1,27 @@ +{ + "iisSettings": { + "windowsAuthentication": false, + "anonymousAuthentication": true, + "iisExpress": { + "applicationUrl": "http://localhost:19251", + "sslPort": 44343 + } + }, + "profiles": { + "IIS Express": { + "commandName": "IISExpress", + "launchBrowser": true, + "environmentVariables": { + "ASPNETCORE_ENVIRONMENT": "Development" + } + }, + "ApplicationB": { + "commandName": "Project", + "launchBrowser": true, + "applicationUrl": "https://localhost:5001;http://localhost:5000", + "environmentVariables": { + "ASPNETCORE_ENVIRONMENT": "Development" + } + } + } +} diff --git a/samples/nginx-ingress/ApplicationB/Startup.cs b/samples/nginx-ingress/ApplicationB/Startup.cs new file mode 100644 index 00000000..1a6b68d8 --- /dev/null +++ b/samples/nginx-ingress/ApplicationB/Startup.cs @@ -0,0 +1,40 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; +using Microsoft.AspNetCore.Builder; +using Microsoft.AspNetCore.Hosting; +using Microsoft.AspNetCore.Http; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Hosting; + +namespace ApplicationB +{ + public class Startup + { + // This method gets called by the runtime. Use this method to add services to the container. + // For more information on how to configure your application, visit https://go.microsoft.com/fwlink/?LinkID=398940 + public void ConfigureServices(IServiceCollection services) + { + } + + // This method gets called by the runtime. Use this method to configure the HTTP request pipeline. + public void Configure(IApplicationBuilder app, IWebHostEnvironment env) + { + if (env.IsDevelopment()) + { + app.UseDeveloperExceptionPage(); + } + + app.UseRouting(); + + app.UseEndpoints(endpoints => + { + endpoints.MapGet("/", async context => + { + await context.Response.WriteAsync("Hello from Application B " + Environment.GetEnvironmentVariable("APP_INSTANCE") ?? Environment.GetEnvironmentVariable("HOSTNAME")); + }); + }); + } + } +} diff --git a/samples/nginx-ingress/ApplicationB/appsettings.Development.json b/samples/nginx-ingress/ApplicationB/appsettings.Development.json new file mode 100644 index 00000000..8983e0fc --- /dev/null +++ b/samples/nginx-ingress/ApplicationB/appsettings.Development.json @@ -0,0 +1,9 @@ +{ + "Logging": { + "LogLevel": { + "Default": "Information", + "Microsoft": "Warning", + "Microsoft.Hosting.Lifetime": "Information" + } + } +} diff --git a/samples/nginx-ingress/ApplicationB/appsettings.json b/samples/nginx-ingress/ApplicationB/appsettings.json new file mode 100644 index 00000000..d9d9a9bf --- /dev/null +++ b/samples/nginx-ingress/ApplicationB/appsettings.json @@ -0,0 +1,10 @@ +{ + "Logging": { + "LogLevel": { + "Default": "Information", + "Microsoft": "Warning", + "Microsoft.Hosting.Lifetime": "Information" + } + }, + "AllowedHosts": "*" +} diff --git a/samples/nginx-ingress/nginx-with-ingress.sln b/samples/nginx-ingress/nginx-with-ingress.sln new file mode 100644 index 00000000..3dcc942e --- /dev/null +++ b/samples/nginx-ingress/nginx-with-ingress.sln @@ -0,0 +1,48 @@ + +Microsoft Visual Studio Solution File, Format Version 12.00 +# Visual Studio 15 +VisualStudioVersion = 15.0.26124.0 +MinimumVisualStudioVersion = 15.0.26124.0 +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "ApplicationA", "ApplicationA\ApplicationA.csproj", "{5A9DC239-55BB-4951-B081-35931BF8C867}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "ApplicationB", "ApplicationB\ApplicationB.csproj", "{AE1F10D3-BFAE-4D23-ADCF-06770237285D}" +EndProject +Global + GlobalSection(SolutionConfigurationPlatforms) = preSolution + Debug|Any CPU = Debug|Any CPU + Debug|x64 = Debug|x64 + Debug|x86 = Debug|x86 + Release|Any CPU = Release|Any CPU + Release|x64 = Release|x64 + Release|x86 = Release|x86 + EndGlobalSection + GlobalSection(SolutionProperties) = preSolution + HideSolutionNode = FALSE + EndGlobalSection + GlobalSection(ProjectConfigurationPlatforms) = postSolution + {5A9DC239-55BB-4951-B081-35931BF8C867}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {5A9DC239-55BB-4951-B081-35931BF8C867}.Debug|Any CPU.Build.0 = Debug|Any CPU + {5A9DC239-55BB-4951-B081-35931BF8C867}.Debug|x64.ActiveCfg = Debug|Any CPU + {5A9DC239-55BB-4951-B081-35931BF8C867}.Debug|x64.Build.0 = Debug|Any CPU + {5A9DC239-55BB-4951-B081-35931BF8C867}.Debug|x86.ActiveCfg = Debug|Any CPU + {5A9DC239-55BB-4951-B081-35931BF8C867}.Debug|x86.Build.0 = Debug|Any CPU + {5A9DC239-55BB-4951-B081-35931BF8C867}.Release|Any CPU.ActiveCfg = Release|Any CPU + {5A9DC239-55BB-4951-B081-35931BF8C867}.Release|Any CPU.Build.0 = Release|Any CPU + {5A9DC239-55BB-4951-B081-35931BF8C867}.Release|x64.ActiveCfg = Release|Any CPU + {5A9DC239-55BB-4951-B081-35931BF8C867}.Release|x64.Build.0 = Release|Any CPU + {5A9DC239-55BB-4951-B081-35931BF8C867}.Release|x86.ActiveCfg = Release|Any CPU + {5A9DC239-55BB-4951-B081-35931BF8C867}.Release|x86.Build.0 = Release|Any CPU + {AE1F10D3-BFAE-4D23-ADCF-06770237285D}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {AE1F10D3-BFAE-4D23-ADCF-06770237285D}.Debug|Any CPU.Build.0 = Debug|Any CPU + {AE1F10D3-BFAE-4D23-ADCF-06770237285D}.Debug|x64.ActiveCfg = Debug|Any CPU + {AE1F10D3-BFAE-4D23-ADCF-06770237285D}.Debug|x64.Build.0 = Debug|Any CPU + {AE1F10D3-BFAE-4D23-ADCF-06770237285D}.Debug|x86.ActiveCfg = Debug|Any CPU + {AE1F10D3-BFAE-4D23-ADCF-06770237285D}.Debug|x86.Build.0 = Debug|Any CPU + {AE1F10D3-BFAE-4D23-ADCF-06770237285D}.Release|Any CPU.ActiveCfg = Release|Any CPU + {AE1F10D3-BFAE-4D23-ADCF-06770237285D}.Release|Any CPU.Build.0 = Release|Any CPU + {AE1F10D3-BFAE-4D23-ADCF-06770237285D}.Release|x64.ActiveCfg = Release|Any CPU + {AE1F10D3-BFAE-4D23-ADCF-06770237285D}.Release|x64.Build.0 = Release|Any CPU + {AE1F10D3-BFAE-4D23-ADCF-06770237285D}.Release|x86.ActiveCfg = Release|Any CPU + {AE1F10D3-BFAE-4D23-ADCF-06770237285D}.Release|x86.Build.0 = Release|Any CPU + EndGlobalSection +EndGlobal diff --git a/samples/nginx-ingress/nginx.conf b/samples/nginx-ingress/nginx.conf new file mode 100644 index 00000000..35f8f2e0 --- /dev/null +++ b/samples/nginx-ingress/nginx.conf @@ -0,0 +1,25 @@ +server { + listen 80; + + location /A { + proxy_pass http://appA/; + proxy_http_version 1.1; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection keep-alive; + proxy_set_header Host $host; + proxy_cache_bypass $http_upgrade; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + } + + location /B { + proxy_pass http://appB/; + proxy_http_version 1.1; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection keep-alive; + proxy_set_header Host $host; + proxy_cache_bypass $http_upgrade; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + } +} \ No newline at end of file diff --git a/samples/nginx-ingress/tye.yaml b/samples/nginx-ingress/tye.yaml new file mode 100644 index 00000000..41d290c1 --- /dev/null +++ b/samples/nginx-ingress/tye.yaml @@ -0,0 +1,16 @@ +services: +- name: nginx + image: nginx + bindings: + - protocol: http + volumes: + - source: nginx.conf + target: /etc/nginx/conf.d/default.conf +- name: appA + project: ApplicationA/ApplicationA.csproj + bindings: + replicas: 2 +- name: appB + project: ApplicationB/ApplicationB.csproj + bindings: + replicas: 2 diff --git a/src/Microsoft.Tye.Hosting/AggregateApplicationProcessor.cs b/src/Microsoft.Tye.Hosting/AggregateApplicationProcessor.cs index 900aa069..cd936d83 100644 --- a/src/Microsoft.Tye.Hosting/AggregateApplicationProcessor.cs +++ b/src/Microsoft.Tye.Hosting/AggregateApplicationProcessor.cs @@ -5,6 +5,7 @@ using System; using System.Collections.Generic; using System.Linq; +using System.Runtime.ExceptionServices; using System.Threading.Tasks; using Microsoft.Tye.Hosting.Model; @@ -28,10 +29,28 @@ namespace Microsoft.Tye.Hosting public async Task StopAsync(Application application) { + var exceptions = new List(); + // Shutdown in the opposite order foreach (var processor in _applicationProcessors.Reverse()) { - await processor.StopAsync(application); + try + { + await processor.StopAsync(application); + } + catch (Exception ex) + { + exceptions.Add(ex); + } + } + + if (exceptions.Count == 1) + { + ExceptionDispatchInfo.Throw(exceptions[0]); + } + else if (exceptions.Count > 0) + { + throw new AggregateException(exceptions); } } } diff --git a/src/Microsoft.Tye.Hosting/DockerRunner.cs b/src/Microsoft.Tye.Hosting/DockerRunner.cs index 021676f4..3807226f 100644 --- a/src/Microsoft.Tye.Hosting/DockerRunner.cs +++ b/src/Microsoft.Tye.Hosting/DockerRunner.cs @@ -49,6 +49,49 @@ namespace Microsoft.Tye.Hosting return; } + var proxies = new List(); + foreach (var service in application.Services.Values) + { + if (service.Description.RunInfo is DockerRunInfo || service.Description.Bindings.Count == 0) + { + continue; + } + + // Inject a proxy per non-container service. This allows the container to use normal host names within the + // container network to talk to services on the host + var proxyContanier = new DockerRunInfo($"mcr.microsoft.com/dotnet/core/sdk:3.1", "dotnet Microsoft.Tye.Proxy.dll") + { + WorkingDirectory = "/app", + NetworkAlias = service.Description.Name, + Private = true + }; + var proxyLocation = Path.GetDirectoryName(typeof(Microsoft.Tye.Proxy.Program).Assembly.Location); + proxyContanier.VolumeMappings.Add(new DockerVolume(proxyLocation, name: null, target: "/app")); + var proxyDescription = new ServiceDescription($"{service.Description.Name}-proxy", proxyContanier); + foreach (var binding in service.Description.Bindings) + { + if (binding.Port == null) + { + continue; + } + + var b = new ServiceBinding() + { + ConnectionString = binding.ConnectionString, + Host = binding.Host, + ContainerPort = binding.ContainerPort, + Name = binding.Name, + Port = binding.Port, + Protocol = binding.Protocol + }; + b.ReplicaPorts.Add(b.Port.Value); + proxyDescription.Bindings.Add(b); + } + var proxyContanierService = new Service(proxyDescription); + containers.Add(proxyContanierService); + proxies.Add(proxyContanierService); + } + string? dockerNetwork = null; if (!string.IsNullOrEmpty(application.Network)) @@ -91,6 +134,9 @@ namespace Microsoft.Tye.Hosting await ProcessUtil.RunAsync("docker", command); } + // Stash information outside of the application services + application.Items[typeof(DockerApplicationInformation)] = new DockerApplicationInformation(dockerNetwork, proxies); + var tasks = new Task[containers.Count]; var index = 0; @@ -106,23 +152,34 @@ namespace Microsoft.Tye.Hosting public async Task StopAsync(Application application) { + if (!application.Items.TryGetValue(typeof(DockerApplicationInformation), out var value)) + { + return; + } + + var info = (DockerApplicationInformation)value; + var services = application.Services; var index = 0; - var tasks = new Task[services.Count]; + var tasks = new Task[services.Count + info.Proxies.Count]; foreach (var s in services.Values) { - var state = s; - tasks[index++] = StopContainerAsync(state); + tasks[index++] = StopContainerAsync(s); + } + + foreach (var s in info.Proxies) + { + tasks[index++] = StopContainerAsync(s); } await Task.WhenAll(tasks); - if (string.IsNullOrEmpty(application.Network) && application.Items.TryGetValue("dockerNetwork", out var dockerNetwork)) + if (string.IsNullOrEmpty(application.Network) && !string.IsNullOrEmpty(info.DockerNetwork)) { - _logger.LogInformation("Removing docker network {Network}", dockerNetwork); + _logger.LogInformation("Removing docker network {Network}", info.DockerNetwork); - var command = $"network rm {dockerNetwork}"; + var command = $"network rm {info.DockerNetwork}"; _logger.LogInformation("Running docker command {Command}", command); @@ -189,7 +246,7 @@ namespace Microsoft.Tye.Hosting // These are the ports that the application should use for binding // 1. Tell the docker container what port to bind to - portString = string.Join(" ", ports.Select(p => $"-p {p.Port}:{p.ContainerPort ?? p.Port}")); + portString = docker.Private ? "" : string.Join(" ", ports.Select(p => $"-p {p.Port}:{p.ContainerPort ?? p.Port}")); // 2. Configure ASP.NET Core to bind to those same ports environment["ASPNETCORE_URLS"] = string.Join(";", ports.Select(p => $"{p.Protocol ?? "http"}://*:{p.ContainerPort ?? p.Port}")); @@ -206,6 +263,9 @@ namespace Microsoft.Tye.Hosting // 3. For non-ASP.NET Core apps, pass the same information in the PORT env variable as a semicolon separated list. environment["PORT"] = string.Join(";", ports.Select(p => $"{p.ContainerPort ?? p.Port}")); + + // This the port for the container proxy (containerport:externalport) + environment["PROXY_PORT"] = string.Join(";", ports.Select(p => $"{p.ContainerPort ?? p.Port}:{p.ExternalPort}")); } // See: https://github.com/docker/for-linux/issues/264 @@ -215,6 +275,7 @@ namespace Microsoft.Tye.Hosting application.PopulateEnvironment(service, (key, value) => environment[key] = value, hostname); environment["APP_INSTANCE"] = replica; + environment["CONTAINER_HOST"] = hostname; status.Environment = environment; @@ -284,9 +345,9 @@ namespace Microsoft.Tye.Hosting if (!string.IsNullOrEmpty(dockerNetwork)) { - status.DockerNetworkAlias = serviceDescription.Name; + status.DockerNetworkAlias = docker.NetworkAlias ?? serviceDescription.Name; - var networkCommand = $"network connect {dockerNetwork} {replica} --alias {serviceDescription.Name}"; + var networkCommand = $"network connect {dockerNetwork} {replica} --alias {status.DockerNetworkAlias}"; service.Logs.OnNext($"[{replica}]: docker {networkCommand}"); @@ -478,5 +539,18 @@ namespace Microsoft.Tye.Hosting public Task[] Tasks { get; } public CancellationTokenSource StoppingTokenSource { get; } = new CancellationTokenSource(); } + + private class DockerApplicationInformation + { + public DockerApplicationInformation(string? dockerNetwork, List proxies) + { + DockerNetwork = dockerNetwork; + Proxies = proxies; + } + + public string? DockerNetwork { get; set; } + + public List Proxies { get; } + } } } diff --git a/src/Microsoft.Tye.Hosting/Microsoft.Tye.Hosting.csproj b/src/Microsoft.Tye.Hosting/Microsoft.Tye.Hosting.csproj index 10d54720..c7ce45bf 100644 --- a/src/Microsoft.Tye.Hosting/Microsoft.Tye.Hosting.csproj +++ b/src/Microsoft.Tye.Hosting/Microsoft.Tye.Hosting.csproj @@ -31,6 +31,7 @@ + diff --git a/src/Microsoft.Tye.Hosting/Model/Application.cs b/src/Microsoft.Tye.Hosting/Model/Application.cs index 7469233d..654df917 100644 --- a/src/Microsoft.Tye.Hosting/Model/Application.cs +++ b/src/Microsoft.Tye.Hosting/Model/Application.cs @@ -112,16 +112,14 @@ namespace Microsoft.Tye.Hosting.Model if (b.Port != null) { - var port = (service.Description.RunInfo is DockerRunInfo && - targetService.RunInfo is DockerRunInfo) ? b.ContainerPort ?? b.Port.Value : b.Port.Value; + var port = (service.Description.RunInfo is DockerRunInfo) ? b.ContainerPort ?? b.Port.Value : b.Port.Value; set($"SERVICE__{configName}__PORT", port.ToString()); set($"{envName}_SERVICE_PORT", port.ToString()); } // Use the container name as the host name if there's a single replica (current limitation) - var host = b.Host ?? (service.Description.RunInfo is DockerRunInfo && - targetService.RunInfo is DockerRunInfo ? targetService.Name : defaultHost); + var host = b.Host ?? (service.Description.RunInfo is DockerRunInfo ? targetService.Name : defaultHost); set($"SERVICE__{configName}__HOST", host); set($"{envName}_SERVICE_HOST", host); diff --git a/src/Microsoft.Tye.Hosting/Model/DockerRunInfo.cs b/src/Microsoft.Tye.Hosting/Model/DockerRunInfo.cs index 91f8cec5..6d56ecae 100644 --- a/src/Microsoft.Tye.Hosting/Model/DockerRunInfo.cs +++ b/src/Microsoft.Tye.Hosting/Model/DockerRunInfo.cs @@ -14,6 +14,10 @@ namespace Microsoft.Tye.Hosting.Model Args = args; } + public bool Private { get; set; } + + public string? NetworkAlias { get; set; } + public string? WorkingDirectory { get; set; } public List VolumeMappings { get; } = new List(); diff --git a/src/Microsoft.Tye.Hosting/PortAssigner.cs b/src/Microsoft.Tye.Hosting/PortAssigner.cs index e2778041..3ef5b0af 100644 --- a/src/Microsoft.Tye.Hosting/PortAssigner.cs +++ b/src/Microsoft.Tye.Hosting/PortAssigner.cs @@ -78,22 +78,18 @@ namespace Microsoft.Tye.Hosting binding.Name ?? binding.Protocol); } - // Only set the container port if we're running in a container - if (service.Description.RunInfo is DockerRunInfo) - { - var httpBinding = service.Description.Bindings.FirstOrDefault(b => b.Protocol == "http"); - var httpsBinding = service.Description.Bindings.FirstOrDefault(b => b.Protocol == "https"); + var httpBinding = service.Description.Bindings.FirstOrDefault(b => b.Protocol == "http"); + var httpsBinding = service.Description.Bindings.FirstOrDefault(b => b.Protocol == "https"); - // Default the first http and https port to 80 and 443 - if (httpBinding != null) - { - httpBinding.ContainerPort ??= 80; - } + // Default the first http and https port to 80 and 443 + if (httpBinding != null) + { + httpBinding.ContainerPort ??= 80; + } - if (httpsBinding != null) - { - httpsBinding.ContainerPort ??= 443; - } + if (httpsBinding != null) + { + httpsBinding.ContainerPort ??= 443; } } diff --git a/src/Microsoft.Tye.Hosting/ProxyService.cs b/src/Microsoft.Tye.Hosting/ProxyService.cs index a49288a2..2723c1b2 100644 --- a/src/Microsoft.Tye.Hosting/ProxyService.cs +++ b/src/Microsoft.Tye.Hosting/ProxyService.cs @@ -7,6 +7,7 @@ using System.IO; using System.IO.Pipelines; using System.Net; using System.Net.Sockets; +using System.Runtime.InteropServices; using System.Threading; using System.Threading.Tasks; using Bedrock.Framework; @@ -59,7 +60,12 @@ namespace Microsoft.Tye.Hosting var ports = binding.ReplicaPorts; - sockets.Listen(IPAddress.Loopback, binding.Port.Value, o => + // We need to bind to all interfaces on linux since the container -> host communication won't work + // if we use the IP address to reach out of the host. This works fine on osx and windows + // but doesn't work on linux. + var host = RuntimeInformation.IsOSPlatform(OSPlatform.Linux) ? IPAddress.Any : IPAddress.Loopback; + + sockets.Listen(host, binding.Port.Value, o => { long count = 0; diff --git a/src/Microsoft.Tye.Hosting/TyeHost.cs b/src/Microsoft.Tye.Hosting/TyeHost.cs index 73fb033c..a56e8c3c 100644 --- a/src/Microsoft.Tye.Hosting/TyeHost.cs +++ b/src/Microsoft.Tye.Hosting/TyeHost.cs @@ -281,7 +281,6 @@ namespace Microsoft.Tye.Hosting { await _processor.StopAsync(_application); } - _processor = null; } catch (Exception ex) { @@ -295,6 +294,8 @@ namespace Microsoft.Tye.Hosting await DashboardWebApplication.StopAsync(); } } + + _processor = null; } public async ValueTask DisposeAsync() diff --git a/src/Microsoft.Tye.Proxy/Microsoft.Tye.Proxy.csproj b/src/Microsoft.Tye.Proxy/Microsoft.Tye.Proxy.csproj new file mode 100644 index 00000000..2e1f5e27 --- /dev/null +++ b/src/Microsoft.Tye.Proxy/Microsoft.Tye.Proxy.csproj @@ -0,0 +1,22 @@ + + + + Exe + netcoreapp3.1 + + + + + + + + + + + + + + + + diff --git a/src/Microsoft.Tye.Proxy/Program.cs b/src/Microsoft.Tye.Proxy/Program.cs new file mode 100644 index 00000000..fb5e2ce1 --- /dev/null +++ b/src/Microsoft.Tye.Proxy/Program.cs @@ -0,0 +1,135 @@ +using System; +using System.IO; +using System.IO.Pipelines; +using System.Linq; +using System.Net; +using System.Net.Sockets; +using System.Threading.Tasks; +using Bedrock.Framework; +using Microsoft.AspNetCore.Connections; +using Microsoft.AspNetCore.Connections.Features; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Logging; + +namespace Microsoft.Tye.Proxy +{ + public class Program + { + static async Task Main(string[] args) + { + var serviceName = Environment.GetEnvironmentVariable("APP_INSTANCE"); + var containerHost = Environment.GetEnvironmentVariable("CONTAINER_HOST"); + + using var host = new HostBuilder() + .ConfigureLogging(logging => + { + logging.AddConsole(); + logging.SetMinimumLevel(LogLevel.Debug); + }) + .ConfigureServer(server => + { + var logger = server.ApplicationServices.GetRequiredService>(); + + logger.LogInformation("Received connection information {Host}:{Port}", containerHost, Environment.GetEnvironmentVariable("PROXY_PORT")); + + static (int Port, int ExternalPort) ResolvePort(string portValue) + { + var pair = portValue.Split(':'); + return (int.Parse(pair[0]), int.Parse(pair[1])); + } + + var ports = Environment.GetEnvironmentVariable("PROXY_PORT")?.Split(';').Select(ResolvePort) ?? Enumerable.Empty<(int, int)>(); + + server.UseSockets(sockets => + { + foreach (var mapping in ports) + { + sockets.Listen(IPAddress.Any, mapping.Port, o => + { + // o.UseConnectionLogging("Microsoft.Tye.Proxy"); + + o.Run(async connection => + { + var notificationFeature = connection.Features.Get(); + + NetworkStream? targetStream = null; + + try + { + var target = new Socket(SocketType.Stream, ProtocolType.Tcp) + { + NoDelay = true + }; + + logger.LogDebug("Attempting to connect to {ServiceName} listening on {Port}:{ExternalPort}", serviceName, mapping.Port, mapping.ExternalPort); + + await target.ConnectAsync(containerHost, mapping.ExternalPort); + + logger.LogDebug("Successfully connected to {ServiceName} listening on {Port}:{ExternalPort}", serviceName, mapping.Port, mapping.ExternalPort); + + targetStream = new NetworkStream(target, ownsSocket: true); + } + catch (Exception ex) + { + logger.LogDebug(ex, "Proxy error for service {ServiceName}", serviceName); + + if (targetStream is object) + { + await targetStream.DisposeAsync(); + } + + connection.Abort(); + return; + } + + try + { + logger.LogDebug("Proxying traffic to {ServiceName} {Port}:{ExternalPort}", serviceName, mapping.Port, mapping.ExternalPort); + + // external -> internal + var reading = Task.Run(() => connection.Transport.Input.CopyToAsync(targetStream, notificationFeature.ConnectionClosedRequested)); + + // internal -> external + var writing = Task.Run(() => targetStream.CopyToAsync(connection.Transport.Output, notificationFeature.ConnectionClosedRequested)); + + await Task.WhenAll(reading, writing); + } + catch (ConnectionResetException) + { + // Connection was reset + } + catch (IOException) + { + // Reset can also appear as an IOException with an inner SocketException + } + catch (OperationCanceledException ex) + { + if (!notificationFeature.ConnectionClosedRequested.IsCancellationRequested) + { + logger.LogDebug(0, ex, "Proxy error for service {ServiceName}", serviceName); + } + } + catch (Exception ex) + { + logger.LogDebug(0, ex, "Proxy error for service {ServiceName}", serviceName); + } + finally + { + await targetStream.DisposeAsync(); + } + + // This needs to reconnect to the target port(s) until its bound + // it has to stop if the service is no longer running + }); + + }); + } + }); + }) + .Build(); + + await host.RunAsync(); + } + } +} diff --git a/test/E2ETest/TyeRunTests.cs b/test/E2ETest/TyeRunTests.cs index 9848b137..2ca7803b 100644 --- a/test/E2ETest/TyeRunTests.cs +++ b/test/E2ETest/TyeRunTests.cs @@ -439,9 +439,15 @@ namespace E2ETest await RunHostingApplication(application, Array.Empty(), async (app, uri) => { - using var client = new HttpClient(); - var ingressUri = await GetServiceUrl(client, uri, "ingress"); + var appAUri = await GetServiceUrl(client, uri, "appA"); + var appBUri = await GetServiceUrl(client, uri, "appB"); + + var appAResponse = await client.GetAsync(appAUri); + var appBResponse = await client.GetAsync(appBUri); + + Assert.True(appAResponse.IsSuccessStatusCode); + Assert.True(appBResponse.IsSuccessStatusCode); var responseA = await client.GetAsync(ingressUri + "/A"); var responseB = await client.GetAsync(ingressUri + "/B"); @@ -462,6 +468,46 @@ namespace E2ETest }); } + [ConditionalFact] + [SkipIfDockerNotRunning] + public async Task NginxIngressTest() + { + using var projectDirectory = CopySampleProjectDirectory("nginx-ingress"); + + var projectFile = new FileInfo(Path.Combine(projectDirectory.DirectoryPath, "tye.yaml")); + var outputContext = new OutputContext(_sink, Verbosity.Debug); + var application = await ApplicationFactory.CreateAsync(outputContext, projectFile); + + var handler = new HttpClientHandler + { + ServerCertificateCustomValidationCallback = (a, b, c, d) => true, + AllowAutoRedirect = false + }; + + var client = new HttpClient(new RetryHandler(handler)); + + await RunHostingApplication(application, Array.Empty(), async (app, uri) => + { + var nginxUri = await GetServiceUrl(client, uri, "nginx"); + var appAUri = await GetServiceUrl(client, uri, "appA"); + var appBUri = await GetServiceUrl(client, uri, "appB"); + + var nginxResponse = await client.GetAsync(nginxUri); + var appAResponse = await client.GetAsync(appAUri); + var appBResponse = await client.GetAsync(appBUri); + + Assert.Equal(HttpStatusCode.NotFound, nginxResponse.StatusCode); + Assert.True(appAResponse.IsSuccessStatusCode); + Assert.True(appBResponse.IsSuccessStatusCode); + + var responseA = await client.GetAsync(nginxUri + "/A"); + var responseB = await client.GetAsync(nginxUri + "/B"); + + Assert.StartsWith("Hello from Application A", await responseA.Content.ReadAsStringAsync()); + Assert.StartsWith("Hello from Application B", await responseB.Content.ReadAsStringAsync()); + }); + } + [Fact] public async Task NullDebugTargetsDoesNotThrow() { diff --git a/tye.sln b/tye.sln index 766b7725..572423bc 100644 --- a/tye.sln +++ b/tye.sln @@ -11,17 +11,19 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "test", "test", "{F19B02EB-A EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "E2ETest", "test\E2ETest\E2ETest.csproj", "{D15E5FF6-C1E7-4110-A2BE-06ADA7ACA82B}" EndProject -Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Tye.Hosting", "src\Microsoft.Tye.Hosting\Microsoft.Tye.Hosting.csproj", "{0F4F5A86-DD27-4AF9-BF97-221EAD5040E7}" +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.Tye.Hosting", "src\Microsoft.Tye.Hosting\Microsoft.Tye.Hosting.csproj", "{0F4F5A86-DD27-4AF9-BF97-221EAD5040E7}" EndProject -Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Tye.Hosting.Diagnostics", "src\Microsoft.Tye.Hosting.Diagnostics\Microsoft.Tye.Hosting.Diagnostics.csproj", "{CEBFC149-8162-4A0A-9AD4-40498B9172CD}" +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.Tye.Hosting.Diagnostics", "src\Microsoft.Tye.Hosting.Diagnostics\Microsoft.Tye.Hosting.Diagnostics.csproj", "{CEBFC149-8162-4A0A-9AD4-40498B9172CD}" EndProject -Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Tye.Hosting.Runtime", "src\Microsoft.Tye.Hosting.Runtime\Microsoft.Tye.Hosting.Runtime.csproj", "{34719884-1338-4965-BA2A-F98DB03733C2}" +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.Tye.Hosting.Runtime", "src\Microsoft.Tye.Hosting.Runtime\Microsoft.Tye.Hosting.Runtime.csproj", "{34719884-1338-4965-BA2A-F98DB03733C2}" EndProject -Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Tye.Core", "src\Microsoft.Tye.Core\Microsoft.Tye.Core.csproj", "{D0359C69-6EA9-4B03-9455-90E8E04F1CB0}" +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.Tye.Core", "src\Microsoft.Tye.Core\Microsoft.Tye.Core.csproj", "{D0359C69-6EA9-4B03-9455-90E8E04F1CB0}" EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Microsoft.Tye.Extensions", "src\Microsoft.Tye.Extensions\Microsoft.Tye.Extensions.csproj", "{AAF0CE0B-E53A-4E10-AA82-BF7200AB2B0C}" +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.Tye.Extensions", "src\Microsoft.Tye.Extensions\Microsoft.Tye.Extensions.csproj", "{AAF0CE0B-E53A-4E10-AA82-BF7200AB2B0C}" EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Microsoft.Tye.Extensions.Configuration", "src\Microsoft.Tye.Extensions.Configuration\Microsoft.Tye.Extensions.Configuration.csproj", "{B07394E4-30A7-429A-BC5A-747B54D5A447}" +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.Tye.Extensions.Configuration", "src\Microsoft.Tye.Extensions.Configuration\Microsoft.Tye.Extensions.Configuration.csproj", "{B07394E4-30A7-429A-BC5A-747B54D5A447}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Microsoft.Tye.Proxy", "src\Microsoft.Tye.Proxy\Microsoft.Tye.Proxy.csproj", "{7C9021B7-64BA-4DA9-88DA-5BC12A1C6233}" EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution @@ -129,6 +131,18 @@ Global {B07394E4-30A7-429A-BC5A-747B54D5A447}.Release|x64.Build.0 = Release|Any CPU {B07394E4-30A7-429A-BC5A-747B54D5A447}.Release|x86.ActiveCfg = Release|Any CPU {B07394E4-30A7-429A-BC5A-747B54D5A447}.Release|x86.Build.0 = Release|Any CPU + {7C9021B7-64BA-4DA9-88DA-5BC12A1C6233}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {7C9021B7-64BA-4DA9-88DA-5BC12A1C6233}.Debug|Any CPU.Build.0 = Debug|Any CPU + {7C9021B7-64BA-4DA9-88DA-5BC12A1C6233}.Debug|x64.ActiveCfg = Debug|Any CPU + {7C9021B7-64BA-4DA9-88DA-5BC12A1C6233}.Debug|x64.Build.0 = Debug|Any CPU + {7C9021B7-64BA-4DA9-88DA-5BC12A1C6233}.Debug|x86.ActiveCfg = Debug|Any CPU + {7C9021B7-64BA-4DA9-88DA-5BC12A1C6233}.Debug|x86.Build.0 = Debug|Any CPU + {7C9021B7-64BA-4DA9-88DA-5BC12A1C6233}.Release|Any CPU.ActiveCfg = Release|Any CPU + {7C9021B7-64BA-4DA9-88DA-5BC12A1C6233}.Release|Any CPU.Build.0 = Release|Any CPU + {7C9021B7-64BA-4DA9-88DA-5BC12A1C6233}.Release|x64.ActiveCfg = Release|Any CPU + {7C9021B7-64BA-4DA9-88DA-5BC12A1C6233}.Release|x64.Build.0 = Release|Any CPU + {7C9021B7-64BA-4DA9-88DA-5BC12A1C6233}.Release|x86.ActiveCfg = Release|Any CPU + {7C9021B7-64BA-4DA9-88DA-5BC12A1C6233}.Release|x86.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE @@ -142,6 +156,7 @@ Global {D0359C69-6EA9-4B03-9455-90E8E04F1CB0} = {8C662D59-A3CB-466F-8E85-A8E6BA5E7601} {AAF0CE0B-E53A-4E10-AA82-BF7200AB2B0C} = {8C662D59-A3CB-466F-8E85-A8E6BA5E7601} {B07394E4-30A7-429A-BC5A-747B54D5A447} = {8C662D59-A3CB-466F-8E85-A8E6BA5E7601} + {7C9021B7-64BA-4DA9-88DA-5BC12A1C6233} = {8C662D59-A3CB-466F-8E85-A8E6BA5E7601} EndGlobalSection GlobalSection(ExtensibilityGlobals) = postSolution SolutionGuid = {D8002603-BB27-4500-BF86-274A8E72D302}