From 1227c5bd1dbd622a1ba0ac8cd0952ed736f95ef0 Mon Sep 17 00:00:00 2001 From: David Fowler Date: Sat, 2 May 2020 01:15:09 -0700 Subject: [PATCH] Ingress improvements - Support a single ingress, not many - Default http and https bindings if none specified - Treat null path the same as / - Make all services private when an ingress is present - Run ingress in a container when --docker is run - Hide bindings in the dasboard for private services --- samples/apps-with-ingress/tye.yaml | 24 +++--- samples/frontend-backend/frontend/Startup.cs | 1 + samples/frontend-backend/tye.yaml | 7 ++ src/Microsoft.Tye.Core/ApplicationBuilder.cs | 2 +- src/Microsoft.Tye.Core/ApplicationExecutor.cs | 3 +- src/Microsoft.Tye.Core/ApplicationFactory.cs | 29 ++++--- .../ApplicationYamlWriter.cs | 5 +- .../ConfigModel/ConfigApplication.cs | 6 +- .../Serialization/ConfigApplicationParser.cs | 6 +- .../Serialization/ConfigIngressParser.cs | 13 +-- .../Serialization/YamlParser.cs | 6 +- .../Dashboard/Pages/Index.razor | 6 +- src/Microsoft.Tye.Hosting/DockerRunner.cs | 4 +- src/Microsoft.Tye.Hosting/HttpProxyService.cs | 11 +-- .../Microsoft.Tye.Hosting.csproj | 1 + .../Model/DockerRunInfo.cs | 2 - .../Model/ProjectRunInfo.cs | 2 - src/Microsoft.Tye.Hosting/Model/RunInfo.cs | 1 + src/Microsoft.Tye.Hosting/ProcessRunner.cs | 2 +- .../TransformProjectsIntoContainers.cs | 79 +++++++++++++++++- .../ConnectionRetryHandler.cs | 4 +- .../IngressHostMatcherPolicy.cs | 2 +- .../IngressHostMetadata.cs | 2 +- .../Microsoft.Tye.HttpProxy.csproj | 18 ++++ src/Microsoft.Tye.HttpProxy/Program.cs | 26 ++++++ .../Properties/launchSettings.json | 27 ++++++ .../ProxyExtensions.cs | 2 +- .../RoundRobinLoadBalancer.cs | 72 ++++++++++++++++ src/Microsoft.Tye.HttpProxy/Startup.cs | 82 +++++++++++++++++++ .../appsettings.Development.json | 9 ++ src/Microsoft.Tye.HttpProxy/appsettings.json | 10 +++ src/tye/ApplicationBuilderExtensions.cs | 65 ++++++++------- .../projects/apps-with-ingress/tye.yaml | 24 +++--- test/Test.Infrastructure/TyeAssert.cs | 13 +-- tye.sln | 19 ++++- 35 files changed, 471 insertions(+), 114 deletions(-) rename src/{Microsoft.Tye.Hosting/Infrastructure => Microsoft.Tye.HttpProxy}/ConnectionRetryHandler.cs (95%) rename src/{Microsoft.Tye.Hosting/Infrastructure => Microsoft.Tye.HttpProxy}/IngressHostMatcherPolicy.cs (98%) rename src/{Microsoft.Tye.Hosting/Infrastructure => Microsoft.Tye.HttpProxy}/IngressHostMetadata.cs (93%) create mode 100644 src/Microsoft.Tye.HttpProxy/Microsoft.Tye.HttpProxy.csproj create mode 100644 src/Microsoft.Tye.HttpProxy/Program.cs create mode 100644 src/Microsoft.Tye.HttpProxy/Properties/launchSettings.json rename src/{Microsoft.Tye.Hosting/Infrastructure => Microsoft.Tye.HttpProxy}/ProxyExtensions.cs (99%) create mode 100644 src/Microsoft.Tye.HttpProxy/RoundRobinLoadBalancer.cs create mode 100644 src/Microsoft.Tye.HttpProxy/Startup.cs create mode 100644 src/Microsoft.Tye.HttpProxy/appsettings.Development.json create mode 100644 src/Microsoft.Tye.HttpProxy/appsettings.json diff --git a/samples/apps-with-ingress/tye.yaml b/samples/apps-with-ingress/tye.yaml index c6aa8cca..8c515b13 100644 --- a/samples/apps-with-ingress/tye.yaml +++ b/samples/apps-with-ingress/tye.yaml @@ -6,18 +6,18 @@ # name: apps-with-ingress ingress: - - name: ingress - bindings: - - port: 8080 - rules: - - path: /A - service: app-a - - path: /B - service: app-b - - host: a.example.com - service: app-a - - host: b.example.com - service: app-b + name: ingress + bindings: + - port: 8080 + rules: + - path: /A + service: app-a + - path: /B + service: app-b + - host: a.example.com + service: app-a + - host: b.example.com + service: app-b services: - name: app-a diff --git a/samples/frontend-backend/frontend/Startup.cs b/samples/frontend-backend/frontend/Startup.cs index 197e7001..0973b5b7 100644 --- a/samples/frontend-backend/frontend/Startup.cs +++ b/samples/frontend-backend/frontend/Startup.cs @@ -66,6 +66,7 @@ namespace Frontend await context.Response.WriteAsync($"Frontend Listening IP: {context.Connection.LocalIpAddress}{Environment.NewLine}"); await context.Response.WriteAsync($"Frontend Hostname: {Dns.GetHostName()}{Environment.NewLine}"); await context.Response.WriteAsync($"EnvVar Configuration value: {Configuration["App:Value"]}{Environment.NewLine}"); + await context.Response.WriteAsync($"Instance ID: {Configuration["APP_INSTANCE"]}{Environment.NewLine}"); await context.Response.WriteAsync($"Backend Listening IP: {backendInfo.IP}{Environment.NewLine}"); await context.Response.WriteAsync($"Backend Hostname: {backendInfo.Hostname}{Environment.NewLine}"); diff --git a/samples/frontend-backend/tye.yaml b/samples/frontend-backend/tye.yaml index 8868a09a..36720dbb 100644 --- a/samples/frontend-backend/tye.yaml +++ b/samples/frontend-backend/tye.yaml @@ -1,8 +1,15 @@ # tye application configuration file # read all about it at https://github.com/dotnet/tye name: frontend-backend + +ingress: + name: ingress + rules: + - service: frontend + services: - name: backend project: backend/backend.csproj - name: frontend project: frontend/frontend.csproj + replicas: 2 diff --git a/src/Microsoft.Tye.Core/ApplicationBuilder.cs b/src/Microsoft.Tye.Core/ApplicationBuilder.cs index c1400492..440b3097 100644 --- a/src/Microsoft.Tye.Core/ApplicationBuilder.cs +++ b/src/Microsoft.Tye.Core/ApplicationBuilder.cs @@ -27,7 +27,7 @@ namespace Microsoft.Tye public List Services { get; } = new List(); - public List Ingress { get; } = new List(); + public IngressBuilder? Ingress { get; set; } public string? Network { get; set; } } diff --git a/src/Microsoft.Tye.Core/ApplicationExecutor.cs b/src/Microsoft.Tye.Core/ApplicationExecutor.cs index c18ebb5d..a8ad5267 100644 --- a/src/Microsoft.Tye.Core/ApplicationExecutor.cs +++ b/src/Microsoft.Tye.Core/ApplicationExecutor.cs @@ -39,7 +39,8 @@ namespace Microsoft.Tye tracker.MarkComplete(); } - foreach (var ingress in application.Ingress) + var ingress = application.Ingress; + if (ingress != null) { using var tracker = output.BeginStep($"Processing Ingress '{ingress.Name}'..."); foreach (var step in IngressSteps) diff --git a/src/Microsoft.Tye.Core/ApplicationFactory.cs b/src/Microsoft.Tye.Core/ApplicationFactory.cs index 53a5b17d..5e161a36 100644 --- a/src/Microsoft.Tye.Core/ApplicationFactory.cs +++ b/src/Microsoft.Tye.Core/ApplicationFactory.cs @@ -289,22 +289,31 @@ namespace Microsoft.Tye } } - foreach (var configIngress in config.Ingress) + if (config.Ingress != null) { + var configIngress = config.Ingress; + var ingress = new IngressBuilder(configIngress.Name!); + root.Ingress = ingress; ingress.Replicas = configIngress.Replicas ?? 1; - root.Ingress.Add(ingress); - - foreach (var configBinding in configIngress.Bindings) + if (configIngress.Bindings.Count > 0) { - var binding = new IngressBindingBuilder() + foreach (var configBinding in configIngress.Bindings) { - Name = configBinding.Name, - Port = configBinding.Port, - Protocol = configBinding.Protocol ?? "http", - }; - ingress.Bindings.Add(binding); + var binding = new IngressBindingBuilder() + { + Name = configBinding.Name, + Port = configBinding.Port, + Protocol = configBinding.Protocol ?? "http", + }; + ingress.Bindings.Add(binding); + } + } + else + { + ingress.Bindings.Add(new IngressBindingBuilder() { Protocol = "http" }); + ingress.Bindings.Add(new IngressBindingBuilder() { Name = "https", Protocol = "https" }); } foreach (var configRule in configIngress.Rules) diff --git a/src/Microsoft.Tye.Core/ApplicationYamlWriter.cs b/src/Microsoft.Tye.Core/ApplicationYamlWriter.cs index 50350f83..47d4aa55 100644 --- a/src/Microsoft.Tye.Core/ApplicationYamlWriter.cs +++ b/src/Microsoft.Tye.Core/ApplicationYamlWriter.cs @@ -17,7 +17,10 @@ namespace Microsoft.Tye var yaml = new List(); yaml.AddRange(application.Services.SelectMany(s => s.Outputs.OfType())); - yaml.AddRange(application.Ingress.SelectMany(i => i.Outputs.OfType())); + if (application.Ingress != null) + { + yaml.AddRange(application.Ingress.Outputs.OfType()); + } if (yaml.Count == 0) { diff --git a/src/Microsoft.Tye.Core/ConfigModel/ConfigApplication.cs b/src/Microsoft.Tye.Core/ConfigModel/ConfigApplication.cs index 70d4e5d4..73b1d05e 100644 --- a/src/Microsoft.Tye.Core/ConfigModel/ConfigApplication.cs +++ b/src/Microsoft.Tye.Core/ConfigModel/ConfigApplication.cs @@ -34,7 +34,7 @@ namespace Microsoft.Tye.ConfigModel public List Services { get; set; } = new List(); - public List Ingress { get; set; } = new List(); + public ConfigIngress? Ingress { get; set; } public void Validate() { @@ -137,7 +137,9 @@ namespace Microsoft.Tye.ConfigModel } } - foreach (var ingress in config.Ingress) + var ingress = config.Ingress; + + if (ingress != null) { context = new ValidationContext(ingress); if (!Validator.TryValidateObject(ingress, context, results, validateAllProperties: true)) diff --git a/src/Microsoft.Tye.Core/Serialization/ConfigApplicationParser.cs b/src/Microsoft.Tye.Core/Serialization/ConfigApplicationParser.cs index 4988d2e1..f4a37c47 100644 --- a/src/Microsoft.Tye.Core/Serialization/ConfigApplicationParser.cs +++ b/src/Microsoft.Tye.Core/Serialization/ConfigApplicationParser.cs @@ -30,8 +30,10 @@ namespace Tye.Serialization app.Registry = YamlParser.GetScalarValue(key, child.Value); break; case "ingress": - YamlParser.ThrowIfNotYamlSequence(key, child.Value); - ConfigIngressParser.HandleIngress((child.Value as YamlSequenceNode)!, app.Ingress); + YamlParser.ThrowIfNotYamlMapping(child.Value); + var ingress = new ConfigIngress(); + ConfigIngressParser.HandleIngressMapping((child.Value as YamlMappingNode)!, ingress); + app.Ingress = ingress; break; case "services": YamlParser.ThrowIfNotYamlSequence(key, child.Value); diff --git a/src/Microsoft.Tye.Core/Serialization/ConfigIngressParser.cs b/src/Microsoft.Tye.Core/Serialization/ConfigIngressParser.cs index 6ccc391c..94d2e5e1 100644 --- a/src/Microsoft.Tye.Core/Serialization/ConfigIngressParser.cs +++ b/src/Microsoft.Tye.Core/Serialization/ConfigIngressParser.cs @@ -10,18 +10,7 @@ namespace Tye.Serialization { public static class ConfigIngressParser { - public static void HandleIngress(YamlSequenceNode yamlSequenceNode, List ingress) - { - foreach (var child in yamlSequenceNode.Children) - { - YamlParser.ThrowIfNotYamlMapping(child); - var configIngress = new ConfigIngress(); - HandleIngressMapping((YamlMappingNode)child, configIngress); - ingress.Add(configIngress); - } - } - - private static void HandleIngressMapping(YamlMappingNode yamlMappingNode, ConfigIngress configIngress) + public static void HandleIngressMapping(YamlMappingNode yamlMappingNode, ConfigIngress configIngress) { foreach (var child in yamlMappingNode!.Children) { diff --git a/src/Microsoft.Tye.Core/Serialization/YamlParser.cs b/src/Microsoft.Tye.Core/Serialization/YamlParser.cs index 78b22470..14d14811 100644 --- a/src/Microsoft.Tye.Core/Serialization/YamlParser.cs +++ b/src/Microsoft.Tye.Core/Serialization/YamlParser.cs @@ -63,10 +63,10 @@ namespace Tye.Serialization service.Volumes ??= new List(); } - foreach (var ingress in app.Ingress) + if (app.Ingress != null) { - ingress.Bindings ??= new List(); - ingress.Rules ??= new List(); + app.Ingress.Bindings ??= new List(); + app.Ingress.Rules ??= new List(); } return app; diff --git a/src/Microsoft.Tye.Hosting/Dashboard/Pages/Index.razor b/src/Microsoft.Tye.Hosting/Dashboard/Pages/Index.razor index 8b9970b4..6185c3dd 100644 --- a/src/Microsoft.Tye.Hosting/Dashboard/Pages/Index.razor +++ b/src/Microsoft.Tye.Hosting/Dashboard/Pages/Index.razor @@ -36,7 +36,11 @@ } - @if (service.Description.Bindings.Any()) + @if (service.Description.RunInfo!.Private) + { +

Cluster IP

+ } + else if (service.Description.Bindings.Any()) { foreach (var b in service.Description.Bindings) { diff --git a/src/Microsoft.Tye.Hosting/DockerRunner.cs b/src/Microsoft.Tye.Hosting/DockerRunner.cs index 0006ed26..7e1d6728 100644 --- a/src/Microsoft.Tye.Hosting/DockerRunner.cs +++ b/src/Microsoft.Tye.Hosting/DockerRunner.cs @@ -52,7 +52,9 @@ namespace Microsoft.Tye.Hosting var proxies = new List(); foreach (var service in application.Services.Values) { - if (service.Description.RunInfo is DockerRunInfo || service.Description.Bindings.Count == 0) + if (service.Description.RunInfo is DockerRunInfo || + service.Description.RunInfo is IngressRunInfo || + service.Description.Bindings.Count == 0) { continue; } diff --git a/src/Microsoft.Tye.Hosting/HttpProxyService.cs b/src/Microsoft.Tye.Hosting/HttpProxyService.cs index 8148ae73..9f14c523 100644 --- a/src/Microsoft.Tye.Hosting/HttpProxyService.cs +++ b/src/Microsoft.Tye.Hosting/HttpProxyService.cs @@ -82,6 +82,8 @@ namespace Microsoft.Tye.Hosting service.ReplicaEvents.OnNext(new ReplicaEvent(ReplicaState.Added, status)); } + _logger.LogInformation("Ingress bound to {Addresses}", string.Join(", ", addresses)); + builder.Server.UseUrls(addresses.ToArray()); var webApp = builder.Build(); @@ -95,8 +97,6 @@ namespace Microsoft.Tye.Hosting continue; } - _logger.LogInformation("Processing ingress rule: Path:{Path}, Host:{Host}, Service:{Service}", rule.Path, rule.Host, rule.Service); - var targetServiceDescription = target.Description; var uris = new List(); @@ -113,15 +113,12 @@ namespace Microsoft.Tye.Hosting // For each of the target service replicas, get the base URL // based on the replica port - for (int i = 0; i < targetServiceDescription.Replicas; i++) + foreach (var port in targetBinding.ReplicaPorts) { - var port = targetBinding.ReplicaPorts[i]; var url = $"{targetBinding.Protocol}://localhost:{port}"; uris.Add(new Uri(url)); } - _logger.LogInformation("Service {ServiceName} is using {Urls}", targetServiceDescription.Name, string.Join(",", uris.Select(u => u.ToString()))); - // The only load balancing strategy here is round robin long count = 0; RequestDelegate del = context => @@ -130,7 +127,7 @@ namespace Microsoft.Tye.Hosting var uri = new UriBuilder(uris[next]) { - Path = (string)context.Request.RouteValues["path"] + Path = (string)context.Request.RouteValues["path"] ?? "/" }; return context.ProxyRequest(invoker, uri.Uri); diff --git a/src/Microsoft.Tye.Hosting/Microsoft.Tye.Hosting.csproj b/src/Microsoft.Tye.Hosting/Microsoft.Tye.Hosting.csproj index d9a13d02..78a773c8 100644 --- a/src/Microsoft.Tye.Hosting/Microsoft.Tye.Hosting.csproj +++ b/src/Microsoft.Tye.Hosting/Microsoft.Tye.Hosting.csproj @@ -29,6 +29,7 @@ + diff --git a/src/Microsoft.Tye.Hosting/Model/DockerRunInfo.cs b/src/Microsoft.Tye.Hosting/Model/DockerRunInfo.cs index 7ea1d170..c15be6d5 100644 --- a/src/Microsoft.Tye.Hosting/Model/DockerRunInfo.cs +++ b/src/Microsoft.Tye.Hosting/Model/DockerRunInfo.cs @@ -2,7 +2,6 @@ // The .NET Foundation licenses this file to you under the MIT license. // See the LICENSE file in the project root for more information. -using System; using System.Collections.Generic; using System.IO; @@ -16,7 +15,6 @@ namespace Microsoft.Tye.Hosting.Model Args = args; } - public bool Private { get; set; } public bool IsAspNet { get; set; } public string? NetworkAlias { get; set; } diff --git a/src/Microsoft.Tye.Hosting/Model/ProjectRunInfo.cs b/src/Microsoft.Tye.Hosting/Model/ProjectRunInfo.cs index 59d054ab..46cde427 100644 --- a/src/Microsoft.Tye.Hosting/Model/ProjectRunInfo.cs +++ b/src/Microsoft.Tye.Hosting/Model/ProjectRunInfo.cs @@ -46,8 +46,6 @@ namespace Microsoft.Tye.Hosting.Model public string RunCommand { get; } public string RunArguments { get; } - - // This exists for running projects as containers public List VolumeMappings { get; } = new List(); } } diff --git a/src/Microsoft.Tye.Hosting/Model/RunInfo.cs b/src/Microsoft.Tye.Hosting/Model/RunInfo.cs index a015751a..db227718 100644 --- a/src/Microsoft.Tye.Hosting/Model/RunInfo.cs +++ b/src/Microsoft.Tye.Hosting/Model/RunInfo.cs @@ -6,5 +6,6 @@ namespace Microsoft.Tye.Hosting.Model { public abstract class RunInfo { + public bool Private { get; set; } } } diff --git a/src/Microsoft.Tye.Hosting/ProcessRunner.cs b/src/Microsoft.Tye.Hosting/ProcessRunner.cs index 30eb38ab..c8a07e37 100644 --- a/src/Microsoft.Tye.Hosting/ProcessRunner.cs +++ b/src/Microsoft.Tye.Hosting/ProcessRunner.cs @@ -273,7 +273,7 @@ namespace Microsoft.Tye.Hosting errorDataReceived: data => service.Logs.OnNext($"[{replica}]: {data}"), onStart: pid => { - if (hasPorts) + if (hasPorts && !serviceDescription.RunInfo!.Private) { _logger.LogInformation("{ServiceName} running on process id {PID} bound to {Address}", replica, pid, string.Join(", ", ports.Select(p => $"{p.Protocol ?? "http"}://localhost:{p.Port}"))); } diff --git a/src/Microsoft.Tye.Hosting/TransformProjectsIntoContainers.cs b/src/Microsoft.Tye.Hosting/TransformProjectsIntoContainers.cs index c4ccc9b1..a60472f5 100644 --- a/src/Microsoft.Tye.Hosting/TransformProjectsIntoContainers.cs +++ b/src/Microsoft.Tye.Hosting/TransformProjectsIntoContainers.cs @@ -1,7 +1,6 @@ // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. // See the LICENSE file in the project root for more information. - using System; using System.Collections.Generic; using System.IO; @@ -33,11 +32,89 @@ namespace Microsoft.Tye.Hosting { tasks.Add(TransformProjectToContainer(s, project)); } + else if (s.Description.RunInfo is IngressRunInfo ingress) + { + tasks.Add(TransformIngressToContainer(application, s, ingress)); + } } return Task.WhenAll(tasks); } + private Task TransformIngressToContainer(Application application, Service service, IngressRunInfo ingress) + { + // Inject a proxy per non-container service. This allows the container to use normal host names within the + // container network to talk to services on the host + var ingressRunInfo = new DockerRunInfo($"mcr.microsoft.com/dotnet/core/aspnet:3.1", "dotnet Microsoft.Tye.HttpProxy.dll") + { + WorkingDirectory = "/app", + IsAspNet = true + }; + + var proxyLocation = Path.GetDirectoryName(typeof(Microsoft.Tye.HttpProxy.Program).Assembly.Location); + ingressRunInfo.VolumeMappings.Add(new DockerVolume(proxyLocation, name: null, target: "/app")); + + int ruleIndex = 0; + foreach (var rule in ingress.Rules) + { + if (!application.Services.TryGetValue(rule.Service, out var target)) + { + continue; + } + + var targetServiceDescription = target.Description; + + var uris = new List(); + + // HTTP before HTTPS (this might change once we figure out certs...) + var targetBinding = targetServiceDescription.Bindings.FirstOrDefault(b => b.Protocol == "http") ?? + targetServiceDescription.Bindings.FirstOrDefault(b => b.Protocol == "https"); + + if (targetBinding == null) + { + _logger.LogInformation("Service {ServiceName} does not have any HTTP or HTTPs bindings", targetServiceDescription.Name); + continue; + } + + // For each of the target service replicas, get the base URL + // based on the replica port + foreach (var port in targetBinding.ReplicaPorts) + { + var url = $"{targetBinding.Protocol}://localhost:{port}"; + uris.Add(new Uri(url)); + } + + // TODO: Use Yarp + // Configuration schema + // "Rules": + // { + // "0": + // { + // "Host": null, + // "Path": null, + // "Service": "frontend", + // "Port": 10067", + // "Protocol": http + // } + // } + + var rulePrefix = $"Rules__{ruleIndex}__"; + + service.Description.Configuration.Add(new EnvironmentVariable($"{rulePrefix}Host", rule.Host)); + service.Description.Configuration.Add(new EnvironmentVariable($"{rulePrefix}Path", rule.Path)); + service.Description.Configuration.Add(new EnvironmentVariable($"{rulePrefix}Service", rule.Service)); + service.Description.Configuration.Add(new EnvironmentVariable($"{rulePrefix}Port", (targetBinding.ContainerPort ?? targetBinding.Port).ToString())); + service.Description.Configuration.Add(new EnvironmentVariable($"{rulePrefix}Protocol", targetBinding.Protocol)); + + ruleIndex++; + } + + + service.Description.RunInfo = ingressRunInfo; + + return Task.CompletedTask; + } + private async Task TransformProjectToContainer(Service service, ProjectRunInfo project) { var serviceDescription = service.Description; diff --git a/src/Microsoft.Tye.Hosting/Infrastructure/ConnectionRetryHandler.cs b/src/Microsoft.Tye.HttpProxy/ConnectionRetryHandler.cs similarity index 95% rename from src/Microsoft.Tye.Hosting/Infrastructure/ConnectionRetryHandler.cs rename to src/Microsoft.Tye.HttpProxy/ConnectionRetryHandler.cs index 43091d40..07dd1011 100644 --- a/src/Microsoft.Tye.Hosting/Infrastructure/ConnectionRetryHandler.cs +++ b/src/Microsoft.Tye.HttpProxy/ConnectionRetryHandler.cs @@ -10,9 +10,9 @@ using System.Runtime.ExceptionServices; using System.Threading; using System.Threading.Tasks; -namespace Microsoft.Tye.Hosting +namespace System.Net.Http { - internal class ConnectionRetryHandler : DelegatingHandler + public class ConnectionRetryHandler : DelegatingHandler { private static readonly int MaxRetries = 3; private static readonly TimeSpan InitialRetryDelay = TimeSpan.FromMilliseconds(1000); diff --git a/src/Microsoft.Tye.Hosting/Infrastructure/IngressHostMatcherPolicy.cs b/src/Microsoft.Tye.HttpProxy/IngressHostMatcherPolicy.cs similarity index 98% rename from src/Microsoft.Tye.Hosting/Infrastructure/IngressHostMatcherPolicy.cs rename to src/Microsoft.Tye.HttpProxy/IngressHostMatcherPolicy.cs index 469fde36..0c7d0082 100644 --- a/src/Microsoft.Tye.Hosting/Infrastructure/IngressHostMatcherPolicy.cs +++ b/src/Microsoft.Tye.HttpProxy/IngressHostMatcherPolicy.cs @@ -10,7 +10,7 @@ using Microsoft.AspNetCore.Http; namespace Microsoft.AspNetCore.Routing.Matching { - internal sealed class IngressHostMatcherPolicy : MatcherPolicy, IEndpointSelectorPolicy + public sealed class IngressHostMatcherPolicy : MatcherPolicy, IEndpointSelectorPolicy { private const string WildcardHost = "*"; private const string WildcardPrefix = "*."; diff --git a/src/Microsoft.Tye.Hosting/Infrastructure/IngressHostMetadata.cs b/src/Microsoft.Tye.HttpProxy/IngressHostMetadata.cs similarity index 93% rename from src/Microsoft.Tye.Hosting/Infrastructure/IngressHostMetadata.cs rename to src/Microsoft.Tye.HttpProxy/IngressHostMetadata.cs index ab46f911..466e6aaa 100644 --- a/src/Microsoft.Tye.Hosting/Infrastructure/IngressHostMetadata.cs +++ b/src/Microsoft.Tye.HttpProxy/IngressHostMetadata.cs @@ -9,7 +9,7 @@ using System.Threading.Tasks; namespace Microsoft.AspNetCore.Routing.Matching { - internal class IngressHostMetadata + public class IngressHostMetadata { public IngressHostMetadata(params string[] hosts) { diff --git a/src/Microsoft.Tye.HttpProxy/Microsoft.Tye.HttpProxy.csproj b/src/Microsoft.Tye.HttpProxy/Microsoft.Tye.HttpProxy.csproj new file mode 100644 index 00000000..2539444f --- /dev/null +++ b/src/Microsoft.Tye.HttpProxy/Microsoft.Tye.HttpProxy.csproj @@ -0,0 +1,18 @@ + + + + Exe + netcoreapp3.1 + + + + + + + + + + + + diff --git a/src/Microsoft.Tye.HttpProxy/Program.cs b/src/Microsoft.Tye.HttpProxy/Program.cs new file mode 100644 index 00000000..aa987d76 --- /dev/null +++ b/src/Microsoft.Tye.HttpProxy/Program.cs @@ -0,0 +1,26 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; +using Microsoft.AspNetCore.Hosting; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Logging; + +namespace Microsoft.Tye.HttpProxy +{ + public class Program + { + public static void Main(string[] args) + { + CreateHostBuilder(args).Build().Run(); + } + + public static IHostBuilder CreateHostBuilder(string[] args) => + Host.CreateDefaultBuilder(args) + .ConfigureWebHostDefaults(webBuilder => + { + webBuilder.UseStartup(); + }); + } +} diff --git a/src/Microsoft.Tye.HttpProxy/Properties/launchSettings.json b/src/Microsoft.Tye.HttpProxy/Properties/launchSettings.json new file mode 100644 index 00000000..9945800f --- /dev/null +++ b/src/Microsoft.Tye.HttpProxy/Properties/launchSettings.json @@ -0,0 +1,27 @@ +{ + "iisSettings": { + "windowsAuthentication": false, + "anonymousAuthentication": true, + "iisExpress": { + "applicationUrl": "http://localhost:54967", + "sslPort": 44309 + } + }, + "profiles": { + "IIS Express": { + "commandName": "IISExpress", + "launchBrowser": true, + "environmentVariables": { + "ASPNETCORE_ENVIRONMENT": "Development" + } + }, + "Microsoft.Tye.HttpProxy": { + "commandName": "Project", + "launchBrowser": true, + "applicationUrl": "https://localhost:5001;http://localhost:5000", + "environmentVariables": { + "ASPNETCORE_ENVIRONMENT": "Development" + } + } + } +} diff --git a/src/Microsoft.Tye.Hosting/Infrastructure/ProxyExtensions.cs b/src/Microsoft.Tye.HttpProxy/ProxyExtensions.cs similarity index 99% rename from src/Microsoft.Tye.Hosting/Infrastructure/ProxyExtensions.cs rename to src/Microsoft.Tye.HttpProxy/ProxyExtensions.cs index 46a509d2..ea0f6fb2 100644 --- a/src/Microsoft.Tye.Hosting/Infrastructure/ProxyExtensions.cs +++ b/src/Microsoft.Tye.HttpProxy/ProxyExtensions.cs @@ -11,7 +11,7 @@ using Microsoft.AspNetCore.Http; namespace Microsoft.AspNetCore.Proxy { - internal static class ProxyAdvancedExtensions + public static class ProxyAdvancedExtensions { private static readonly string[] NotForwardedWebSocketHeaders = new[] { "Connection", "Host", "Upgrade", "Sec-WebSocket-Accept", "Sec-WebSocket-Protocol", "Sec-WebSocket-Key", "Sec-WebSocket-Version", "Sec-WebSocket-Extensions" }; private const int DefaultWebSocketBufferSize = 4096; diff --git a/src/Microsoft.Tye.HttpProxy/RoundRobinLoadBalancer.cs b/src/Microsoft.Tye.HttpProxy/RoundRobinLoadBalancer.cs new file mode 100644 index 00000000..d0ad1cae --- /dev/null +++ b/src/Microsoft.Tye.HttpProxy/RoundRobinLoadBalancer.cs @@ -0,0 +1,72 @@ +using System; +using System.Collections.Concurrent; +using System.Linq; +using System.Net; +using System.Net.Http; +using System.Net.Sockets; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; + +namespace Microsoft.Tye.HttpProxy +{ + public class RoundRobinLoadBalancer : DelegatingHandler + { + private readonly ConcurrentDictionary _dnsCache = new ConcurrentDictionary(); + private readonly ILogger _logger; + public RoundRobinLoadBalancer(ILogger logger, HttpMessageHandler innerHandler) : base(innerHandler) + { + _logger = logger; + } + + protected override async Task SendAsync(HttpRequestMessage request, CancellationToken cancellationToken) + { + var host = request.RequestUri.Host; + if (!_dnsCache.TryGetValue(host, out var cache)) + { + var addresses = await Dns.GetHostAddressesAsync(host); + + _logger.LogInformation("Resolved {Host} to {Addresses}", host, addresses); + + cache = new DnsCache(addresses); + _dnsCache[host] = cache; + } + + // Allocations! + request.RequestUri = new UriBuilder(request.RequestUri) + { + Host = cache.GetAddress() + }.Uri; + + try + { + return await base.SendAsync(request, cancellationToken); + } + catch (HttpRequestException ex) when (ex.InnerException is SocketException) + { + // Connection error, remove this host (the target might have died) + _dnsCache.TryRemove(host, out _); + + throw; + } + + } + + private class DnsCache + { + private int _index; + private readonly string[] _addresses; + + public DnsCache(IPAddress[] addresses) + { + _addresses = addresses.Select(a => a.ToString()).ToArray(); + } + + public string GetAddress() + { + var next = Interlocked.Increment(ref _index) % _addresses.Length; + return _addresses[next]; + } + } + } +} diff --git a/src/Microsoft.Tye.HttpProxy/Startup.cs b/src/Microsoft.Tye.HttpProxy/Startup.cs new file mode 100644 index 00000000..46e9aecd --- /dev/null +++ b/src/Microsoft.Tye.HttpProxy/Startup.cs @@ -0,0 +1,82 @@ +using System; +using System.Net; +using System.Net.Http; +using Microsoft.AspNetCore.Builder; +using Microsoft.AspNetCore.Hosting; +using Microsoft.AspNetCore.Http; +using Microsoft.AspNetCore.Proxy; +using Microsoft.AspNetCore.Routing; +using Microsoft.AspNetCore.Routing.Matching; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Logging; + +namespace Microsoft.Tye.HttpProxy +{ + public class Startup + { + public void ConfigureServices(IServiceCollection services) + { + services.AddSingleton(); + } + + // This method gets called by the runtime. Use this method to configure the HTTP request pipeline. + public void Configure(IApplicationBuilder app, ILogger logger, IWebHostEnvironment env, IConfiguration configuration) + { + var invoker = new HttpMessageInvoker(new ConnectionRetryHandler(new RoundRobinLoadBalancer(logger, new SocketsHttpHandler + { + AllowAutoRedirect = false, + AutomaticDecompression = DecompressionMethods.None, + UseProxy = false + }))); + + if (env.IsDevelopment()) + { + app.UseDeveloperExceptionPage(); + } + + app.UseRouting(); + + app.UseEndpoints(endpoints => + { + foreach (var rule in configuration.GetSection("Rules").GetChildren()) + { + var host = rule["Host"]; + var path = rule["Path"]; + var service = rule["Service"]; + var port = rule["Port"]; + var protocol = rule["Protocol"]; + + var url = $"{protocol}://{service}:{port}"; + + RequestDelegate del = context => + { + var uri = new UriBuilder(url) + { + Path = (string)context.Request.RouteValues["path"] ?? "/" + }; + + return context.ProxyRequest(invoker, uri.Uri); + }; + + IEndpointConventionBuilder conventions = null!; + + if (path != null) + { + conventions = endpoints.Map(path.TrimEnd('/') + "/{**path}", del); + } + else + { + conventions = endpoints.MapFallback(del); + } + + if (host != null) + { + conventions.WithMetadata(new IngressHostMetadata(host)); + } + } + }); + } + } +} diff --git a/src/Microsoft.Tye.HttpProxy/appsettings.Development.json b/src/Microsoft.Tye.HttpProxy/appsettings.Development.json new file mode 100644 index 00000000..63ed6e14 --- /dev/null +++ b/src/Microsoft.Tye.HttpProxy/appsettings.Development.json @@ -0,0 +1,9 @@ +{ + "Logging": { + "LogLevel": { + "Default": "Information", + "Microsoft": "Information", + "Microsoft.Hosting.Lifetime": "Information" + } + } +} diff --git a/src/Microsoft.Tye.HttpProxy/appsettings.json b/src/Microsoft.Tye.HttpProxy/appsettings.json new file mode 100644 index 00000000..214d63f9 --- /dev/null +++ b/src/Microsoft.Tye.HttpProxy/appsettings.json @@ -0,0 +1,10 @@ +{ + "Logging": { + "LogLevel": { + "Default": "Information", + "Microsoft": "Information", + "Microsoft.Hosting.Lifetime": "Information" + } + }, + "AllowedHosts": "*" +} diff --git a/src/tye/ApplicationBuilderExtensions.cs b/src/tye/ApplicationBuilderExtensions.cs index 53aaaf3a..288045d2 100644 --- a/src/tye/ApplicationBuilderExtensions.cs +++ b/src/tye/ApplicationBuilderExtensions.cs @@ -32,11 +32,43 @@ namespace Microsoft.Tye public static Application ToHostingApplication(this ApplicationBuilder application) { var services = new Dictionary(); + var ingress = application.Ingress; + + // Ingress get turned into services for hosting + if (ingress != null) + { + var rules = new List(); + + foreach (var rule in ingress.Rules) + { + rules.Add(new IngressRule(rule.Host, rule.Path, rule.Service!)); + } + + var runInfo = new IngressRunInfo(rules); + + var description = new ServiceDescription(ingress.Name, runInfo) + { + Replicas = ingress.Replicas, + }; + + foreach (var binding in ingress.Bindings) + { + description.Bindings.Add(new ServiceBinding() + { + Name = binding.Name, + Port = binding.Port, + Protocol = binding.Protocol, + }); + } + + services.Add(ingress.Name, new Service(description)); + } foreach (var service in application.Services) { RunInfo? runInfo; int replicas; + var env = new List(); if (service is ExternalServiceBuilder) { @@ -138,37 +170,10 @@ namespace Microsoft.Tye services.Add(service.Name, new Service(description)); } - // Ingress get turned into services for hosting - foreach (var ingress in application.Ingress) + return new Application(application.Source, services) { - var rules = new List(); - - foreach (var rule in ingress.Rules) - { - rules.Add(new IngressRule(rule.Host, rule.Path, rule.Service!)); - } - - var runInfo = new IngressRunInfo(rules); - - var description = new ServiceDescription(ingress.Name, runInfo) - { - Replicas = ingress.Replicas, - }; - - foreach (var binding in ingress.Bindings) - { - description.Bindings.Add(new ServiceBinding() - { - Name = binding.Name, - Port = binding.Port, - Protocol = binding.Protocol, - }); - } - - services.Add(ingress.Name, new Service(description)); - } - - return new Application(application.Source, services) { Network = application.Network }; + Network = application.Network + }; } public static Tye.Hosting.Model.EnvironmentVariable ToHostingEnvironmentVariable(this EnvironmentVariableBuilder builder) diff --git a/test/E2ETest/testassets/projects/apps-with-ingress/tye.yaml b/test/E2ETest/testassets/projects/apps-with-ingress/tye.yaml index c6aa8cca..f5f04d54 100644 --- a/test/E2ETest/testassets/projects/apps-with-ingress/tye.yaml +++ b/test/E2ETest/testassets/projects/apps-with-ingress/tye.yaml @@ -6,18 +6,18 @@ # name: apps-with-ingress ingress: - - name: ingress - bindings: - - port: 8080 - rules: - - path: /A - service: app-a - - path: /B - service: app-b - - host: a.example.com - service: app-a - - host: b.example.com - service: app-b + name: ingress + bindings: + - port: 8080 + rules: + - path: /A + service: app-a + - path: /B + service: app-b + - host: a.example.com + service: app-a + - host: b.example.com + service: app-b services: - name: app-a diff --git a/test/Test.Infrastructure/TyeAssert.cs b/test/Test.Infrastructure/TyeAssert.cs index e9ca4470..16e5acfb 100644 --- a/test/Test.Infrastructure/TyeAssert.cs +++ b/test/Test.Infrastructure/TyeAssert.cs @@ -16,13 +16,14 @@ namespace Test.Infrastructure Assert.Equal(expected.Registry, actual.Registry); Assert.Equal(expected.Network, actual.Network); - foreach (var ingress in actual.Ingress) + + var ingress = actual.Ingress; + var otherIngress = expected.Ingress; + Assert.True(ingress == null && otherIngress == null || + ingress != null && otherIngress != null); + + if (ingress != null && otherIngress != null) { - var otherIngress = expected - .Ingress - .Where(o => o.Name == ingress.Name) - .Single(); - Assert.NotNull(otherIngress); Assert.Equal(otherIngress.Replicas, ingress.Replicas); foreach (var rule in ingress.Rules) diff --git a/tye.sln b/tye.sln index 6cda0ced..5d61b4da 100644 --- a/tye.sln +++ b/tye.sln @@ -29,9 +29,11 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.Tye.Proxy", "src\ EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.Tye.Extensions.Configuration.Tests", "test\Microsoft.Tye.Extensions.Configuration.Tests\Microsoft.Tye.Extensions.Configuration.Tests.csproj", "{FCE7C889-16D1-42E7-A514-EA096E9D41A7}" EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Test.Infrastructure", "test\Test.Infrastructure\Test.Infrastructure.csproj", "{F9A7C801-26F3-4D20-B084-2A6A6343E869}" +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Test.Infrastructure", "test\Test.Infrastructure\Test.Infrastructure.csproj", "{F9A7C801-26F3-4D20-B084-2A6A6343E869}" EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "tye-diag-agent", "src\tye-diag-agent\tye-diag-agent.csproj", "{7173D2B6-B102-4DCA-8000-5DAE684BFC13}" +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "tye-diag-agent", "src\tye-diag-agent\tye-diag-agent.csproj", "{7173D2B6-B102-4DCA-8000-5DAE684BFC13}" +EndProject +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.Tye.HttpProxy", "src\Microsoft.Tye.HttpProxy\Microsoft.Tye.HttpProxy.csproj", "{CACCDF7E-70FF-43DB-A822-91BADCED8E13}" EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution @@ -199,6 +201,18 @@ Global {7173D2B6-B102-4DCA-8000-5DAE684BFC13}.Release|x64.Build.0 = Release|Any CPU {7173D2B6-B102-4DCA-8000-5DAE684BFC13}.Release|x86.ActiveCfg = Release|Any CPU {7173D2B6-B102-4DCA-8000-5DAE684BFC13}.Release|x86.Build.0 = Release|Any CPU + {CACCDF7E-70FF-43DB-A822-91BADCED8E13}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {CACCDF7E-70FF-43DB-A822-91BADCED8E13}.Debug|Any CPU.Build.0 = Debug|Any CPU + {CACCDF7E-70FF-43DB-A822-91BADCED8E13}.Debug|x64.ActiveCfg = Debug|Any CPU + {CACCDF7E-70FF-43DB-A822-91BADCED8E13}.Debug|x64.Build.0 = Debug|Any CPU + {CACCDF7E-70FF-43DB-A822-91BADCED8E13}.Debug|x86.ActiveCfg = Debug|Any CPU + {CACCDF7E-70FF-43DB-A822-91BADCED8E13}.Debug|x86.Build.0 = Debug|Any CPU + {CACCDF7E-70FF-43DB-A822-91BADCED8E13}.Release|Any CPU.ActiveCfg = Release|Any CPU + {CACCDF7E-70FF-43DB-A822-91BADCED8E13}.Release|Any CPU.Build.0 = Release|Any CPU + {CACCDF7E-70FF-43DB-A822-91BADCED8E13}.Release|x64.ActiveCfg = Release|Any CPU + {CACCDF7E-70FF-43DB-A822-91BADCED8E13}.Release|x64.Build.0 = Release|Any CPU + {CACCDF7E-70FF-43DB-A822-91BADCED8E13}.Release|x86.ActiveCfg = Release|Any CPU + {CACCDF7E-70FF-43DB-A822-91BADCED8E13}.Release|x86.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE @@ -217,6 +231,7 @@ Global {FCE7C889-16D1-42E7-A514-EA096E9D41A7} = {F19B02EB-A372-417A-B2C2-EA0D5A3C76D5} {F9A7C801-26F3-4D20-B084-2A6A6343E869} = {F19B02EB-A372-417A-B2C2-EA0D5A3C76D5} {7173D2B6-B102-4DCA-8000-5DAE684BFC13} = {8C662D59-A3CB-466F-8E85-A8E6BA5E7601} + {CACCDF7E-70FF-43DB-A822-91BADCED8E13} = {8C662D59-A3CB-466F-8E85-A8E6BA5E7601} EndGlobalSection GlobalSection(ExtensibilityGlobals) = postSolution SolutionGuid = {D8002603-BB27-4500-BF86-274A8E72D302}