diff --git a/samples/apps-with-ingress/ApplicationA/ApplicationA.csproj b/samples/apps-with-ingress/ApplicationA/ApplicationA.csproj new file mode 100644 index 00000000..92605c5a --- /dev/null +++ b/samples/apps-with-ingress/ApplicationA/ApplicationA.csproj @@ -0,0 +1,7 @@ + + + + netcoreapp3.1 + + + diff --git a/samples/apps-with-ingress/ApplicationA/Program.cs b/samples/apps-with-ingress/ApplicationA/Program.cs new file mode 100644 index 00000000..56492c75 --- /dev/null +++ b/samples/apps-with-ingress/ApplicationA/Program.cs @@ -0,0 +1,26 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; +using Microsoft.AspNetCore.Hosting; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Logging; + +namespace ApplicationA +{ + public class Program + { + public static void Main(string[] args) + { + CreateHostBuilder(args).Build().Run(); + } + + public static IHostBuilder CreateHostBuilder(string[] args) => + Host.CreateDefaultBuilder(args) + .ConfigureWebHostDefaults(webBuilder => + { + webBuilder.UseStartup(); + }); + } +} diff --git a/samples/apps-with-ingress/ApplicationA/Properties/launchSettings.json b/samples/apps-with-ingress/ApplicationA/Properties/launchSettings.json new file mode 100644 index 00000000..767a1d0b --- /dev/null +++ b/samples/apps-with-ingress/ApplicationA/Properties/launchSettings.json @@ -0,0 +1,27 @@ +{ + "iisSettings": { + "windowsAuthentication": false, + "anonymousAuthentication": true, + "iisExpress": { + "applicationUrl": "http://localhost:2755", + "sslPort": 44369 + } + }, + "profiles": { + "IIS Express": { + "commandName": "IISExpress", + "launchBrowser": true, + "environmentVariables": { + "ASPNETCORE_ENVIRONMENT": "Development" + } + }, + "ApplicationA": { + "commandName": "Project", + "launchBrowser": true, + "applicationUrl": "https://localhost:5001;http://localhost:5000", + "environmentVariables": { + "ASPNETCORE_ENVIRONMENT": "Development" + } + } + } +} diff --git a/samples/apps-with-ingress/ApplicationA/Startup.cs b/samples/apps-with-ingress/ApplicationA/Startup.cs new file mode 100644 index 00000000..257fd799 --- /dev/null +++ b/samples/apps-with-ingress/ApplicationA/Startup.cs @@ -0,0 +1,40 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; +using Microsoft.AspNetCore.Builder; +using Microsoft.AspNetCore.Hosting; +using Microsoft.AspNetCore.Http; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Hosting; + +namespace ApplicationA +{ + public class Startup + { + // This method gets called by the runtime. Use this method to add services to the container. + // For more information on how to configure your application, visit https://go.microsoft.com/fwlink/?LinkID=398940 + public void ConfigureServices(IServiceCollection services) + { + } + + // This method gets called by the runtime. Use this method to configure the HTTP request pipeline. + public void Configure(IApplicationBuilder app, IWebHostEnvironment env) + { + if (env.IsDevelopment()) + { + app.UseDeveloperExceptionPage(); + } + + app.UseRouting(); + + app.UseEndpoints(endpoints => + { + endpoints.MapGet("/", async context => + { + await context.Response.WriteAsync("Hello from Application A " + Environment.GetEnvironmentVariable("APP_INSTANCE") ?? Environment.GetEnvironmentVariable("HOSTNAME")); + }); + }); + } + } +} diff --git a/samples/apps-with-ingress/ApplicationA/appsettings.Development.json b/samples/apps-with-ingress/ApplicationA/appsettings.Development.json new file mode 100644 index 00000000..8983e0fc --- /dev/null +++ b/samples/apps-with-ingress/ApplicationA/appsettings.Development.json @@ -0,0 +1,9 @@ +{ + "Logging": { + "LogLevel": { + "Default": "Information", + "Microsoft": "Warning", + "Microsoft.Hosting.Lifetime": "Information" + } + } +} diff --git a/samples/apps-with-ingress/ApplicationA/appsettings.json b/samples/apps-with-ingress/ApplicationA/appsettings.json new file mode 100644 index 00000000..d9d9a9bf --- /dev/null +++ b/samples/apps-with-ingress/ApplicationA/appsettings.json @@ -0,0 +1,10 @@ +{ + "Logging": { + "LogLevel": { + "Default": "Information", + "Microsoft": "Warning", + "Microsoft.Hosting.Lifetime": "Information" + } + }, + "AllowedHosts": "*" +} diff --git a/samples/apps-with-ingress/ApplicationB/ApplicationB.csproj b/samples/apps-with-ingress/ApplicationB/ApplicationB.csproj new file mode 100644 index 00000000..92605c5a --- /dev/null +++ b/samples/apps-with-ingress/ApplicationB/ApplicationB.csproj @@ -0,0 +1,7 @@ + + + + netcoreapp3.1 + + + diff --git a/samples/apps-with-ingress/ApplicationB/Program.cs b/samples/apps-with-ingress/ApplicationB/Program.cs new file mode 100644 index 00000000..8b3c38af --- /dev/null +++ b/samples/apps-with-ingress/ApplicationB/Program.cs @@ -0,0 +1,26 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; +using Microsoft.AspNetCore.Hosting; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Logging; + +namespace ApplicationB +{ + public class Program + { + public static void Main(string[] args) + { + CreateHostBuilder(args).Build().Run(); + } + + public static IHostBuilder CreateHostBuilder(string[] args) => + Host.CreateDefaultBuilder(args) + .ConfigureWebHostDefaults(webBuilder => + { + webBuilder.UseStartup(); + }); + } +} diff --git a/samples/apps-with-ingress/ApplicationB/Properties/launchSettings.json b/samples/apps-with-ingress/ApplicationB/Properties/launchSettings.json new file mode 100644 index 00000000..2b1eba0f --- /dev/null +++ b/samples/apps-with-ingress/ApplicationB/Properties/launchSettings.json @@ -0,0 +1,27 @@ +{ + "iisSettings": { + "windowsAuthentication": false, + "anonymousAuthentication": true, + "iisExpress": { + "applicationUrl": "http://localhost:19251", + "sslPort": 44343 + } + }, + "profiles": { + "IIS Express": { + "commandName": "IISExpress", + "launchBrowser": true, + "environmentVariables": { + "ASPNETCORE_ENVIRONMENT": "Development" + } + }, + "ApplicationB": { + "commandName": "Project", + "launchBrowser": true, + "applicationUrl": "https://localhost:5001;http://localhost:5000", + "environmentVariables": { + "ASPNETCORE_ENVIRONMENT": "Development" + } + } + } +} diff --git a/samples/apps-with-ingress/ApplicationB/Startup.cs b/samples/apps-with-ingress/ApplicationB/Startup.cs new file mode 100644 index 00000000..1a6b68d8 --- /dev/null +++ b/samples/apps-with-ingress/ApplicationB/Startup.cs @@ -0,0 +1,40 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; +using Microsoft.AspNetCore.Builder; +using Microsoft.AspNetCore.Hosting; +using Microsoft.AspNetCore.Http; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Hosting; + +namespace ApplicationB +{ + public class Startup + { + // This method gets called by the runtime. Use this method to add services to the container. + // For more information on how to configure your application, visit https://go.microsoft.com/fwlink/?LinkID=398940 + public void ConfigureServices(IServiceCollection services) + { + } + + // This method gets called by the runtime. Use this method to configure the HTTP request pipeline. + public void Configure(IApplicationBuilder app, IWebHostEnvironment env) + { + if (env.IsDevelopment()) + { + app.UseDeveloperExceptionPage(); + } + + app.UseRouting(); + + app.UseEndpoints(endpoints => + { + endpoints.MapGet("/", async context => + { + await context.Response.WriteAsync("Hello from Application B " + Environment.GetEnvironmentVariable("APP_INSTANCE") ?? Environment.GetEnvironmentVariable("HOSTNAME")); + }); + }); + } + } +} diff --git a/samples/apps-with-ingress/ApplicationB/appsettings.Development.json b/samples/apps-with-ingress/ApplicationB/appsettings.Development.json new file mode 100644 index 00000000..8983e0fc --- /dev/null +++ b/samples/apps-with-ingress/ApplicationB/appsettings.Development.json @@ -0,0 +1,9 @@ +{ + "Logging": { + "LogLevel": { + "Default": "Information", + "Microsoft": "Warning", + "Microsoft.Hosting.Lifetime": "Information" + } + } +} diff --git a/samples/apps-with-ingress/ApplicationB/appsettings.json b/samples/apps-with-ingress/ApplicationB/appsettings.json new file mode 100644 index 00000000..d9d9a9bf --- /dev/null +++ b/samples/apps-with-ingress/ApplicationB/appsettings.json @@ -0,0 +1,10 @@ +{ + "Logging": { + "LogLevel": { + "Default": "Information", + "Microsoft": "Warning", + "Microsoft.Hosting.Lifetime": "Information" + } + }, + "AllowedHosts": "*" +} diff --git a/samples/apps-with-ingress/apps-with-ingress.sln b/samples/apps-with-ingress/apps-with-ingress.sln new file mode 100644 index 00000000..3dcc942e --- /dev/null +++ b/samples/apps-with-ingress/apps-with-ingress.sln @@ -0,0 +1,48 @@ + +Microsoft Visual Studio Solution File, Format Version 12.00 +# Visual Studio 15 +VisualStudioVersion = 15.0.26124.0 +MinimumVisualStudioVersion = 15.0.26124.0 +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "ApplicationA", "ApplicationA\ApplicationA.csproj", "{5A9DC239-55BB-4951-B081-35931BF8C867}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "ApplicationB", "ApplicationB\ApplicationB.csproj", "{AE1F10D3-BFAE-4D23-ADCF-06770237285D}" +EndProject +Global + GlobalSection(SolutionConfigurationPlatforms) = preSolution + Debug|Any CPU = Debug|Any CPU + Debug|x64 = Debug|x64 + Debug|x86 = Debug|x86 + Release|Any CPU = Release|Any CPU + Release|x64 = Release|x64 + Release|x86 = Release|x86 + EndGlobalSection + GlobalSection(SolutionProperties) = preSolution + HideSolutionNode = FALSE + EndGlobalSection + GlobalSection(ProjectConfigurationPlatforms) = postSolution + {5A9DC239-55BB-4951-B081-35931BF8C867}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {5A9DC239-55BB-4951-B081-35931BF8C867}.Debug|Any CPU.Build.0 = Debug|Any CPU + {5A9DC239-55BB-4951-B081-35931BF8C867}.Debug|x64.ActiveCfg = Debug|Any CPU + {5A9DC239-55BB-4951-B081-35931BF8C867}.Debug|x64.Build.0 = Debug|Any CPU + {5A9DC239-55BB-4951-B081-35931BF8C867}.Debug|x86.ActiveCfg = Debug|Any CPU + {5A9DC239-55BB-4951-B081-35931BF8C867}.Debug|x86.Build.0 = Debug|Any CPU + {5A9DC239-55BB-4951-B081-35931BF8C867}.Release|Any CPU.ActiveCfg = Release|Any CPU + {5A9DC239-55BB-4951-B081-35931BF8C867}.Release|Any CPU.Build.0 = Release|Any CPU + {5A9DC239-55BB-4951-B081-35931BF8C867}.Release|x64.ActiveCfg = Release|Any CPU + {5A9DC239-55BB-4951-B081-35931BF8C867}.Release|x64.Build.0 = Release|Any CPU + {5A9DC239-55BB-4951-B081-35931BF8C867}.Release|x86.ActiveCfg = Release|Any CPU + {5A9DC239-55BB-4951-B081-35931BF8C867}.Release|x86.Build.0 = Release|Any CPU + {AE1F10D3-BFAE-4D23-ADCF-06770237285D}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {AE1F10D3-BFAE-4D23-ADCF-06770237285D}.Debug|Any CPU.Build.0 = Debug|Any CPU + {AE1F10D3-BFAE-4D23-ADCF-06770237285D}.Debug|x64.ActiveCfg = Debug|Any CPU + {AE1F10D3-BFAE-4D23-ADCF-06770237285D}.Debug|x64.Build.0 = Debug|Any CPU + {AE1F10D3-BFAE-4D23-ADCF-06770237285D}.Debug|x86.ActiveCfg = Debug|Any CPU + {AE1F10D3-BFAE-4D23-ADCF-06770237285D}.Debug|x86.Build.0 = Debug|Any CPU + {AE1F10D3-BFAE-4D23-ADCF-06770237285D}.Release|Any CPU.ActiveCfg = Release|Any CPU + {AE1F10D3-BFAE-4D23-ADCF-06770237285D}.Release|Any CPU.Build.0 = Release|Any CPU + {AE1F10D3-BFAE-4D23-ADCF-06770237285D}.Release|x64.ActiveCfg = Release|Any CPU + {AE1F10D3-BFAE-4D23-ADCF-06770237285D}.Release|x64.Build.0 = Release|Any CPU + {AE1F10D3-BFAE-4D23-ADCF-06770237285D}.Release|x86.ActiveCfg = Release|Any CPU + {AE1F10D3-BFAE-4D23-ADCF-06770237285D}.Release|x86.Build.0 = Release|Any CPU + EndGlobalSection +EndGlobal diff --git a/samples/apps-with-ingress/tye.yaml b/samples/apps-with-ingress/tye.yaml new file mode 100644 index 00000000..1ca6410b --- /dev/null +++ b/samples/apps-with-ingress/tye.yaml @@ -0,0 +1,28 @@ +# tye application configuration file +# read all about it at https://github.com/dotnet/tye +# +# when you've given us a try, we'd love to know what you think: +# https://aka.ms/AA7q20u +# +name: apps-with-ingress +ingress: + - name: ingress + bindings: + - port: 8080 + rules: + - path: /A + service: appA + - path: /B + service: appB + - host: a.example.com + service: appA + - host: b.example.com + service: appB + +services: +- name: appA + project: ApplicationA/ApplicationA.csproj + replicas: 2 +- name: appB + project: ApplicationB/ApplicationB.csproj + replicas: 2 diff --git a/src/Microsoft.Tye.Hosting/Dashboard/Pages/Logs.razor b/src/Microsoft.Tye.Hosting/Dashboard/Pages/Logs.razor index 8014cb3f..4c921126 100644 --- a/src/Microsoft.Tye.Hosting/Dashboard/Pages/Logs.razor +++ b/src/Microsoft.Tye.Hosting/Dashboard/Pages/Logs.razor @@ -13,7 +13,7 @@ else
@foreach (var log in ApplicationLogs) { -
@log.Text
+
@log.Text
}
} diff --git a/src/Microsoft.Tye.Hosting/Infrastructure/IngressHostMatcherPolicy.cs b/src/Microsoft.Tye.Hosting/Infrastructure/IngressHostMatcherPolicy.cs new file mode 100644 index 00000000..469fde36 --- /dev/null +++ b/src/Microsoft.Tye.Hosting/Infrastructure/IngressHostMatcherPolicy.cs @@ -0,0 +1,209 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; +using Microsoft.AspNetCore.Http; + +namespace Microsoft.AspNetCore.Routing.Matching +{ + internal sealed class IngressHostMatcherPolicy : MatcherPolicy, IEndpointSelectorPolicy + { + private const string WildcardHost = "*"; + private const string WildcardPrefix = "*."; + + // Run after HTTP methods, but before 'default'. + public override int Order { get; } = -100; + + public bool AppliesToEndpoints(IReadOnlyList endpoints) + { + return endpoints.Any(e => + { + var hosts = e.Metadata.GetMetadata()?.Hosts; + if (hosts == null || hosts.Count == 0) + { + return false; + } + + foreach (var host in hosts) + { + // Don't run policy on endpoints that match everything + var key = CreateEdgeKey(host); + if (!key.MatchesAll) + { + return true; + } + } + + return false; + }); + } + + public Task ApplyAsync(HttpContext httpContext, CandidateSet candidates) + { + if (httpContext == null) + { + throw new ArgumentNullException(nameof(httpContext)); + } + + if (candidates == null) + { + throw new ArgumentNullException(nameof(candidates)); + } + + for (var i = 0; i < candidates.Count; i++) + { + if (!candidates.IsValidCandidate(i)) + { + continue; + } + + var hosts = candidates[i].Endpoint.Metadata.GetMetadata()?.Hosts; + if (hosts == null || hosts.Count == 0) + { + // Can match any host. + continue; + } + + var matched = false; + var (requestHost, requestPort) = GetHostAndPort(httpContext); + for (var j = 0; j < hosts.Count; j++) + { + var host = hosts[j].AsSpan(); + var port = ReadOnlySpan.Empty; + + // Split into host and port + var pivot = host.IndexOf(':'); + if (pivot >= 0) + { + port = host.Slice(pivot + 1); + host = host.Slice(0, pivot); + } + + if (host == null || MemoryExtensions.Equals(host, WildcardHost, StringComparison.OrdinalIgnoreCase)) + { + // Can match any host + } + else if ( + host.StartsWith(WildcardPrefix) && + + // Note that we only slice of the `*`. We want to match the leading `.` also. + MemoryExtensions.EndsWith(requestHost, host.Slice(WildcardHost.Length), StringComparison.OrdinalIgnoreCase)) + { + // Matches a suffix wildcard. + } + else if (MemoryExtensions.Equals(requestHost, host, StringComparison.OrdinalIgnoreCase)) + { + // Matches exactly + } + else + { + // If we get here then the host doesn't match. + continue; + } + + if (MemoryExtensions.Equals(port, WildcardHost, StringComparison.OrdinalIgnoreCase)) + { + // Port is a wildcard, we allow any port. + } + else if (port.Length > 0 && (!int.TryParse(port, out var parsed) || parsed != requestPort)) + { + // If we get here then the port doesn't match. + continue; + } + + matched = true; + break; + } + + if (!matched) + { + candidates.SetValidity(i, false); + } + } + + return Task.CompletedTask; + } + + private static EdgeKey CreateEdgeKey(string host) + { + if (host == null) + { + return EdgeKey.WildcardEdgeKey; + } + + var hostParts = host.Split(':'); + if (hostParts.Length == 1) + { + if (!string.IsNullOrEmpty(hostParts[0])) + { + return new EdgeKey(hostParts[0], null); + } + } + if (hostParts.Length == 2) + { + if (!string.IsNullOrEmpty(hostParts[0])) + { + if (int.TryParse(hostParts[1], out var port)) + { + return new EdgeKey(hostParts[0], port); + } + else if (string.Equals(hostParts[1], WildcardHost, StringComparison.Ordinal)) + { + return new EdgeKey(hostParts[0], null); + } + } + } + + throw new InvalidOperationException($"Could not parse host: {host}"); + } + + private static (string host, int? port) GetHostAndPort(HttpContext httpContext) + { + var hostString = httpContext.Request.Host; + if (hostString.Port != null) + { + return (hostString.Host, hostString.Port); + } + else if (string.Equals("https", httpContext.Request.Scheme, StringComparison.OrdinalIgnoreCase)) + { + return (hostString.Host, 443); + } + else if (string.Equals("http", httpContext.Request.Scheme, StringComparison.OrdinalIgnoreCase)) + { + return (hostString.Host, 80); + } + else + { + return (hostString.Host, null); + } + } + + private readonly struct EdgeKey + { + internal static readonly EdgeKey WildcardEdgeKey = new EdgeKey(null, null); + + public readonly int? Port; + public readonly string Host; + + public EdgeKey(string? host, int? port) + { + Host = host ?? WildcardHost; + Port = port; + + HasHostWildcard = Host.StartsWith(WildcardPrefix, StringComparison.Ordinal); + } + + public bool HasHostWildcard { get; } + + public bool MatchesHost => !string.Equals(Host, WildcardHost, StringComparison.Ordinal); + + public bool MatchesPort => Port != null; + + public bool MatchesAll => !MatchesHost && !MatchesPort; + } + } +} diff --git a/src/Microsoft.Tye.Hosting/Infrastructure/IngressHostMetadata.cs b/src/Microsoft.Tye.Hosting/Infrastructure/IngressHostMetadata.cs new file mode 100644 index 00000000..ab46f911 --- /dev/null +++ b/src/Microsoft.Tye.Hosting/Infrastructure/IngressHostMetadata.cs @@ -0,0 +1,21 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; + +namespace Microsoft.AspNetCore.Routing.Matching +{ + internal class IngressHostMetadata + { + public IngressHostMetadata(params string[] hosts) + { + Hosts = new List(hosts).AsReadOnly(); + } + + public IReadOnlyList Hosts { get; } + } +} diff --git a/src/Microsoft.Tye.Hosting/IngressService.cs b/src/Microsoft.Tye.Hosting/IngressService.cs new file mode 100644 index 00000000..9adefcac --- /dev/null +++ b/src/Microsoft.Tye.Hosting/IngressService.cs @@ -0,0 +1,287 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System; +using System.Collections.Generic; +using System.Net; +using System.Net.Http; +using System.Net.Sockets; +using System.Reactive.Subjects; +using System.Runtime.ExceptionServices; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.AspNetCore.Builder; +using Microsoft.AspNetCore.Hosting; +using Microsoft.AspNetCore.Http; +using Microsoft.AspNetCore.Proxy; +using Microsoft.AspNetCore.Routing; +using Microsoft.AspNetCore.Routing.Matching; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; +using Microsoft.Tye.Hosting.Model; + +namespace Microsoft.Tye.Hosting +{ + public class IngressService : IApplicationProcessor + { + private List _webApplications = new List(); + private readonly ILogger _logger; + + public IngressService(ILogger logger) + { + _logger = logger; + } + + public async Task StartAsync(Model.Application application) + { + var invoker = new HttpMessageInvoker(new ConnectionRetryHandler(new SocketsHttpHandler + { + AllowAutoRedirect = false, + AutomaticDecompression = DecompressionMethods.None, + UseProxy = false + })); + + foreach (var service in application.Services.Values) + { + var serviceDescription = service.Description; + + if (service.Description.RunInfo is IngressRunInfo runInfo) + { + var builder = new WebApplicationBuilder(); + + builder.Services.AddSingleton(); + + builder.Logging.AddProvider(new ServiceLoggerProvider(service.Logs)); + + var addresses = new List(); + + // Bind to the addresses on this resource + for (int i = 0; i < serviceDescription.Replicas; i++) + { + // Fake replicas since it's all running processes + var replica = service.Description.Name + "_" + Guid.NewGuid().ToString().Substring(0, 10).ToLower(); + var status = new IngressStatus(service, replica); + service.Replicas[replica] = status; + + var ports = new List(); + + foreach (var binding in serviceDescription.Bindings) + { + if (binding.Port == null) + { + continue; + } + + var port = service.PortMap[binding.Port.Value][i]; + ports.Add(port); + var url = $"{binding.Protocol ?? "http"}://localhost:{port}"; + addresses.Add(url); + } + + status.Ports = ports; + + service.ReplicaEvents.OnNext(new ReplicaEvent(ReplicaState.Added, status)); + } + + builder.Server.UseUrls(addresses.ToArray()); + var webApp = builder.Build(); + + _webApplications.Add(webApp); + + // For each ingress rule, bind to the path and host + foreach (var rule in runInfo.Rules) + { + if (!application.Services.TryGetValue(rule.Service, out var target)) + { + continue; + } + + _logger.LogInformation("Processing ingress rule: Path:{Path}, Host:{Host}, Service:{Service}", rule.Path, rule.Host, rule.Service); + + var targetServiceDescription = target.Description; + + var uris = new List(); + + // For each of the target service replicas, get the base URL + // based on the replica port + for (int i = 0; i < targetServiceDescription.Replicas; i++) + { + foreach (var binding in targetServiceDescription.Bindings) + { + if (binding.Port == null) + { + continue; + } + + var port = target.PortMap[binding.Port.Value][i]; + var url = $"{binding.Protocol ?? "http"}://localhost:{port}"; + uris.Add(new Uri(url)); + } + } + + // The only load balancing strategy here is round robin + long count = 0; + RequestDelegate del = context => + { + var next = (int)(Interlocked.Increment(ref count) % uris.Count); + + var uri = new UriBuilder(uris[next]) + { + Path = (string)context.Request.RouteValues["path"] + }; + + return context.ProxyRequest(invoker, uri.Uri); + }; + + IEndpointConventionBuilder conventions = null!; + + if (rule.Path != null) + { + conventions = ((IEndpointRouteBuilder)webApp).Map(rule.Path.TrimEnd('/') + "/{**path}", del); + } + else + { + conventions = webApp.MapFallback(del); + } + + if (rule.Host != null) + { + conventions.WithMetadata(new IngressHostMetadata(rule.Host)); + } + + conventions.WithDisplayName(rule.Service); + } + } + } + + foreach (var app in _webApplications) + { + await app.StartAsync(); + } + } + + public async Task StopAsync(Model.Application application) + { + foreach (var webApp in _webApplications) + { + try + { + await webApp.StopAsync(); + } + catch (OperationCanceledException) + { + + } + finally + { + webApp.Dispose(); + } + } + } + + private class ConnectionRetryHandler : DelegatingHandler + { + private static readonly int MaxRetries = 3; + private static readonly TimeSpan InitialRetryDelay = TimeSpan.FromMilliseconds(1000); + + public ConnectionRetryHandler(HttpMessageHandler innerHandler) + : base(innerHandler) + { + } + + protected override async Task SendAsync( + HttpRequestMessage request, + CancellationToken cancellationToken) + { + HttpResponseMessage? response = null; + var delay = InitialRetryDelay; + Exception? exception = null; + + for (var i = 0; i < MaxRetries; i++) + { + try + { + response = await base.SendAsync(request, cancellationToken); + } + catch (HttpRequestException ex) when (ex.InnerException is SocketException) + { + if (i == MaxRetries - 1) + { + throw; + } + + exception = ex; + } + + if (response != null && + (response.IsSuccessStatusCode || response.StatusCode != HttpStatusCode.ServiceUnavailable)) + { + return response; + } + + await Task.Delay(delay, cancellationToken); + delay *= 2; + } + + if (exception != null) + { + ExceptionDispatchInfo.Throw(exception); + } + + throw new TimeoutException(); + } + } + + private class ServiceLoggerProvider : ILoggerProvider + { + private readonly Subject _logs; + + public ServiceLoggerProvider(Subject logs) + { + _logs = logs; + } + + public ILogger CreateLogger(string categoryName) + { + return new ServiceLogger(categoryName, _logs); + } + + public void Dispose() + { + } + + private class ServiceLogger : ILogger + { + private readonly string _categoryName; + private readonly Subject _logs; + + public ServiceLogger(string categoryName, Subject logs) + { + _categoryName = categoryName; + _logs = logs; + } + + public IDisposable? BeginScope(TState state) + { + return null; + } + + public bool IsEnabled(LogLevel logLevel) + { + return true; + } + + public void Log(LogLevel logLevel, EventId eventId, TState state, Exception exception, Func formatter) + { + _logs.OnNext($"[{logLevel}]: {formatter(state, exception)}"); + + if (exception != null) + { + _logs.OnNext(exception.ToString()); + } + } + } + } + } +} diff --git a/src/Microsoft.Tye.Hosting/Model/IngressRule.cs b/src/Microsoft.Tye.Hosting/Model/IngressRule.cs new file mode 100644 index 00000000..0a4bfbc5 --- /dev/null +++ b/src/Microsoft.Tye.Hosting/Model/IngressRule.cs @@ -0,0 +1,25 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; + +namespace Microsoft.Tye.Hosting.Model +{ + public class IngressRule + { + public IngressRule(string? host, string? path, string service) + { + Host = host; + Path = path; + Service = service; + } + + public string? Host { get; } + public string? Path { get; } + public string Service { get; } + } +} diff --git a/src/Microsoft.Tye.Hosting/Model/IngressRunInfo.cs b/src/Microsoft.Tye.Hosting/Model/IngressRunInfo.cs new file mode 100644 index 00000000..029b9736 --- /dev/null +++ b/src/Microsoft.Tye.Hosting/Model/IngressRunInfo.cs @@ -0,0 +1,21 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; + +namespace Microsoft.Tye.Hosting.Model +{ + public class IngressRunInfo : RunInfo + { + public IngressRunInfo(List rules) + { + Rules = rules; + } + + public List Rules { get; } + } +} diff --git a/src/Microsoft.Tye.Hosting/Model/IngressStatus.cs b/src/Microsoft.Tye.Hosting/Model/IngressStatus.cs new file mode 100644 index 00000000..c2530895 --- /dev/null +++ b/src/Microsoft.Tye.Hosting/Model/IngressStatus.cs @@ -0,0 +1,19 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; + +namespace Microsoft.Tye.Hosting.Model +{ + public class IngressStatus : ReplicaStatus + { + public IngressStatus(Service service, string name) : base(service, name) + { + } + + } +} diff --git a/src/Microsoft.Tye.Hosting/Model/Service.cs b/src/Microsoft.Tye.Hosting/Model/Service.cs index 6f035727..44bd9970 100644 --- a/src/Microsoft.Tye.Hosting/Model/Service.cs +++ b/src/Microsoft.Tye.Hosting/Model/Service.cs @@ -6,7 +6,6 @@ using System; using System.Collections.Concurrent; using System.Collections.Generic; using System.Reactive.Subjects; -using System.Text.Json.Serialization; namespace Microsoft.Tye.Hosting.Model { @@ -50,6 +49,11 @@ namespace Microsoft.Tye.Hosting.Model return ServiceType.Project; } + if (Description.RunInfo is IngressRunInfo) + { + return ServiceType.Ingress; + } + return ServiceType.External; } } diff --git a/src/Microsoft.Tye.Hosting/Model/ServiceType.cs b/src/Microsoft.Tye.Hosting/Model/ServiceType.cs index e78f5e2f..7e6e4797 100644 --- a/src/Microsoft.Tye.Hosting/Model/ServiceType.cs +++ b/src/Microsoft.Tye.Hosting/Model/ServiceType.cs @@ -9,6 +9,7 @@ namespace Microsoft.Tye.Hosting.Model External, Project, Executable, - Container + Container, + Ingress } } diff --git a/src/Microsoft.Tye.Hosting/ProcessRunner.cs b/src/Microsoft.Tye.Hosting/ProcessRunner.cs index 1d759b63..800c58e5 100644 --- a/src/Microsoft.Tye.Hosting/ProcessRunner.cs +++ b/src/Microsoft.Tye.Hosting/ProcessRunner.cs @@ -34,13 +34,10 @@ namespace Microsoft.Tye.Hosting { tasks[index++] = s.Value.ServiceType switch { - ServiceType.Container => Task.CompletedTask, - ServiceType.External => Task.CompletedTask, - ServiceType.Executable => LaunchService(application, s.Value), ServiceType.Project => LaunchService(application, s.Value), - _ => throw new InvalidOperationException("Unknown ServiceType."), + _ => Task.CompletedTask, }; } diff --git a/src/Microsoft.Tye.Hosting/ProxyExtensions.cs b/src/Microsoft.Tye.Hosting/ProxyExtensions.cs new file mode 100644 index 00000000..448778cf --- /dev/null +++ b/src/Microsoft.Tye.Hosting/ProxyExtensions.cs @@ -0,0 +1,213 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Linq; +using System.Net.Http; +using System.Net.WebSockets; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.AspNetCore.Http; + +namespace Microsoft.AspNetCore.Proxy +{ + internal static class ProxyAdvancedExtensions + { + private static readonly string[] NotForwardedWebSocketHeaders = new[] { "Connection", "Host", "Upgrade", "Sec-WebSocket-Accept", "Sec-WebSocket-Protocol", "Sec-WebSocket-Key", "Sec-WebSocket-Version", "Sec-WebSocket-Extensions" }; + private const int DefaultWebSocketBufferSize = 4096; + private const int StreamCopyBufferSize = 81920; + + public static async Task ProxyRequest(this HttpContext context, HttpMessageInvoker invoker, Uri destinationUri) + { + if (context == null) + { + throw new ArgumentNullException(nameof(context)); + } + if (destinationUri == null) + { + throw new ArgumentNullException(nameof(destinationUri)); + } + + if (context.WebSockets.IsWebSocketRequest) + { + await context.AcceptProxyWebSocketRequest(destinationUri.ToWebSocketScheme()); + } + else + { + using (var requestMessage = context.CreateProxyHttpRequest(destinationUri)) + { + using (var responseMessage = await context.SendProxyHttpRequest(invoker, requestMessage)) + { + await context.CopyProxyHttpResponse(responseMessage); + } + } + } + } + + public static Uri ToWebSocketScheme(this Uri uri) + { + if (uri == null) + { + throw new ArgumentNullException(nameof(uri)); + } + + var uriBuilder = new UriBuilder(uri); + if (string.Equals(uriBuilder.Scheme, "https", StringComparison.OrdinalIgnoreCase)) + { + uriBuilder.Scheme = "wss"; + } + else if (string.Equals(uriBuilder.Scheme, "http", StringComparison.OrdinalIgnoreCase)) + { + uriBuilder.Scheme = "ws"; + } + + return uriBuilder.Uri; + } + + public static HttpRequestMessage CreateProxyHttpRequest(this HttpContext context, Uri uri) + { + var request = context.Request; + + var requestMessage = new HttpRequestMessage(); + var requestMethod = request.Method; + if (!HttpMethods.IsGet(requestMethod) && + !HttpMethods.IsHead(requestMethod) && + !HttpMethods.IsDelete(requestMethod) && + !HttpMethods.IsTrace(requestMethod)) + { + var streamContent = new StreamContent(request.Body); + requestMessage.Content = streamContent; + } + + // Copy the request headers + foreach (var header in request.Headers) + { + if (!requestMessage.Headers.TryAddWithoutValidation(header.Key, header.Value.ToArray()) && requestMessage.Content != null) + { + requestMessage.Content?.Headers.TryAddWithoutValidation(header.Key, header.Value.ToArray()); + } + } + + requestMessage.Headers.Host = uri.Authority; + requestMessage.RequestUri = uri; + requestMessage.Method = new HttpMethod(request.Method); + + return requestMessage; + } + + public static async Task AcceptProxyWebSocketRequest(this HttpContext context, Uri destinationUri) + { + if (context == null) + { + throw new ArgumentNullException(nameof(context)); + } + if (destinationUri == null) + { + throw new ArgumentNullException(nameof(destinationUri)); + } + if (!context.WebSockets.IsWebSocketRequest) + { + throw new InvalidOperationException(); + } + + using var client = new ClientWebSocket(); + foreach (var protocol in context.WebSockets.WebSocketRequestedProtocols) + { + client.Options.AddSubProtocol(protocol); + } + + foreach (var headerEntry in context.Request.Headers) + { + if (!NotForwardedWebSocketHeaders.Contains(headerEntry.Key, StringComparer.OrdinalIgnoreCase)) + { + client.Options.SetRequestHeader(headerEntry.Key, headerEntry.Value); + } + } + + try + { + await client.ConnectAsync(destinationUri, context.RequestAborted); + } + catch (WebSocketException) + { + context.Response.StatusCode = 400; + return false; + } + + using var server = await context.WebSockets.AcceptWebSocketAsync(client.SubProtocol); + + var bufferSize = DefaultWebSocketBufferSize; + await Task.WhenAll(PumpWebSocket(client, server, bufferSize, context.RequestAborted), PumpWebSocket(server, client, bufferSize, context.RequestAborted)); + + return true; + } + + private static async Task PumpWebSocket(WebSocket source, WebSocket destination, int bufferSize, CancellationToken cancellationToken) + { + if (bufferSize <= 0) + { + throw new ArgumentOutOfRangeException(nameof(bufferSize)); + } + + var buffer = new byte[bufferSize]; + while (true) + { + WebSocketReceiveResult result; + try + { + result = await source.ReceiveAsync(new ArraySegment(buffer), cancellationToken); + } + catch (OperationCanceledException) + { + await destination.CloseOutputAsync(WebSocketCloseStatus.EndpointUnavailable, null, cancellationToken); + return; + } + if (result.MessageType == WebSocketMessageType.Close) + { + await destination.CloseOutputAsync(source.CloseStatus!.Value, source.CloseStatusDescription, cancellationToken); + return; + } + await destination.SendAsync(new ArraySegment(buffer, 0, result.Count), result.MessageType, result.EndOfMessage, cancellationToken); + } + } + + public static Task SendProxyHttpRequest(this HttpContext context, HttpMessageInvoker invoker, HttpRequestMessage requestMessage) + { + if (requestMessage == null) + { + throw new ArgumentNullException(nameof(requestMessage)); + } + + return invoker.SendAsync(requestMessage, context.RequestAborted); + } + + public static async Task CopyProxyHttpResponse(this HttpContext context, HttpResponseMessage responseMessage) + { + if (responseMessage == null) + { + throw new ArgumentNullException(nameof(responseMessage)); + } + + var response = context.Response; + + response.StatusCode = (int)responseMessage.StatusCode; + foreach (var header in responseMessage.Headers) + { + response.Headers[header.Key] = header.Value.ToArray(); + } + + foreach (var header in responseMessage.Content.Headers) + { + response.Headers[header.Key] = header.Value.ToArray(); + } + + // SendAsync removes chunking from the response. This removes the header so it doesn't expect a chunked response. + response.Headers.Remove("transfer-encoding"); + + using (var responseStream = await responseMessage.Content.ReadAsStreamAsync()) + { + await responseStream.CopyToAsync(response.Body, StreamCopyBufferSize, context.RequestAborted); + } + } + } +} diff --git a/src/Microsoft.Tye.Hosting/TyeHost.cs b/src/Microsoft.Tye.Hosting/TyeHost.cs index 73ce2e4c..0af35ac2 100644 --- a/src/Microsoft.Tye.Hosting/TyeHost.cs +++ b/src/Microsoft.Tye.Hosting/TyeHost.cs @@ -257,6 +257,7 @@ namespace Microsoft.Tye.Hosting { new EventPipeDiagnosticsRunner(logger, diagnosticsCollector), new ProxyService(logger), + new IngressService(logger), new DockerRunner(logger), new ProcessRunner(logger, ProcessRunnerOptions.FromArgs(args, _servicesToDebug)), }; diff --git a/src/tye/ConfigModel/ConfigApplication.cs b/src/tye/ConfigModel/ConfigApplication.cs index 7607dfb6..8ec6622e 100644 --- a/src/tye/ConfigModel/ConfigApplication.cs +++ b/src/tye/ConfigModel/ConfigApplication.cs @@ -20,6 +20,8 @@ namespace Microsoft.Tye.ConfigModel public string? Registry { get; set; } + public List Ingress { get; set; } = new List(); + public List Services { get; set; } = new List(); public Tye.Hosting.Model.Application ToHostingApplication() @@ -90,6 +92,36 @@ namespace Microsoft.Tye.ConfigModel services.Add(service.Name, new Tye.Hosting.Model.Service(description)); } + foreach (var ingress in Ingress) + { + var rules = new List(); + + foreach (var rule in ingress.Rules) + { + rules.Add(new IngressRule(rule.Host, rule.Path, rule.Service!)); + } + + var runInfo = new IngressRunInfo(rules); + + var description = new Tye.Hosting.Model.ServiceDescription(ingress.Name, runInfo) + { + Replicas = ingress.Replicas ?? 1, + }; + + foreach (var binding in ingress.Bindings) + { + description.Bindings.Add(new Tye.Hosting.Model.ServiceBinding() + { + AutoAssignPort = binding.AutoAssignPort, + Name = binding.Name, + Port = binding.Port, + Protocol = binding.Protocol, + }); + } + + services.Add(ingress.Name, new Tye.Hosting.Model.Service(description)); + } + return new Tye.Hosting.Model.Application(Source, services); } } diff --git a/src/tye/ConfigModel/ConfigIngress.cs b/src/tye/ConfigModel/ConfigIngress.cs new file mode 100644 index 00000000..ac3be59d --- /dev/null +++ b/src/tye/ConfigModel/ConfigIngress.cs @@ -0,0 +1,18 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System; +using System.Collections.Generic; +using System.Text; + +namespace Microsoft.Tye.ConfigModel +{ + public class ConfigIngress + { + public string Name { get; set; } = default!; + public int? Replicas { get; set; } + public List Rules { get; set; } = new List(); + public List Bindings { get; set; } = new List(); + } +} diff --git a/src/tye/ConfigModel/ConfigIngressRule.cs b/src/tye/ConfigModel/ConfigIngressRule.cs new file mode 100644 index 00000000..a0228180 --- /dev/null +++ b/src/tye/ConfigModel/ConfigIngressRule.cs @@ -0,0 +1,17 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System; +using System.Collections.Generic; +using System.Text; + +namespace Microsoft.Tye.ConfigModel +{ + public class ConfigIngressRule + { + public string? Path { get; set; } + public string? Host { get; set; } + public string? Service { get; set; } + } +} diff --git a/src/tye/ConfigModel/ConfigIngressServiceBinding.cs b/src/tye/ConfigModel/ConfigIngressServiceBinding.cs new file mode 100644 index 00000000..a44f1a0a --- /dev/null +++ b/src/tye/ConfigModel/ConfigIngressServiceBinding.cs @@ -0,0 +1,18 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System; +using System.Collections.Generic; +using System.Text; + +namespace Microsoft.Tye.ConfigModel +{ + public class ConfigIngressServiceBinding + { + public string? Name { get; set; } + public bool AutoAssignPort { get; set; } + public int? Port { get; set; } + public string? Protocol { get; set; } // HTTP or HTTPS + } +} diff --git a/src/tye/ConfigModel/ConfigVolume.cs b/src/tye/ConfigModel/ConfigVolume.cs index 81a53b11..b0d3964d 100644 --- a/src/tye/ConfigModel/ConfigVolume.cs +++ b/src/tye/ConfigModel/ConfigVolume.cs @@ -1,4 +1,8 @@ -using System; +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System; using System.Collections.Generic; using System.Text; diff --git a/src/tye/InitHost.cs b/src/tye/InitHost.cs index 3853a971..c5ec45bc 100644 --- a/src/tye/InitHost.cs +++ b/src/tye/InitHost.cs @@ -1,5 +1,6 @@ using System.IO; using Microsoft.Tye.ConfigModel; +using Microsoft.Tye.Hosting.Dashboard; using YamlDotNet.Serialization; using YamlDotNet.Serialization.NamingConventions; @@ -71,6 +72,8 @@ services: // If the input file is a project or solution then use that as the name application.Name = Path.GetFileNameWithoutExtension(path.Name).ToLowerInvariant(); + application.Ingress = null!; + foreach (var service in application.Services) { service.Bindings = null!; diff --git a/test/E2ETest/RetryHandler.cs b/test/E2ETest/RetryHandler.cs index 9f50daee..c30e7e1c 100644 --- a/test/E2ETest/RetryHandler.cs +++ b/test/E2ETest/RetryHandler.cs @@ -13,7 +13,7 @@ namespace E2ETest public class RetryHandler : DelegatingHandler { private static readonly int MaxRetries = 5; - private static readonly TimeSpan InitialRetryDelay = TimeSpan.FromMilliseconds(100); + private static readonly TimeSpan InitialRetryDelay = TimeSpan.FromMilliseconds(500); public RetryHandler(HttpMessageHandler innerHandler) : base(innerHandler) diff --git a/test/E2ETest/TyeRunTests.cs b/test/E2ETest/TyeRunTests.cs index 2f63a815..10adcea0 100644 --- a/test/E2ETest/TyeRunTests.cs +++ b/test/E2ETest/TyeRunTests.cs @@ -135,6 +135,72 @@ namespace E2ETest } } + [Fact] + public async Task IngressRunTest() + { + var projectDirectory = new DirectoryInfo(Path.Combine(TestHelpers.GetSolutionRootDirectory("tye"), "samples", "apps-with-ingress")); + using var tempDirectory = TempDirectory.Create(); + DirectoryCopy.Copy(projectDirectory.FullName, tempDirectory.DirectoryPath); + + var projectFile = new FileInfo(Path.Combine(tempDirectory.DirectoryPath, "tye.yaml")); + using var host = new TyeHost(ConfigFactory.FromFile(projectFile).ToHostingApplication(), Array.Empty()) + { + Sink = sink, + }; + + var handler = new HttpClientHandler + { + ServerCertificateCustomValidationCallback = (a, b, c, d) => true, + AllowAutoRedirect = false + }; + + using var client = new HttpClient(new RetryHandler(handler)); + await host.StartAsync(); + var serviceApi = new Uri(host.DashboardWebApplication!.Addresses.First()); + + try + { + var ingressService = await client.GetStringAsync($"{serviceApi}api/v1/services/ingress"); + + var service = JsonSerializer.Deserialize(ingressService, _options); + var binding = service.Description!.Bindings.Single(); + var ingressUri = $"http://localhost:{binding.Port}"; + + var responseA = await client.GetAsync(ingressUri + "/A"); + var responseB = await client.GetAsync(ingressUri + "/B"); + + Assert.StartsWith("Hello from Application A", await responseA.Content.ReadAsStringAsync()); + Assert.StartsWith("Hello from Application B", await responseB.Content.ReadAsStringAsync()); + + var requestA = new HttpRequestMessage(HttpMethod.Get, ingressUri); + requestA.Headers.Host = "a.example.com"; + var requestB = new HttpRequestMessage(HttpMethod.Get, ingressUri); + requestB.Headers.Host = "b.example.com"; + + responseA = await client.SendAsync(requestA); + responseB = await client.SendAsync(requestB); + + Assert.StartsWith("Hello from Application A", await responseA.Content.ReadAsStringAsync()); + Assert.StartsWith("Hello from Application B", await responseB.Content.ReadAsStringAsync()); + } + finally + { + // If we failed, there's a good chance the service isn't running. Let's get the logs either way and put + // them in the output. + foreach (var s in host.Application.Services.Values) + { + var request = new HttpRequestMessage(HttpMethod.Get, new Uri(serviceApi, $"/api/v1/logs/{s.Description.Name}")); + var response = await client.SendAsync(request); + var text = await response.Content.ReadAsStringAsync(); + + output.WriteLine($"Logs for service: {s.Description.Name}"); + output.WriteLine(text); + } + + await host.StopAsync(); + } + } + [ConditionalFact] [SkipIfDockerNotRunning] [SkipOnLinux]