From ba1a5c2f36a574c7ef127d0e5a1e50efe0427147 Mon Sep 17 00:00:00 2001 From: Justin Kotalik Date: Tue, 28 Apr 2020 18:56:40 -0700 Subject: [PATCH] Add concept of services have direct dependencies (#414) --- src/Microsoft.Tye.Core/ApplicationFactory.cs | 30 ++++- src/Microsoft.Tye.Core/CombineStep.cs | 8 +- src/Microsoft.Tye.Core/ServiceBuilder.cs | 2 + .../Model/Application.cs | 4 +- .../Model/ServiceDescription.cs | 1 + .../Properties/AssemblyInfo.cs | 7 ++ src/tye/ApplicationBuilderExtensions.cs | 3 + .../ApplicationFactoryTests.cs | 2 +- test/E2ETest/ApplicationTests.cs | 43 +++++++ test/E2ETest/Microsoft.Tye.E2ETests.csproj | 2 +- test/E2ETest/TyeGenerateTests.cs | 38 ++++++ test/E2ETest/TyeRunTests.cs | 1 + .../testassets/generate/multirepo.yaml | 113 ++++++++++++++++++ .../projects/multirepo/results/tye.yaml | 2 +- .../projects/multirepo/vote/tye.yaml | 5 + .../TyeDeserializationValidationTests.cs | 5 +- 16 files changed, 249 insertions(+), 17 deletions(-) create mode 100644 src/Microsoft.Tye.Hosting/Properties/AssemblyInfo.cs rename test/{UnitTests => E2ETest}/ApplicationFactoryTests.cs (99%) create mode 100644 test/E2ETest/ApplicationTests.cs create mode 100644 test/E2ETest/testassets/generate/multirepo.yaml diff --git a/src/Microsoft.Tye.Core/ApplicationFactory.cs b/src/Microsoft.Tye.Core/ApplicationFactory.cs index 2a09bc3a..4e1be27b 100644 --- a/src/Microsoft.Tye.Core/ApplicationFactory.cs +++ b/src/Microsoft.Tye.Core/ApplicationFactory.cs @@ -21,7 +21,7 @@ namespace Microsoft.Tye throw new ArgumentNullException(nameof(source)); } - var queue = new Queue(); + var queue = new Queue<(ConfigApplication, HashSet)>(); var visited = new HashSet(StringComparer.OrdinalIgnoreCase); var rootConfig = ConfigFactory.FromFile(source); @@ -29,11 +29,14 @@ namespace Microsoft.Tye var root = new ApplicationBuilder(source, rootConfig.Name ?? source.Directory.Name.ToLowerInvariant()); root.Namespace = rootConfig.Namespace; - queue.Enqueue(rootConfig); + + queue.Enqueue((rootConfig, new HashSet())); while (queue.Count > 0) { - var config = queue.Dequeue(); + var item = queue.Dequeue(); + var config = item.Item1; + var parentDependencies = item.Item2; if (!visited.Add(config.Source.FullName)) { continue; @@ -70,6 +73,7 @@ namespace Microsoft.Tye ServiceBuilder service; if (root.Services.Any(s => s.Name == configService.Name)) { + AddToRootServices(root, parentDependencies, configService, configService.Name); // Don't add a service which has already been added by name continue; } @@ -140,8 +144,9 @@ namespace Microsoft.Tye throw new CommandException($"Nested configuration must have the same \"name\" in the tye.yaml. Root config: {rootConfig.Source}, nested config: {nestedConfig.Source}"); } - queue.Enqueue(nestedConfig); + queue.Enqueue((nestedConfig, new HashSet())); + AddToRootServices(root, parentDependencies, configService, configService.Name); continue; } @@ -155,6 +160,11 @@ namespace Microsoft.Tye throw new CommandException("Unable to determine service type."); } + service.Dependencies.AddRange(parentDependencies); + parentDependencies.Add(service.Name); + + AddToRootServices(root, parentDependencies, configService, service.Name); + root.Services.Add(service); // If there are no bindings and we're in ASP.NET Core project then add an HTTP and HTTPS binding @@ -274,5 +284,17 @@ namespace Microsoft.Tye return root; } + + private static void AddToRootServices(ApplicationBuilder root, HashSet parentDependencies, ConfigService configService, string serviceName) + { + parentDependencies.Add(serviceName); + foreach (var s in root.Services) + { + if (parentDependencies.Contains(s.Name, StringComparer.OrdinalIgnoreCase) && !s.Name.Equals(configService.Name, StringComparison.OrdinalIgnoreCase)) + { + s.Dependencies.Add(serviceName); + } + } + } } } diff --git a/src/Microsoft.Tye.Core/CombineStep.cs b/src/Microsoft.Tye.Core/CombineStep.cs index 479f517d..823f0b41 100644 --- a/src/Microsoft.Tye.Core/CombineStep.cs +++ b/src/Microsoft.Tye.Core/CombineStep.cs @@ -3,6 +3,7 @@ // See the LICENSE file in the project root for more information. using System.Globalization; +using System.Linq; using System.Threading.Tasks; namespace Microsoft.Tye @@ -51,12 +52,9 @@ namespace Microsoft.Tye var bindings = new ComputedBindings(); service.Outputs.Add(bindings); - foreach (var other in application.Services) + foreach (var o in service.Dependencies) { - if (object.ReferenceEquals(service, other)) - { - continue; - } + var other = application.Services.Single(a => a.Name == o); foreach (var binding in other.Bindings) { diff --git a/src/Microsoft.Tye.Core/ServiceBuilder.cs b/src/Microsoft.Tye.Core/ServiceBuilder.cs index 95ede542..18cdc0de 100644 --- a/src/Microsoft.Tye.Core/ServiceBuilder.cs +++ b/src/Microsoft.Tye.Core/ServiceBuilder.cs @@ -15,5 +15,7 @@ namespace Microsoft.Tye // TODO: this is temporary while refactoring public List Outputs { get; } = new List(); + + public List Dependencies { get; } = new List(); } } diff --git a/src/Microsoft.Tye.Hosting/Model/Application.cs b/src/Microsoft.Tye.Hosting/Model/Application.cs index 34873c30..ac2a7fe0 100644 --- a/src/Microsoft.Tye.Hosting/Model/Application.cs +++ b/src/Microsoft.Tye.Hosting/Model/Application.cs @@ -132,8 +132,10 @@ namespace Microsoft.Tye.Hosting.Model { var bindings = new List(); - foreach (var s in Services.Values) + foreach (var serv in service.Description.Dependencies) { + var s = Services[serv]; + foreach (var b in s.Description.Bindings) { var protocol = b.Protocol; diff --git a/src/Microsoft.Tye.Hosting/Model/ServiceDescription.cs b/src/Microsoft.Tye.Hosting/Model/ServiceDescription.cs index fa09033e..52df9f69 100644 --- a/src/Microsoft.Tye.Hosting/Model/ServiceDescription.cs +++ b/src/Microsoft.Tye.Hosting/Model/ServiceDescription.cs @@ -19,5 +19,6 @@ namespace Microsoft.Tye.Hosting.Model public int Replicas { get; set; } = 1; public List Bindings { get; } = new List(); public List Configuration { get; } = new List(); + public List Dependencies { get; } = new List(); } } diff --git a/src/Microsoft.Tye.Hosting/Properties/AssemblyInfo.cs b/src/Microsoft.Tye.Hosting/Properties/AssemblyInfo.cs new file mode 100644 index 00000000..b27e7aca --- /dev/null +++ b/src/Microsoft.Tye.Hosting/Properties/AssemblyInfo.cs @@ -0,0 +1,7 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System.Runtime.CompilerServices; + +[assembly: InternalsVisibleTo("Microsoft.Tye.UnitTests, PublicKey=002400000480000094000000060200000024000052534131000400000100010007d1fa57c4aed9f0a32e84aa0faefd0de9e8fd6aec8f87fb03766c834c99921eb23be79ad9d5dcc1dd9ad236132102900b723cf980957fc4e177108fc607774f29e8320e92ea05ece4e821c0a5efe8f1645c4c0c93c1ab99285d622caa652c1dfad63d745d6f2de5f17e5eaf0fc4963d261c8a12436518206dc093344d5ad293")] diff --git a/src/tye/ApplicationBuilderExtensions.cs b/src/tye/ApplicationBuilderExtensions.cs index 8ac4f471..e8c351a2 100644 --- a/src/tye/ApplicationBuilderExtensions.cs +++ b/src/tye/ApplicationBuilderExtensions.cs @@ -31,6 +31,7 @@ namespace Microsoft.Tye public static Application ToHostingApplication(this ApplicationBuilder application) { var services = new Dictionary(); + foreach (var service in application.Services) { RunInfo? runInfo; @@ -104,7 +105,9 @@ namespace Microsoft.Tye { Replicas = replicas, }; + description.Configuration.AddRange(env); + description.Dependencies.AddRange(service.Dependencies); foreach (var binding in service.Bindings) { diff --git a/test/UnitTests/ApplicationFactoryTests.cs b/test/E2ETest/ApplicationFactoryTests.cs similarity index 99% rename from test/UnitTests/ApplicationFactoryTests.cs rename to test/E2ETest/ApplicationFactoryTests.cs index eef19c11..4ac412bf 100644 --- a/test/UnitTests/ApplicationFactoryTests.cs +++ b/test/E2ETest/ApplicationFactoryTests.cs @@ -6,7 +6,7 @@ using Test.Infrastructure; using Xunit; using Xunit.Abstractions; -namespace UnitTests +namespace E2ETest { public class ApplicationFactoryTests { diff --git a/test/E2ETest/ApplicationTests.cs b/test/E2ETest/ApplicationTests.cs new file mode 100644 index 00000000..b4dc11cf --- /dev/null +++ b/test/E2ETest/ApplicationTests.cs @@ -0,0 +1,43 @@ +using System.Collections.Generic; +using System.IO; +using System.Threading.Tasks; +using Microsoft.Tye; +using Test.Infrastructure; +using Xunit; +using Xunit.Abstractions; + +namespace E2ETest +{ + public class ApplicationTests + { + private readonly TestOutputLogEventSink _sink; + + public ApplicationTests(ITestOutputHelper output) + { + _sink = new TestOutputLogEventSink(output); + } + + [Fact] + public async Task EnvironmentVariablesOnlySetForDirectDependencies() + { + using var projectDirectory = TestHelpers.CopyTestProjectDirectory(Path.Combine("multirepo")); + var yamlFile = Path.Combine(projectDirectory.DirectoryPath, "results", "tye.yaml"); + + // Debug targets can be null if not specified, so make sure calling host.Start does not throw. + var outputContext = new OutputContext(_sink, Verbosity.Debug); + var application = await ApplicationFactory.CreateAsync(outputContext, new FileInfo(yamlFile)); + + var app = application.ToHostingApplication(); + var dictionary = new Dictionary(); + app.PopulateEnvironment(app.Services["results"], (s1, s2) => dictionary[s1] = s2); + + // Just the WORKER is defined. + Assert.Equal(8, dictionary.Count); + + Assert.Equal("http", dictionary["SERVICE__WORKER__PROTOCOL"]); + // No POSTGRES or REDIS + Assert.False(dictionary.ContainsKey("SERVICE__POSTGRES__PROTOCOL")); + Assert.False(dictionary.ContainsKey("SERVICE__REDIS__PROTOCOL")); + } + } +} diff --git a/test/E2ETest/Microsoft.Tye.E2ETests.csproj b/test/E2ETest/Microsoft.Tye.E2ETests.csproj index c5e4db30..92a15968 100644 --- a/test/E2ETest/Microsoft.Tye.E2ETests.csproj +++ b/test/E2ETest/Microsoft.Tye.E2ETests.csproj @@ -1,4 +1,4 @@ - + netcoreapp3.1 diff --git a/test/E2ETest/TyeGenerateTests.cs b/test/E2ETest/TyeGenerateTests.cs index adf23475..03973904 100644 --- a/test/E2ETest/TyeGenerateTests.cs +++ b/test/E2ETest/TyeGenerateTests.cs @@ -318,6 +318,44 @@ namespace E2ETest } } + [ConditionalFact] + [SkipIfDockerNotRunning] + public async Task Generate_DirectDependencyForEnvVars() + { + var applicationName = "multirepo"; + var projectName = "results"; + var otherProject = "worker"; + var environment = "production"; + + await DockerAssert.DeleteDockerImagesAsync(output, projectName); + await DockerAssert.DeleteDockerImagesAsync(output, otherProject); + + using var projectDirectory = TestHelpers.CopyTestProjectDirectory(applicationName); + + var projectFile = new FileInfo(Path.Combine(projectDirectory.DirectoryPath, "results", "tye.yaml")); + + var outputContext = new OutputContext(sink, Verbosity.Debug); + var application = await ApplicationFactory.CreateAsync(outputContext, projectFile); + + try + { + await GenerateHost.ExecuteGenerateAsync(outputContext, application, environment, interactive: false); + + // name of application is the folder + var content = await File.ReadAllTextAsync(Path.Combine(projectDirectory.DirectoryPath, "results", $"VotingSample-generate-{environment}.yaml")); + var expectedContent = await File.ReadAllTextAsync($"testassets/generate/{applicationName}.yaml"); + + YamlAssert.Equals(expectedContent, content, output); + await DockerAssert.AssertImageExistsAsync(output, projectName); + await DockerAssert.AssertImageExistsAsync(output, otherProject); + } + finally + { + await DockerAssert.DeleteDockerImagesAsync(output, projectName); + await DockerAssert.DeleteDockerImagesAsync(output, otherProject); + } + } + [ConditionalFact] [SkipIfDockerNotRunning] public async Task Generate_Ingress() diff --git a/test/E2ETest/TyeRunTests.cs b/test/E2ETest/TyeRunTests.cs index 11d90020..346cfc1f 100644 --- a/test/E2ETest/TyeRunTests.cs +++ b/test/E2ETest/TyeRunTests.cs @@ -210,6 +210,7 @@ namespace E2ETest var outputFileName = project.AssemblyName + ".dll"; var container = new ContainerServiceBuilder(project.Name, $"mcr.microsoft.com/dotnet/core/sdk:{project.TargetFrameworkVersion}"); + container.Dependencies.AddRange(project.Dependencies); container.Volumes.Add(new VolumeBuilder(project.PublishDir, name: null, target: "/app")); container.Args = $"dotnet /app/{outputFileName} {project.Args}"; container.Bindings.AddRange(project.Bindings); diff --git a/test/E2ETest/testassets/generate/multirepo.yaml b/test/E2ETest/testassets/generate/multirepo.yaml new file mode 100644 index 00000000..dce6a2a6 --- /dev/null +++ b/test/E2ETest/testassets/generate/multirepo.yaml @@ -0,0 +1,113 @@ +kind: Deployment +apiVersion: apps/v1 +metadata: + name: results + labels: + app.kubernetes.io/name: 'results' + app.kubernetes.io/part-of: 'VotingSample' +spec: + replicas: 1 + selector: + matchLabels: + app.kubernetes.io/name: results + template: + metadata: + labels: + app.kubernetes.io/name: 'results' + app.kubernetes.io/part-of: 'VotingSample' + spec: + containers: + - name: results + image: results:1.0.0 + imagePullPolicy: Always + env: + - name: ASPNETCORE_URLS + value: 'http://*' + - name: PORT + value: '80' + - name: SERVICE__WORKER__PROTOCOL + value: 'http' + - name: SERVICE__WORKER__PORT + value: '80' + - name: SERVICE__WORKER__HOST + value: 'worker' + ports: + - containerPort: 80 +... +--- +kind: Service +apiVersion: v1 +metadata: + name: results + labels: + app.kubernetes.io/name: 'results' + app.kubernetes.io/part-of: 'VotingSample' +spec: + selector: + app.kubernetes.io/name: results + type: ClusterIP + ports: + - name: http + protocol: TCP + port: 80 + targetPort: 80 +... +--- +kind: Deployment +apiVersion: apps/v1 +metadata: + name: worker + labels: + app.kubernetes.io/name: 'worker' + app.kubernetes.io/part-of: 'VotingSample' +spec: + replicas: 1 + selector: + matchLabels: + app.kubernetes.io/name: worker + template: + metadata: + labels: + app.kubernetes.io/name: 'worker' + app.kubernetes.io/part-of: 'VotingSample' + spec: + containers: + - name: worker + image: worker:1.0.0 + imagePullPolicy: Always + env: + - name: ASPNETCORE_URLS + value: 'http://*' + - name: PORT + value: '80' + - name: CONNECTIONSTRINGS__REDIS + valueFrom: + secretKeyRef: + name: 'binding-production-redis-secret' + key: 'connectionstring' + - name: CONNECTIONSTRINGS__POSTGRES + valueFrom: + secretKeyRef: + name: 'binding-production-postgres-secret' + key: 'connectionstring' + ports: + - containerPort: 80 +... +--- +kind: Service +apiVersion: v1 +metadata: + name: worker + labels: + app.kubernetes.io/name: 'worker' + app.kubernetes.io/part-of: 'VotingSample' +spec: + selector: + app.kubernetes.io/name: worker + type: ClusterIP + ports: + - name: http + protocol: TCP + port: 80 + targetPort: 80 +... diff --git a/test/E2ETest/testassets/projects/multirepo/results/tye.yaml b/test/E2ETest/testassets/projects/multirepo/results/tye.yaml index 48f494f2..7dbb2970 100644 --- a/test/E2ETest/testassets/projects/multirepo/results/tye.yaml +++ b/test/E2ETest/testassets/projects/multirepo/results/tye.yaml @@ -1,6 +1,6 @@ name: VotingSample services: -- name: include +- name: worker include: ../worker/tye.yaml - name: results project: results.csproj diff --git a/test/E2ETest/testassets/projects/multirepo/vote/tye.yaml b/test/E2ETest/testassets/projects/multirepo/vote/tye.yaml index 14d0bec7..739361fc 100644 --- a/test/E2ETest/testassets/projects/multirepo/vote/tye.yaml +++ b/test/E2ETest/testassets/projects/multirepo/vote/tye.yaml @@ -4,3 +4,8 @@ services: project: vote.csproj - name: worker include: ../worker/tye.yaml +- name: redis + image: redis + bindings: + - port: 6379 + connectionString: ${host}:${port} diff --git a/test/UnitTests/TyeDeserializationValidationTests.cs b/test/UnitTests/TyeDeserializationValidationTests.cs index 8b39714e..58cae2f5 100644 --- a/test/UnitTests/TyeDeserializationValidationTests.cs +++ b/test/UnitTests/TyeDeserializationValidationTests.cs @@ -1,7 +1,4 @@ -using System; -using System.Collections.Generic; -using System.Text; -using Tye; +using Tye; using Tye.Serialization; using Xunit;