diff --git a/src/Microsoft.Tye.Core/ApplicationFactory.cs b/src/Microsoft.Tye.Core/ApplicationFactory.cs index 4e1be27b..f738e708 100644 --- a/src/Microsoft.Tye.Core/ApplicationFactory.cs +++ b/src/Microsoft.Tye.Core/ApplicationFactory.cs @@ -36,7 +36,9 @@ namespace Microsoft.Tye { var item = queue.Dequeue(); var config = item.Item1; - var parentDependencies = item.Item2; + + // dependencies represents a set of all dependencies + var dependencies = item.Item2; if (!visited.Add(config.Source.FullName)) { continue; @@ -73,8 +75,9 @@ namespace Microsoft.Tye ServiceBuilder service; if (root.Services.Any(s => s.Name == configService.Name)) { - AddToRootServices(root, parentDependencies, configService, configService.Name); - // Don't add a service which has already been added by name + // Even though this service has already created a service, we still need + // to update dependency information + AddToRootServices(root, dependencies, configService.Name); continue; } @@ -136,17 +139,50 @@ namespace Microsoft.Tye else if (!string.IsNullOrEmpty(configService.Include)) { var expandedYaml = Environment.ExpandEnvironmentVariables(configService.Include); - var nestedConfig = ConfigFactory.FromFile(new FileInfo(Path.Combine(config.Source.DirectoryName, expandedYaml))); - nestedConfig.Validate(); - if (nestedConfig.Name != rootConfig.Name) + var nestedConfig = GetNestedConfig(rootConfig, Path.Combine(config.Source.DirectoryName, expandedYaml)); + queue.Enqueue((nestedConfig, new HashSet())); + + AddToRootServices(root, dependencies, configService.Name); + continue; + } + else if (!string.IsNullOrEmpty(configService.Repository)) + { + // clone to .tye folder + var path = Path.Join(rootConfig.Source.DirectoryName, ".tye", "deps"); + if (!Directory.Exists(path)) + { + Directory.CreateDirectory(path); + } + + var clonePath = Path.Combine(path, configService.Name); + + if (!Directory.Exists(clonePath)) + { + if (!await GitDetector.Instance.IsGitInstalled.Value) + { + throw new CommandException($"Cannot clone repository {configService.Repository} because git is not installed. Please install git if you'd like to use \"repository\" in tye.yaml."); + } + + var result = await ProcessUtil.RunAsync("git", $"clone {configService.Repository} {clonePath}", workingDirectory: path, throwOnError: false); + + if (result.ExitCode != 0) + { + throw new CommandException($"Failed to clone repository {configService.Repository} with exit code {result.ExitCode}.{Environment.NewLine}{result.StandardError}{result.StandardOutput}."); + } + } + + if (!ConfigFileFinder.TryFindSupportedFile(clonePath, out var file, out var errorMessage)) { - throw new CommandException($"Nested configuration must have the same \"name\" in the tye.yaml. Root config: {rootConfig.Source}, nested config: {nestedConfig.Source}"); + throw new CommandException(errorMessage!); } + // pick different service type based on what is in the repo. + var nestedConfig = GetNestedConfig(rootConfig, file); + queue.Enqueue((nestedConfig, new HashSet())); - AddToRootServices(root, parentDependencies, configService, configService.Name); + AddToRootServices(root, dependencies, configService.Name); continue; } @@ -160,10 +196,10 @@ namespace Microsoft.Tye throw new CommandException("Unable to determine service type."); } - service.Dependencies.AddRange(parentDependencies); - parentDependencies.Add(service.Name); + // Add dependencies to ourself before adding ourself to avoid self reference + service.Dependencies.UnionWith(dependencies); - AddToRootServices(root, parentDependencies, configService, service.Name); + AddToRootServices(root, dependencies, service.Name); root.Services.Add(service); @@ -285,12 +321,29 @@ namespace Microsoft.Tye return root; } - private static void AddToRootServices(ApplicationBuilder root, HashSet parentDependencies, ConfigService configService, string serviceName) + private static ConfigApplication GetNestedConfig(ConfigApplication rootConfig, string? file) + { + var nestedConfig = ConfigFactory.FromFile(new FileInfo(file)); + nestedConfig.Validate(); + + if (nestedConfig.Name != rootConfig.Name) + { + throw new CommandException($"Nested configuration must have the same \"name\" in the tye.yaml. Root config: {rootConfig.Source}, nested config: {nestedConfig.Source}"); + } + + return nestedConfig; + } + + private static void AddToRootServices(ApplicationBuilder root, HashSet dependencies, string serviceName) { - parentDependencies.Add(serviceName); + // Add ourselves in the set of all current dependencies. + dependencies.Add(serviceName); + + // Iterate through all services and add the current services as a dependency (except ourselves) foreach (var s in root.Services) { - if (parentDependencies.Contains(s.Name, StringComparer.OrdinalIgnoreCase) && !s.Name.Equals(configService.Name, StringComparison.OrdinalIgnoreCase)) + if (dependencies.Contains(s.Name, StringComparer.OrdinalIgnoreCase) + && !s.Name.Equals(serviceName, StringComparison.OrdinalIgnoreCase)) { s.Dependencies.Add(serviceName); } diff --git a/src/Microsoft.Tye.Core/ConfigFileFinder.cs b/src/Microsoft.Tye.Core/ConfigFileFinder.cs new file mode 100644 index 00000000..d724dda8 --- /dev/null +++ b/src/Microsoft.Tye.Core/ConfigFileFinder.cs @@ -0,0 +1,39 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System.IO; + +namespace Microsoft.Tye +{ + public class ConfigFileFinder + { + private static readonly string[] FileFormats = new[] { "tye.yaml", "tye.yml", "*.csproj", "*.fsproj", "*.sln" }; + + public static bool TryFindSupportedFile(string directoryPath, out string? filePath, out string? errorMessage) + { + foreach (var format in FileFormats) + { + var files = Directory.GetFiles(directoryPath, format); + + if (files.Length == 1) + { + errorMessage = null; + filePath = files[0]; + return true; + } + + if (files.Length > 1) + { + errorMessage = $"More than one matching file was found in directory '{directoryPath}'."; + filePath = default; + return false; + } + } + + errorMessage = $"No project project file or solution was found in directory '{directoryPath}'."; + filePath = default; + return false; + } + } +} diff --git a/src/Microsoft.Tye.Core/ConfigModel/ConfigService.cs b/src/Microsoft.Tye.Core/ConfigModel/ConfigService.cs index 9a3973bc..36cad357 100644 --- a/src/Microsoft.Tye.Core/ConfigModel/ConfigService.cs +++ b/src/Microsoft.Tye.Core/ConfigModel/ConfigService.cs @@ -16,6 +16,7 @@ namespace Microsoft.Tye.ConfigModel public string? Image { get; set; } public string? Project { get; set; } public string? Include { get; set; } + public string? Repository { get; set; } public bool? Build { get; set; } public string? Executable { get; set; } public string? WorkingDirectory { get; set; } diff --git a/src/Microsoft.Tye.Core/GitDetector.cs b/src/Microsoft.Tye.Core/GitDetector.cs new file mode 100644 index 00000000..79417004 --- /dev/null +++ b/src/Microsoft.Tye.Core/GitDetector.cs @@ -0,0 +1,35 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System; +using System.Threading.Tasks; + +namespace Microsoft.Tye +{ + public class GitDetector + { + public static GitDetector Instance { get; } = new GitDetector(); + + private GitDetector() + { + IsGitInstalled = new Lazy>(GetIsGitInstalled); + } + + public Lazy> IsGitInstalled { get; } + + private async Task GetIsGitInstalled() + { + try + { + var result = await ProcessUtil.RunAsync("git", "--version", throwOnError: false); + return result.ExitCode == 0; + } + catch (Exception) + { + // Unfortunately, process throws + return false; + } + } + } +} diff --git a/src/Microsoft.Tye.Core/Microsoft.Tye.Core.csproj b/src/Microsoft.Tye.Core/Microsoft.Tye.Core.csproj index b3451504..9ce3e6b3 100644 --- a/src/Microsoft.Tye.Core/Microsoft.Tye.Core.csproj +++ b/src/Microsoft.Tye.Core/Microsoft.Tye.Core.csproj @@ -37,6 +37,7 @@ + diff --git a/src/Microsoft.Tye.Core/Serialization/ConfigServiceParser.cs b/src/Microsoft.Tye.Core/Serialization/ConfigServiceParser.cs index f5d763c4..0d5c91f0 100644 --- a/src/Microsoft.Tye.Core/Serialization/ConfigServiceParser.cs +++ b/src/Microsoft.Tye.Core/Serialization/ConfigServiceParser.cs @@ -55,6 +55,9 @@ namespace Tye.Serialization case "include": service.Include = YamlParser.GetScalarValue(key, child.Value); break; + case "repository": + service.Repository = YamlParser.GetScalarValue(key, child.Value); + break; case "build": if (!bool.TryParse(YamlParser.GetScalarValue(key, child.Value), out var build)) { diff --git a/src/Microsoft.Tye.Core/ServiceBuilder.cs b/src/Microsoft.Tye.Core/ServiceBuilder.cs index 18cdc0de..dfcb0a9e 100644 --- a/src/Microsoft.Tye.Core/ServiceBuilder.cs +++ b/src/Microsoft.Tye.Core/ServiceBuilder.cs @@ -16,6 +16,6 @@ namespace Microsoft.Tye // TODO: this is temporary while refactoring public List Outputs { get; } = new List(); - public List Dependencies { get; } = new List(); + public HashSet Dependencies { get; } = new HashSet(); } } diff --git a/src/Microsoft.Tye.Core/TempDirectory.cs b/src/Microsoft.Tye.Core/TempDirectory.cs index b730b69a..984cf161 100644 --- a/src/Microsoft.Tye.Core/TempDirectory.cs +++ b/src/Microsoft.Tye.Core/TempDirectory.cs @@ -42,7 +42,7 @@ namespace Microsoft.Tye public void Dispose() { - Directory.Delete(DirectoryPath, recursive: true); + DirectoryExtensions.DeleteDirectory(DirectoryPath); } } } diff --git a/src/Microsoft.Tye.Hosting/Microsoft.Tye.Hosting.csproj b/src/Microsoft.Tye.Hosting/Microsoft.Tye.Hosting.csproj index 57fb8d46..d9a13d02 100644 --- a/src/Microsoft.Tye.Hosting/Microsoft.Tye.Hosting.csproj +++ b/src/Microsoft.Tye.Hosting/Microsoft.Tye.Hosting.csproj @@ -32,4 +32,8 @@ + + + + diff --git a/src/Microsoft.Tye.Hosting/ReplicaRegistry.cs b/src/Microsoft.Tye.Hosting/ReplicaRegistry.cs index 6ad7f934..53efeef9 100644 --- a/src/Microsoft.Tye.Hosting/ReplicaRegistry.cs +++ b/src/Microsoft.Tye.Hosting/ReplicaRegistry.cs @@ -101,7 +101,14 @@ namespace Microsoft.Tye.Hosting { if (Directory.Exists(_tyeFolderPath)) { - Directory.Delete(_tyeFolderPath, true); + try + { + DirectoryExtensions.DeleteDirectory(_tyeFolderPath); + } + catch (Exception ex) + { + Console.WriteLine(ex.Message); + } } } } diff --git a/src/shared/DirectoryExtensions.cs b/src/shared/DirectoryExtensions.cs new file mode 100644 index 00000000..11b335aa --- /dev/null +++ b/src/shared/DirectoryExtensions.cs @@ -0,0 +1,28 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System.IO; + +namespace Microsoft.Tye +{ + internal static class DirectoryExtensions + { + // Calling Directory.Delete causes an exception for .git folders: + // System.UnauthorizedAccessException : Access to the path '17a475ecca365c678e907bd4c73e4c65b341c6' is denied. + public static void DeleteDirectory(string d) + { + foreach (var sub in Directory.EnumerateDirectories(d)) + { + DeleteDirectory(sub); + } + foreach (var f in Directory.EnumerateFiles(d)) + { + var fi = new FileInfo(f); + fi.Attributes = FileAttributes.Normal; + fi.Delete(); + } + Directory.Delete(d); + } + } +} diff --git a/src/tye/CommonArguments.cs b/src/tye/CommonArguments.cs index 0709e700..1a24d285 100644 --- a/src/tye/CommonArguments.cs +++ b/src/tye/CommonArguments.cs @@ -11,8 +11,6 @@ namespace Microsoft.Tye { internal static class CommonArguments { - private static readonly string[] FileFormats = new[] { "tye.yaml", "tye.yml", "*.csproj", "*.fsproj", "*.sln" }; - public static Argument Path_Optional { get @@ -39,32 +37,6 @@ namespace Microsoft.Tye } } - public static bool TryFindSupportedFile(string directoryPath, out string? filePath, out string? errorMessage) - { - foreach (var format in FileFormats) - { - var files = Directory.GetFiles(directoryPath, format); - - if (files.Length == 1) - { - errorMessage = null; - filePath = files[0]; - return true; - } - - if (files.Length > 1) - { - errorMessage = $"More than one matching file was found in directory '{directoryPath}'."; - filePath = default; - return false; - } - } - - errorMessage = $"No project project file or solution was found in directory '{directoryPath}'."; - filePath = default; - return false; - } - static FileInfo TryParsePath(ArgumentResult result, bool required) { var token = result.Tokens.Count switch @@ -86,7 +58,7 @@ namespace Microsoft.Tye if (Directory.Exists(token)) { - if (TryFindSupportedFile(token, out var filePath, out var errorMessage)) + if (ConfigFileFinder.TryFindSupportedFile(token, out var filePath, out var errorMessage)) { return new FileInfo(filePath); } diff --git a/test/E2ETest/ApplicationFactoryTests.cs b/test/E2ETest/ApplicationFactoryTests.cs index 4ac412bf..88ab1b7d 100644 --- a/test/E2ETest/ApplicationFactoryTests.cs +++ b/test/E2ETest/ApplicationFactoryTests.cs @@ -55,6 +55,31 @@ services: var application = await ApplicationFactory.CreateAsync(outputContext, new FileInfo(yamlFile)); Assert.Equal(5, application.Services.Count); + + var vote = application.Services.Single(s => s.Name == "vote"); + Assert.Equal(2, vote.Dependencies.Count); + Assert.Contains("worker", vote.Dependencies); + Assert.Contains("redis", vote.Dependencies); + + var results = application.Services.Single(s => s.Name == "results"); + Assert.Single(results.Dependencies); + Assert.Contains("worker", results.Dependencies); + + var worker = application.Services.Single(s => s.Name == "worker"); + Assert.Equal(2, worker.Dependencies.Count); + Assert.Contains("redis", worker.Dependencies); + Assert.Contains("postgres", worker.Dependencies); + + var redis = application.Services.Single(s => s.Name == "redis"); + Assert.Equal(3, redis.Dependencies.Count); + Assert.Contains("postgres", redis.Dependencies); + Assert.Contains("worker", redis.Dependencies); + Assert.Contains("vote", redis.Dependencies); + + var postgres = application.Services.Single(s => s.Name == "postgres"); + Assert.Equal(2, postgres.Dependencies.Count); + Assert.Contains("worker", postgres.Dependencies); + Assert.Contains("redis", postgres.Dependencies); } [Fact] diff --git a/test/E2ETest/TyeRunTests.cs b/test/E2ETest/TyeRunTests.cs index 346cfc1f..b96ed29f 100644 --- a/test/E2ETest/TyeRunTests.cs +++ b/test/E2ETest/TyeRunTests.cs @@ -210,7 +210,7 @@ namespace E2ETest var outputFileName = project.AssemblyName + ".dll"; var container = new ContainerServiceBuilder(project.Name, $"mcr.microsoft.com/dotnet/core/sdk:{project.TargetFrameworkVersion}"); - container.Dependencies.AddRange(project.Dependencies); + container.Dependencies.UnionWith(project.Dependencies); container.Volumes.Add(new VolumeBuilder(project.PublishDir, name: null, target: "/app")); container.Args = $"dotnet /app/{outputFileName} {project.Args}"; container.Bindings.AddRange(project.Bindings); @@ -579,6 +579,47 @@ namespace E2ETest }); } + [ConditionalFact] + [SkipIfDockerNotRunning] + public async Task MultiRepo_WorksWithCloning() + { + using var projectDirectory = TempDirectory.Create(preferUserDirectoryOnMacOS: true); + + var content = @" +name: VotingSample +services: +- name: vote + repository: https://github.com/jkotalik/TyeMultiRepoVoting +- name: results + repository: https://github.com/jkotalik/TyeMultiRepoResults"; + var yamlFile = Path.Combine(projectDirectory.DirectoryPath, "tye.yaml"); + await File.WriteAllTextAsync(yamlFile, content); + + // Debug targets can be null if not specified, so make sure calling host.Start does not throw. + var outputContext = new OutputContext(_sink, Verbosity.Debug); + var application = await ApplicationFactory.CreateAsync(outputContext, new FileInfo(yamlFile)); + + var handler = new HttpClientHandler + { + ServerCertificateCustomValidationCallback = (a, b, c, d) => true, + AllowAutoRedirect = false + }; + + var client = new HttpClient(new RetryHandler(handler)); + + await RunHostingApplication(application, Array.Empty(), async (app, uri) => + { + var votingUri = await GetServiceUrl(client, uri, "vote"); + var workerUri = await GetServiceUrl(client, uri, "worker"); + + var votingResponse = await client.GetAsync(votingUri); + var workerResponse = await client.GetAsync(workerUri); + + Assert.True(votingResponse.IsSuccessStatusCode); + Assert.Equal(HttpStatusCode.NotFound, workerResponse.StatusCode); + }); + } + private async Task GetServiceUrl(HttpClient client, Uri uri, string serviceName) { var serviceResult = await client.GetStringAsync($"{uri}api/v1/services/{serviceName}"); diff --git a/test/Test.Infrastructure/TestHelpers.cs b/test/Test.Infrastructure/TestHelpers.cs index 3a8ddf72..4c6bd801 100644 --- a/test/Test.Infrastructure/TestHelpers.cs +++ b/test/Test.Infrastructure/TestHelpers.cs @@ -13,12 +13,13 @@ using Microsoft.Tye.Hosting; using Microsoft.Tye.Hosting.Model; using Xunit; using Microsoft.Tye; +using System.Diagnostics; namespace Test.Infrastructure { public static class TestHelpers { - private static readonly TimeSpan WaitForServicesTimeout = TimeSpan.FromSeconds(20); + private static readonly TimeSpan WaitForServicesTimeout = Debugger.IsAttached ? TimeSpan.FromMinutes(5) : TimeSpan.FromSeconds(20); // https://github.com/dotnet/aspnetcore/blob/5a0526dfd991419d5bce0d8ea525b50df2e37b04/src/Testing/src/TestPathUtilities.cs // This can get into a bad pattern for having crazy paths in places. Eventually, especially if we use helix,