Allows cloning dependencies (#424)

src/Microsoft.Tye.Core/ApplicationFactory.cs

@@ -36,7 +36,9 @@ namespace Microsoft.Tye
             {
                 var item = queue.Dequeue();
                 var config = item.Item1;
-                var parentDependencies = item.Item2;
+
+                // dependencies represents a set of all dependencies
+                var dependencies = item.Item2;
                 if (!visited.Add(config.Source.FullName))
                 {
                     continue;
@@ -73,8 +75,9 @@ namespace Microsoft.Tye
                     ServiceBuilder service;
                     if (root.Services.Any(s => s.Name == configService.Name))
                     {
-                        AddToRootServices(root, parentDependencies, configService, configService.Name);
+                        // Even though this service has already created a service, we still need
-                        // Don't add a service which has already been added by name
+                        // to update dependency information
+                        AddToRootServices(root, dependencies, configService.Name);
                         continue;
                     }
 
@@ -136,17 +139,50 @@ namespace Microsoft.Tye
                     else if (!string.IsNullOrEmpty(configService.Include))
                     {
                         var expandedYaml = Environment.ExpandEnvironmentVariables(configService.Include);
-                        var nestedConfig = ConfigFactory.FromFile(new FileInfo(Path.Combine(config.Source.DirectoryName, expandedYaml)));
-                        nestedConfig.Validate();
 
-                        if (nestedConfig.Name != rootConfig.Name)
+                        var nestedConfig = GetNestedConfig(rootConfig, Path.Combine(config.Source.DirectoryName, expandedYaml));
+                        queue.Enqueue((nestedConfig, new HashSet<string>()));
+
+                        AddToRootServices(root, dependencies, configService.Name);
+                        continue;
+                    }
+                    else if (!string.IsNullOrEmpty(configService.Repository))
+                    {
+                        // clone to .tye folder
+                        var path = Path.Join(rootConfig.Source.DirectoryName, ".tye", "deps");
+                        if (!Directory.Exists(path))
+                        {
+                            Directory.CreateDirectory(path);
+                        }
+
+                        var clonePath = Path.Combine(path, configService.Name);
+
+                        if (!Directory.Exists(clonePath))
+                        {
+                            if (!await GitDetector.Instance.IsGitInstalled.Value)
+                            {
+                                throw new CommandException($"Cannot clone repository {configService.Repository} because git is not installed. Please install git if you'd like to use \"repository\" in tye.yaml.");
+                            }
+
+                            var result = await ProcessUtil.RunAsync("git", $"clone {configService.Repository} {clonePath}", workingDirectory: path, throwOnError: false);
+
+                            if (result.ExitCode != 0)
+                            {
+                                throw new CommandException($"Failed to clone repository {configService.Repository} with exit code {result.ExitCode}.{Environment.NewLine}{result.StandardError}{result.StandardOutput}.");
+                            }
+                        }
+
+                        if (!ConfigFileFinder.TryFindSupportedFile(clonePath, out var file, out var errorMessage))
                         {
-                            throw new CommandException($"Nested configuration must have the same \"name\" in the tye.yaml. Root config: {rootConfig.Source}, nested config: {nestedConfig.Source}");
+                            throw new CommandException(errorMessage!);
                         }
 
+                        // pick different service type based on what is in the repo.
+                        var nestedConfig = GetNestedConfig(rootConfig, file);
+
                         queue.Enqueue((nestedConfig, new HashSet<string>()));
 
-                        AddToRootServices(root, parentDependencies, configService, configService.Name);
+                        AddToRootServices(root, dependencies, configService.Name);
 
                         continue;
                     }
@@ -160,10 +196,10 @@ namespace Microsoft.Tye
                         throw new CommandException("Unable to determine service type.");
                     }
 
-                    service.Dependencies.AddRange(parentDependencies);
+                    // Add dependencies to ourself before adding ourself to avoid self reference
-                    parentDependencies.Add(service.Name);
+                    service.Dependencies.UnionWith(dependencies);
 
-                    AddToRootServices(root, parentDependencies, configService, service.Name);
+                    AddToRootServices(root, dependencies, service.Name);
 
                     root.Services.Add(service);
 
@@ -285,12 +321,29 @@ namespace Microsoft.Tye
             return root;
         }
 
-        private static void AddToRootServices(ApplicationBuilder root, HashSet<string> parentDependencies, ConfigService configService, string serviceName)
+        private static ConfigApplication GetNestedConfig(ConfigApplication rootConfig, string? file)
+        {
+            var nestedConfig = ConfigFactory.FromFile(new FileInfo(file));
+            nestedConfig.Validate();
+
+            if (nestedConfig.Name != rootConfig.Name)
+            {
+                throw new CommandException($"Nested configuration must have the same \"name\" in the tye.yaml. Root config: {rootConfig.Source}, nested config: {nestedConfig.Source}");
+            }
+
+            return nestedConfig;
+        }
+
+        private static void AddToRootServices(ApplicationBuilder root, HashSet<string> dependencies, string serviceName)
         {
-            parentDependencies.Add(serviceName);
+            // Add ourselves in the set of all current dependencies.
+            dependencies.Add(serviceName);
+
+            // Iterate through all services and add the current services as a dependency (except ourselves)
             foreach (var s in root.Services)
             {
-                if (parentDependencies.Contains(s.Name, StringComparer.OrdinalIgnoreCase) && !s.Name.Equals(configService.Name, StringComparison.OrdinalIgnoreCase))
+                if (dependencies.Contains(s.Name, StringComparer.OrdinalIgnoreCase)
+                    && !s.Name.Equals(serviceName, StringComparison.OrdinalIgnoreCase))
                 {
                     s.Dependencies.Add(serviceName);
                 }

src/Microsoft.Tye.Core/ConfigFileFinder.cs

@@ -0,0 +1,39 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+// See the LICENSE file in the project root for more information.
+
+using System.IO;
+
+namespace Microsoft.Tye
+{
+    public class ConfigFileFinder
+    {
+        private static readonly string[] FileFormats = new[] { "tye.yaml", "tye.yml", "*.csproj", "*.fsproj", "*.sln" };
+
+        public static bool TryFindSupportedFile(string directoryPath, out string? filePath, out string? errorMessage)
+        {
+            foreach (var format in FileFormats)
+            {
+                var files = Directory.GetFiles(directoryPath, format);
+
+                if (files.Length == 1)
+                {
+                    errorMessage = null;
+                    filePath = files[0];
+                    return true;
+                }
+
+                if (files.Length > 1)
+                {
+                    errorMessage = $"More than one matching file was found in directory '{directoryPath}'.";
+                    filePath = default;
+                    return false;
+                }
+            }
+
+            errorMessage = $"No project project file or solution was found in directory '{directoryPath}'.";
+            filePath = default;
+            return false;
+        }
+    }
+}

src/Microsoft.Tye.Core/ConfigModel/ConfigService.cs

@@ -16,6 +16,7 @@ namespace Microsoft.Tye.ConfigModel
         public string? Image { get; set; }
         public string? Project { get; set; }
         public string? Include { get; set; }
+        public string? Repository { get; set; }
         public bool? Build { get; set; }
         public string? Executable { get; set; }
         public string? WorkingDirectory { get; set; }

src/Microsoft.Tye.Core/GitDetector.cs

@@ -0,0 +1,35 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+// See the LICENSE file in the project root for more information.
+
+using System;
+using System.Threading.Tasks;
+
+namespace Microsoft.Tye
+{
+    public class GitDetector
+    {
+        public static GitDetector Instance { get; } = new GitDetector();
+
+        private GitDetector()
+        {
+            IsGitInstalled = new Lazy<Task<bool>>(GetIsGitInstalled);
+        }
+
+        public Lazy<Task<bool>> IsGitInstalled { get; }
+
+        private async Task<bool> GetIsGitInstalled()
+        {
+            try
+            {
+                var result = await ProcessUtil.RunAsync("git", "--version", throwOnError: false);
+                return result.ExitCode == 0;
+            }
+            catch (Exception)
+            {
+                // Unfortunately, process throws
+                return false;
+            }
+        }
+    }
+}

src/Microsoft.Tye.Core/Microsoft.Tye.Core.csproj

@@ -37,6 +37,7 @@
   <ItemGroup>
     <Compile Include="..\shared\KubectlDetector.cs" Link="KubectlDetector.cs" />
     <Compile Include="..\shared\TempFile.cs" Link="TempFile.cs" />
+    <Compile Include="..\shared\DirectoryExtensions.cs" Link="DirectoryExtensions.cs" />
   </ItemGroup>
 
 </Project>

src/Microsoft.Tye.Core/Serialization/ConfigServiceParser.cs

@@ -55,6 +55,9 @@ namespace Tye.Serialization
                     case "include":
                         service.Include = YamlParser.GetScalarValue(key, child.Value);
                         break;
+                    case "repository":
+                        service.Repository = YamlParser.GetScalarValue(key, child.Value);
+                        break;
                     case "build":
                         if (!bool.TryParse(YamlParser.GetScalarValue(key, child.Value), out var build))
                         {

src/Microsoft.Tye.Core/ServiceBuilder.cs

@@ -16,6 +16,6 @@ namespace Microsoft.Tye
         // TODO: this is temporary while refactoring
         public List<ServiceOutput> Outputs { get; } = new List<ServiceOutput>();
 
-        public List<string> Dependencies { get; } = new List<string>();
+        public HashSet<string> Dependencies { get; } = new HashSet<string>();
     }
 }

src/Microsoft.Tye.Core/TempDirectory.cs

@@ -42,7 +42,7 @@ namespace Microsoft.Tye
 
         public void Dispose()
         {
-            Directory.Delete(DirectoryPath, recursive: true);
+            DirectoryExtensions.DeleteDirectory(DirectoryPath);
         }
     }
 }

src/Microsoft.Tye.Hosting/Microsoft.Tye.Hosting.csproj

@@ -32,4 +32,8 @@
     <ProjectReference Include="..\Microsoft.Tye.Proxy\Microsoft.Tye.Proxy.csproj" />
   </ItemGroup>
 
+  <ItemGroup>
+    <Compile Include="..\shared\DirectoryExtensions.cs" Link="DirectoryExtensions.cs" />
+  </ItemGroup>
+
 </Project>

src/Microsoft.Tye.Hosting/ReplicaRegistry.cs

@@ -101,7 +101,14 @@ namespace Microsoft.Tye.Hosting
         {
             if (Directory.Exists(_tyeFolderPath))
             {
-                Directory.Delete(_tyeFolderPath, true);
+                try
+                {
+                    DirectoryExtensions.DeleteDirectory(_tyeFolderPath);
+                }
+                catch (Exception ex)
+                {
+                    Console.WriteLine(ex.Message);
+                }
             }
         }
     }

src/shared/DirectoryExtensions.cs

@@ -0,0 +1,28 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+// See the LICENSE file in the project root for more information.
+
+using System.IO;
+
+namespace Microsoft.Tye
+{
+    internal static class DirectoryExtensions
+    {
+        // Calling Directory.Delete causes an exception for .git folders:
+        //     System.UnauthorizedAccessException : Access to the path '17a475ecca365c678e907bd4c73e4c65b341c6' is denied.
+        public static void DeleteDirectory(string d)
+        {
+            foreach (var sub in Directory.EnumerateDirectories(d))
+            {
+                DeleteDirectory(sub);
+            }
+            foreach (var f in Directory.EnumerateFiles(d))
+            {
+                var fi = new FileInfo(f);
+                fi.Attributes = FileAttributes.Normal;
+                fi.Delete();
+            }
+            Directory.Delete(d);
+        }
+    }
+}

src/tye/CommonArguments.cs

@@ -11,8 +11,6 @@ namespace Microsoft.Tye
 {
     internal static class CommonArguments
     {
-        private static readonly string[] FileFormats = new[] { "tye.yaml", "tye.yml", "*.csproj", "*.fsproj", "*.sln" };
-
         public static Argument<FileInfo> Path_Optional
         {
             get
@@ -39,32 +37,6 @@ namespace Microsoft.Tye
             }
         }
 
-        public static bool TryFindSupportedFile(string directoryPath, out string? filePath, out string? errorMessage)
-        {
-            foreach (var format in FileFormats)
-            {
-                var files = Directory.GetFiles(directoryPath, format);
-
-                if (files.Length == 1)
-                {
-                    errorMessage = null;
-                    filePath = files[0];
-                    return true;
-                }
-
-                if (files.Length > 1)
-                {
-                    errorMessage = $"More than one matching file was found in directory '{directoryPath}'.";
-                    filePath = default;
-                    return false;
-                }
-            }
-
-            errorMessage = $"No project project file or solution was found in directory '{directoryPath}'.";
-            filePath = default;
-            return false;
-        }
-
         static FileInfo TryParsePath(ArgumentResult result, bool required)
         {
             var token = result.Tokens.Count switch
@@ -86,7 +58,7 @@ namespace Microsoft.Tye
 
             if (Directory.Exists(token))
             {
-                if (TryFindSupportedFile(token, out var filePath, out var errorMessage))
+                if (ConfigFileFinder.TryFindSupportedFile(token, out var filePath, out var errorMessage))
                 {
                     return new FileInfo(filePath);
                 }

test/E2ETest/ApplicationFactoryTests.cs

@@ -55,6 +55,31 @@ services:
             var application = await ApplicationFactory.CreateAsync(outputContext, new FileInfo(yamlFile));
 
             Assert.Equal(5, application.Services.Count);
+
+            var vote = application.Services.Single(s => s.Name == "vote");
+            Assert.Equal(2, vote.Dependencies.Count);
+            Assert.Contains("worker", vote.Dependencies);
+            Assert.Contains("redis", vote.Dependencies);
+
+            var results = application.Services.Single(s => s.Name == "results");
+            Assert.Single(results.Dependencies);
+            Assert.Contains("worker", results.Dependencies);
+
+            var worker = application.Services.Single(s => s.Name == "worker");
+            Assert.Equal(2, worker.Dependencies.Count);
+            Assert.Contains("redis", worker.Dependencies);
+            Assert.Contains("postgres", worker.Dependencies);
+
+            var redis = application.Services.Single(s => s.Name == "redis");
+            Assert.Equal(3, redis.Dependencies.Count);
+            Assert.Contains("postgres", redis.Dependencies);
+            Assert.Contains("worker", redis.Dependencies);
+            Assert.Contains("vote", redis.Dependencies);
+
+            var postgres = application.Services.Single(s => s.Name == "postgres");
+            Assert.Equal(2, postgres.Dependencies.Count);
+            Assert.Contains("worker", postgres.Dependencies);
+            Assert.Contains("redis", postgres.Dependencies);
         }
 
         [Fact]

test/E2ETest/TyeRunTests.cs

@@ -210,7 +210,7 @@ namespace E2ETest
 
             var outputFileName = project.AssemblyName + ".dll";
             var container = new ContainerServiceBuilder(project.Name, $"mcr.microsoft.com/dotnet/core/sdk:{project.TargetFrameworkVersion}");
-            container.Dependencies.AddRange(project.Dependencies);
+            container.Dependencies.UnionWith(project.Dependencies);
             container.Volumes.Add(new VolumeBuilder(project.PublishDir, name: null, target: "/app"));
             container.Args = $"dotnet /app/{outputFileName} {project.Args}";
             container.Bindings.AddRange(project.Bindings);
@@ -579,6 +579,47 @@ namespace E2ETest
             });
         }
 
+        [ConditionalFact]
+        [SkipIfDockerNotRunning]
+        public async Task MultiRepo_WorksWithCloning()
+        {
+            using var projectDirectory = TempDirectory.Create(preferUserDirectoryOnMacOS: true);
+
+            var content = @"
+name: VotingSample
+services:
+- name: vote
+  repository: https://github.com/jkotalik/TyeMultiRepoVoting
+- name: results
+  repository: https://github.com/jkotalik/TyeMultiRepoResults";
+            var yamlFile = Path.Combine(projectDirectory.DirectoryPath, "tye.yaml");
+            await File.WriteAllTextAsync(yamlFile, content);
+
+            // Debug targets can be null if not specified, so make sure calling host.Start does not throw.
+            var outputContext = new OutputContext(_sink, Verbosity.Debug);
+            var application = await ApplicationFactory.CreateAsync(outputContext, new FileInfo(yamlFile));
+
+            var handler = new HttpClientHandler
+            {
+                ServerCertificateCustomValidationCallback = (a, b, c, d) => true,
+                AllowAutoRedirect = false
+            };
+
+            var client = new HttpClient(new RetryHandler(handler));
+
+            await RunHostingApplication(application, Array.Empty<string>(), async (app, uri) =>
+            {
+                var votingUri = await GetServiceUrl(client, uri, "vote");
+                var workerUri = await GetServiceUrl(client, uri, "worker");
+
+                var votingResponse = await client.GetAsync(votingUri);
+                var workerResponse = await client.GetAsync(workerUri);
+
+                Assert.True(votingResponse.IsSuccessStatusCode);
+                Assert.Equal(HttpStatusCode.NotFound, workerResponse.StatusCode);
+            });
+        }
+
         private async Task<string> GetServiceUrl(HttpClient client, Uri uri, string serviceName)
         {
             var serviceResult = await client.GetStringAsync($"{uri}api/v1/services/{serviceName}");

test/Test.Infrastructure/TestHelpers.cs

@@ -13,12 +13,13 @@ using Microsoft.Tye.Hosting;
 using Microsoft.Tye.Hosting.Model;
 using Xunit;
 using Microsoft.Tye;
+using System.Diagnostics;
 
 namespace Test.Infrastructure
 {
     public static class TestHelpers
     {
-        private static readonly TimeSpan WaitForServicesTimeout = TimeSpan.FromSeconds(20);
+        private static readonly TimeSpan WaitForServicesTimeout = Debugger.IsAttached ? TimeSpan.FromMinutes(5) : TimeSpan.FromSeconds(20);
 
         // https://github.com/dotnet/aspnetcore/blob/5a0526dfd991419d5bce0d8ea525b50df2e37b04/src/Testing/src/TestPathUtilities.cs
         // This can get into a bad pattern for having crazy paths in places. Eventually, especially if we use helix,