Browse Source

enhanced project build approach

pull/5418/head
İsmail ÇAĞDAŞ 6 years ago
parent
commit
943e52e1b5
  1. 299
      framework/src/Volo.Abp.Cli.Core/Volo/Abp/Cli/Build/DefaultChangedProjectFinder.cs
  2. 34
      framework/src/Volo.Abp.Cli.Core/Volo/Abp/Cli/Build/DefaultDotNetProjectBuilder.cs
  3. 2
      framework/src/Volo.Abp.Cli.Core/Volo/Abp/Cli/Build/DotNetProjectDependencyFiller.cs
  4. 5
      framework/src/Volo.Abp.Cli.Core/Volo/Abp/Cli/Build/DotNetProjectInfo.cs
  5. 47
      framework/src/Volo.Abp.Cli.Core/Volo/Abp/Cli/Build/DotNetProjectInfoExtensions.cs
  6. 12
      framework/src/Volo.Abp.Cli.Core/Volo/Abp/Cli/Commands/BuildCommand.cs
  7. 29
      framework/test/Volo.Abp.Cli.Core.Tests/Volo/Abp/Cli/Build/BuildProjectListSorter_Tests.cs
  8. 18
      framework/test/Volo.Abp.Cli.Core.Tests/Volo/Abp/Cli/Build/BuildStatusGenerator_Tests.cs

299
framework/src/Volo.Abp.Cli.Core/Volo/Abp/Cli/Build/DefaultChangedProjectFinder.cs

@ -14,6 +14,7 @@ namespace Volo.Abp.Cli.Build
private readonly IGitRepositoryHelper _gitRepositoryHelper;
private readonly IDotNetProjectDependencyFiller _dotNetProjectDependencyFiller;
private readonly IBuildProjectListSorter _buildProjectListSorter;
private readonly List<string> _changeDetectionFileExtensions = new List<string>
{
@ -22,14 +23,18 @@ namespace Volo.Abp.Cli.Build
".cshtml"
};
// TODO: consider ignored folders !
public DefaultChangedProjectFinder(
IRepositoryBuildStatusStore repositoryBuildStatusStore,
IGitRepositoryHelper gitRepositoryHelper,
IDotNetProjectDependencyFiller dotNetProjectDependencyFiller)
IDotNetProjectDependencyFiller dotNetProjectDependencyFiller,
IBuildProjectListSorter buildProjectListSorter)
{
_repositoryBuildStatusStore = repositoryBuildStatusStore;
_gitRepositoryHelper = gitRepositoryHelper;
_dotNetProjectDependencyFiller = dotNetProjectDependencyFiller;
_buildProjectListSorter = buildProjectListSorter;
}
public List<DotNetProjectInfo> Find(DotNetProjectBuildConfig buildConfig)
@ -61,7 +66,7 @@ namespace Volo.Abp.Cli.Build
).ToList();
projects.AddRange(
allCsProjFiles.Select(csProjPath => new DotNetProjectInfo(gitRepository.Name, csProjPath))
allCsProjFiles.Select(csProjPath => new DotNetProjectInfo(gitRepository.Name, csProjPath, false))
);
foreach (var dependingRepository in gitRepository.DependingRepositories)
@ -70,10 +75,8 @@ namespace Volo.Abp.Cli.Build
}
}
private List<DotNetProjectInfo> FindByRepository(DotNetProjectBuildConfig buildConfig)
{
var changedProjectList = new List<DotNetProjectInfo>();
var gitRepositoryBuildStatus = _repositoryBuildStatusStore.Get(
buildConfig.BuildName,
buildConfig.GitRepository
@ -81,104 +84,37 @@ namespace Volo.Abp.Cli.Build
// Create a List which contains all csproj files and their 1-level dependencies
var allProjectList = FindAllProjects(buildConfig.GitRepository);
var allSortedProjectList = _buildProjectListSorter.SortByDependencies(
allProjectList,
new DotNetProjectInfoEqualityComparer()
);
FindChangedFiles(
MarkProjectsForBuild(
buildConfig.GitRepository,
gitRepositoryBuildStatus,
changedProjectList,
buildConfig.ForceBuild,
allProjectList
allSortedProjectList
);
return changedProjectList;
}
private List<DotNetProjectInfo> FindBySlnFile(GitRepository gitRepository, string slnFilePath)
{
var slFine = new FileInfo(slnFilePath);
var changedProjectList = new List<DotNetProjectInfo>();
var csProjFiles = slFine.Directory.GetFiles(
"*.csproj",
SearchOption.AllDirectories
).Select(e => e.FullName)
.ToList();
foreach (var csProjFile in csProjFiles)
{
AddDependantProjectsToList(gitRepository, csProjFile, changedProjectList);
}
_dotNetProjectDependencyFiller.Fill(changedProjectList);
return changedProjectList;
}
private void AddDependantProjectsToList(
GitRepository gitRepository,
string csProjFilePath,
List<DotNetProjectInfo> changedProjectList)
{
var repositoryName = gitRepository.FindRepositoryOf(csProjFilePath);
var project = new DotNetProjectInfo(repositoryName, csProjFilePath);
if (changedProjectList.Any(e => e.CsProjPath == csProjFilePath))
{
return;
}
changedProjectList.Add(project);
AddProjectDependencies(gitRepository, project, changedProjectList);
}
private void AddProjectDependencies(
GitRepository gitRepository,
DotNetProjectInfo project,
List<DotNetProjectInfo> changedProjectList)
{
var projectNode = XElement.Load(project.CsProjPath);
var referenceNodes = projectNode.Descendants("ItemGroup").Descendants("ProjectReference");
foreach (var referenceNode in referenceNodes)
{
if (referenceNode.Attribute("Include") == null)
{
continue;
}
var relativePath = referenceNode.Attribute("Include").Value;
var file = new FileInfo(project.CsProjPath);
var referenceProjectInfo = new FileInfo(Path.Combine(file.Directory.FullName, relativePath));
if (changedProjectList.Any(e => e.CsProjPath == referenceProjectInfo.FullName))
{
continue;
}
var repositoryName = gitRepository.FindRepositoryOf(referenceProjectInfo.FullName);
var referenceProject = new DotNetProjectInfo(repositoryName, referenceProjectInfo.FullName);
changedProjectList.Add(referenceProject);
AddProjectDependencies(gitRepository, referenceProject, changedProjectList);
}
return allSortedProjectList.Where(e => e.ShouldBuild).ToList();
}
private void FindChangedFiles(
private void MarkProjectsForBuild(
GitRepository repository,
GitRepositoryBuildStatus repositoryBuildStatus,
List<DotNetProjectInfo> changedProjectList,
bool forceBuild,
List<DotNetProjectInfo> allProjectList)
{
if (forceBuild || repositoryBuildStatus == null || repositoryBuildStatus.CommitId.IsNullOrEmpty())
{
AddAllCsProjFiles(repository, changedProjectList, repositoryBuildStatus);
// Mark all projects for build
allProjectList.ForEach(e => e.ShouldBuild = true);
}
else
{
AddChangedCsProjFiles(
MarkChangedProjectsForBuild(
repository,
changedProjectList,
repositoryBuildStatus,
false,
allProjectList
);
}
@ -188,10 +124,9 @@ namespace Volo.Abp.Cli.Build
foreach (var dependingRepository in repository.DependingRepositories)
{
var dependingRepositoryBuildStatus = repositoryBuildStatus?.GetChild(dependingRepository.Name);
FindChangedFiles(
MarkProjectsForBuild(
dependingRepository,
dependingRepositoryBuildStatus,
changedProjectList,
forceBuild,
allProjectList
);
@ -199,56 +134,9 @@ namespace Volo.Abp.Cli.Build
}
}
private void AddAllCsProjFiles(
private void MarkChangedProjectsForBuild(
GitRepository repository,
List<DotNetProjectInfo> changedFiles,
GitRepositoryBuildStatus status)
{
var allCsProjFiles = Directory.GetFiles(
repository.RootPath,
"*.csproj",
SearchOption.AllDirectories
).ToList();
// Filter ignored directories
foreach (var ignoredDirectory in repository.IgnoredDirectories)
{
allCsProjFiles = allCsProjFiles.Where(e =>
!e.StartsWith(Path.Combine(repository.RootPath, ignoredDirectory)))
.ToList();
}
// Filter already built files.
var lastCommitId = _gitRepositoryHelper.GetLastCommitId(repository);
foreach (var file in allCsProjFiles)
{
if (status == null)
{
changedFiles.Add(
new DotNetProjectInfo(repository.Name, Path.Combine(repository.RootPath, file))
);
continue;
}
if (status.GetSelfOrChild(repository.Name).SucceedProjects
.Any(e => e.CsProjPath == file && e.CommitId == lastCommitId))
{
continue;
}
changedFiles.Add(
new DotNetProjectInfo(repository.Name, Path.Combine(repository.RootPath, file))
);
}
}
private void AddChangedCsProjFiles(
GitRepository repository,
List<DotNetProjectInfo> changedProjectList,
GitRepositoryBuildStatus status,
bool forceBuild,
List<DotNetProjectInfo> allProjectList)
{
using (var repo = new Repository(string.Concat(repository.RootPath, @"\.git")))
@ -256,6 +144,7 @@ namespace Volo.Abp.Cli.Build
var firstCommit = status.CommitId.IsNullOrEmpty()
? null
: repo.Lookup<Commit>(status.CommitId);
var repoDifferences = repo.Diff.Compare<Patch>(firstCommit?.Tree, repo.Head.Tip.Tree);
var fileExtensionPredicate = PredicateBuilder.New<PatchEntryChanges>(true);
@ -292,49 +181,147 @@ namespace Volo.Abp.Cli.Build
continue;
}
var project = new DotNetProjectInfo(repository.Name, csProjPath);
changedProjectList.Add(project);
AddDependingProjectsToList(project, changedProjectList, allProjectList);
allProjectList.MarkForBuild(repository.Name, csProjPath);
AddDependingProjectsToList(repository.Name, csProjPath, allProjectList);
}
}
}
if (!repository.DependingRepositories.Any())
{
return;
}
private List<DotNetProjectInfo> FindBySlnFile(GitRepository gitRepository, string slnFilePath)
{
// Create a List which contains all csproj files and their 1-level dependencies
var allProjectList = FindAllProjects(gitRepository);
var slFine = new FileInfo(slnFilePath);
var csProjFiles = slFine.Directory.GetFiles(
"*.csproj",
SearchOption.AllDirectories
).Select(e => e.FullName)
.ToList();
foreach (var subRepository in repository.DependingRepositories)
foreach (var csProjFile in csProjFiles)
{
MarkDependantProjectsForBuild(gitRepository, csProjFile, allProjectList);
}
return _buildProjectListSorter.SortByDependencies(
allProjectList,
new DotNetProjectInfoEqualityComparer()
).Where(e => e.ShouldBuild).ToList();
}
private void MarkDependantProjectsForBuild(
GitRepository gitRepository,
string csProjFilePath,
List<DotNetProjectInfo> allProjectList)
{
var repositoryName = gitRepository.FindRepositoryOf(csProjFilePath);
var project = new DotNetProjectInfo(repositoryName, csProjFilePath, true);
if (allProjectList.IsMarkedForBuild(repositoryName, csProjFilePath))
{
return;
}
allProjectList.MarkForBuild(project);
AddProjectDependencies(gitRepository, project, allProjectList);
}
private void AddProjectDependencies(
GitRepository gitRepository,
DotNetProjectInfo project,
List<DotNetProjectInfo> allProjectList)
{
var projectInfo = allProjectList.FirstOrDefault(e => e.CsProjPath == project.CsProjPath);
if (projectInfo == null)
{
return;
}
var dependencies = projectInfo.Dependencies;
foreach (var dependency in dependencies)
{
if (allProjectList.IsMarkedForBuild(dependency.RepositoryName, dependency.CsProjPath))
{
var subRepositoryBuildStatus = status.GetChild(subRepository.Name);
FindChangedFiles(
subRepository,
subRepositoryBuildStatus,
changedProjectList,
forceBuild,
allProjectList
);
continue;
}
allProjectList.MarkForBuild(dependency.RepositoryName, dependency.CsProjPath);
AddProjectDependencies(gitRepository, dependency, allProjectList);
}
}
private void AddDependingProjectsToList(DotNetProjectInfo project, List<DotNetProjectInfo> changedProjectList,
// private void AddAllCsProjFiles(
// GitRepository repository,
// List<DotNetProjectInfo> changedFiles,
// GitRepositoryBuildStatus status)
// {
// var allCsProjFiles = Directory.GetFiles(
// repository.RootPath,
// "*.csproj",
// SearchOption.AllDirectories
// ).ToList();
//
// // Filter ignored directories
// foreach (var ignoredDirectory in repository.IgnoredDirectories)
// {
// allCsProjFiles = allCsProjFiles.Where(e =>
// !e.StartsWith(Path.Combine(repository.RootPath, ignoredDirectory)))
// .ToList();
// }
//
// // Filter already built files.
// var lastCommitId = _gitRepositoryHelper.GetLastCommitId(repository);
//
// foreach (var file in allCsProjFiles)
// {
// if (status == null)
// {
// changedFiles.Add(
// new DotNetProjectInfo(repository.Name, Path.Combine(repository.RootPath, file))
// );
//
// continue;
// }
//
// if (status.GetSelfOrChild(repository.Name).SucceedProjects
// .Any(e => e.CsProjPath == file && e.CommitId == lastCommitId))
// {
// continue;
// }
//
// changedFiles.Add(
// new DotNetProjectInfo(repository.Name, Path.Combine(repository.RootPath, file))
// );
// }
// }
private void AddDependingProjectsToList(
string repositoryName,
string csProjPath,
List<DotNetProjectInfo> allProjectList)
{
var dependingProjects = allProjectList.Where(
e => e.Dependencies.Any(d => d.CsProjPath == project.CsProjPath)
).Select(e => new DotNetProjectInfo(e.RepositoryName, e.CsProjPath)) // don't get dependencies
.ToList();
e => e.Dependencies.Any(d => d.RepositoryName == repositoryName && d.CsProjPath == csProjPath)
).Select(e => new DotNetProjectInfo(e.RepositoryName, e.CsProjPath, true)).ToList();
if (!dependingProjects.Any())
{
return;
}
changedProjectList.AddRange(dependingProjects);
foreach (var dependingProject in dependingProjects)
{
AddDependingProjectsToList(dependingProject, changedProjectList, allProjectList);
if (allProjectList.IsMarkedForBuild(dependingProject))
{
continue;
}
allProjectList.MarkForBuild(dependingProject);
AddDependingProjectsToList(dependingProject.RepositoryName, dependingProject.CsProjPath,
allProjectList);
}
}

34
framework/src/Volo.Abp.Cli.Core/Volo/Abp/Cli/Build/DefaultDotNetProjectBuilder.cs

@ -1,9 +1,7 @@
using System;
using System.Collections.Concurrent;
using System.Collections.Generic;
using System.Drawing;
using System.Linq;
using System.Threading.Tasks;
using Volo.Abp.Cli.Utils;
using Volo.Abp.DependencyInjection;
@ -14,7 +12,7 @@ namespace Volo.Abp.Cli.Build
public List<string> Build(List<DotNetProjectInfo> projects, int maxParallelBuildCount, string arguments)
{
var builtProjects = new ConcurrentBag<string>();
var totalProjectCountToBuild = GetTotalProjectCountToBuild(projects);
var totalProjectCountToBuild = projects.Count;
var buildingProjectIndex = 0;
try
@ -34,29 +32,6 @@ namespace Volo.Abp.Cli.Build
);
BuildInternal(project, arguments, builtProjects);
if (!project.Dependencies.Any())
{
continue;
}
Parallel.ForEach(
project.Dependencies,
new ParallelOptions {MaxDegreeOfParallelism = maxParallelBuildCount},
(projectDependency) =>
{
if (!builtProjects.Contains(project.CsProjPath))
{
buildingProjectIndex++;
Console.WriteLine(
"Building....: " + " (" + buildingProjectIndex + "/" +
totalProjectCountToBuild + ")" + project.CsProjPath
);
BuildInternal(projectDependency, arguments, builtProjects);
}
});
}
}
catch (Exception e)
@ -67,13 +42,6 @@ namespace Volo.Abp.Cli.Build
return builtProjects.ToList();
}
private int GetTotalProjectCountToBuild(List<DotNetProjectInfo> projects)
{
var mainCsProjPaths = projects.Select(e => e.CsProjPath).ToList();
var dependingCsProjPaths = projects.SelectMany(e => e.Dependencies).Select(e => e.CsProjPath).ToList();
return mainCsProjPaths.Union(dependingCsProjPaths).Distinct().Count();
}
private void BuildInternal(DotNetProjectInfo project, string arguments, ConcurrentBag<string> builtProjects)
{
var buildArguments = arguments.TrimStart('"').TrimEnd('"');

2
framework/src/Volo.Abp.Cli.Core/Volo/Abp/Cli/Build/DotNetProjectDependencyFiller.cs

@ -33,7 +33,7 @@ namespace Volo.Abp.Cli.Build
var file = new FileInfo(project.CsProjPath);
var referenceProjectInfo = new FileInfo(Path.Combine(file.Directory.FullName, relativePath));
var referenceProject = new DotNetProjectInfo(project.RepositoryName, referenceProjectInfo.FullName);
var referenceProject = new DotNetProjectInfo(project.RepositoryName, referenceProjectInfo.FullName, false);
project.Dependencies.Add(referenceProject);
}
}

5
framework/src/Volo.Abp.Cli.Core/Volo/Abp/Cli/Build/DotNetProjectInfo.cs

@ -9,13 +9,16 @@ namespace Volo.Abp.Cli.Build
public string RepositoryName { get; set; }
public string CsProjPath { get; set; }
public bool ShouldBuild { get; set; }
public List<DotNetProjectInfo> Dependencies { get; set; }
public DotNetProjectInfo(string repositoryName, string csProjPath)
public DotNetProjectInfo(string repositoryName, string csProjPath, bool shouldBuild)
{
RepositoryName = repositoryName;
CsProjPath = csProjPath;
ShouldBuild = shouldBuild;
Dependencies = new List<DotNetProjectInfo>();
}
}

47
framework/src/Volo.Abp.Cli.Core/Volo/Abp/Cli/Build/DotNetProjectInfoExtensions.cs

@ -0,0 +1,47 @@
using System.Collections.Generic;
using System.Linq;
namespace Volo.Abp.Cli.Build
{
public static class DotNetProjectInfoExtensions
{
public static void MarkForBuild(this List<DotNetProjectInfo> projects, string repositoryName, string csProjPath)
{
var project = projects.FirstOrDefault(e =>
e.RepositoryName == repositoryName && e.CsProjPath == csProjPath
);
if (project == null)
{
return;
}
project.ShouldBuild = true;
}
public static void MarkForBuild(this List<DotNetProjectInfo> projects, DotNetProjectInfo project)
{
projects.MarkForBuild(project.RepositoryName, project.CsProjPath);
}
public static bool IsMarkedForBuild(this List<DotNetProjectInfo> projects, string repositoryName,
string csProjPath)
{
var project = projects.FirstOrDefault(e =>
e.RepositoryName == repositoryName && e.CsProjPath == csProjPath
);
if (project == null)
{
return false;
}
return project.ShouldBuild;
}
public static bool IsMarkedForBuild(this List<DotNetProjectInfo> projects, DotNetProjectInfo project)
{
return projects.IsMarkedForBuild(project.RepositoryName, project.CsProjPath);
}
}
}

12
framework/src/Volo.Abp.Cli.Core/Volo/Abp/Cli/Commands/BuildCommand.cs

@ -56,15 +56,15 @@ namespace Volo.Abp.Cli.Commands
buildConfig.BuildName = buildName;
buildConfig.ForceBuild = forceBuild;
var changedProjectFiles = ChangedProjectFinder.Find(buildConfig);
Console.WriteLine("Finding changed projects...");
var sortedProjects = BuildProjectListSorter.SortByDependencies(
changedProjectFiles,
new DotNetProjectInfoEqualityComparer()
);
var changedProjectFiles = ChangedProjectFinder.Find(buildConfig);
Console.WriteLine(sw.ElapsedMilliseconds + " ms passed...");
sw.Restart();
var buildSucceededProjects = DotNetProjectBuilder.Build(
sortedProjects,
changedProjectFiles,
string.IsNullOrEmpty(maxParallelBuild) ? 1 : Convert.ToInt32(maxParallelBuild),
dotnetBuildArguments ?? ""
);

29
framework/test/Volo.Abp.Cli.Core.Tests/Volo/Abp/Cli/Build/BuildProjectListSorter_Tests.cs

@ -22,50 +22,51 @@ namespace Volo.Abp.Cli.Build
// F -> C
// C -> G
// Final build order must be: G,
var repositoryName = "volo";
var source = new List<DotNetProjectInfo>
{
new DotNetProjectInfo(repositoryName, "A")
new DotNetProjectInfo(repositoryName, "A", true)
{
Dependencies = new List<DotNetProjectInfo>()
{
new DotNetProjectInfo(repositoryName, "B"),
new DotNetProjectInfo(repositoryName, "C")
new DotNetProjectInfo(repositoryName, "B", true),
new DotNetProjectInfo(repositoryName, "C", true)
}
},
new DotNetProjectInfo(repositoryName, "B")
new DotNetProjectInfo(repositoryName, "B", true)
{
Dependencies = new List<DotNetProjectInfo>()
{
new DotNetProjectInfo(repositoryName, "D")
new DotNetProjectInfo(repositoryName, "D", true)
}
},
new DotNetProjectInfo(repositoryName, "D")
new DotNetProjectInfo(repositoryName, "D", true)
{
Dependencies = new List<DotNetProjectInfo>()
{
new DotNetProjectInfo(repositoryName, "F")
new DotNetProjectInfo(repositoryName, "F", true)
}
},
new DotNetProjectInfo(repositoryName, "F")
new DotNetProjectInfo(repositoryName, "F", true)
{
Dependencies = new List<DotNetProjectInfo>()
{
new DotNetProjectInfo(repositoryName, "C")
new DotNetProjectInfo(repositoryName, "C", true)
}
},
new DotNetProjectInfo(repositoryName, "C")
new DotNetProjectInfo(repositoryName, "C", true)
{
Dependencies = new List<DotNetProjectInfo>()
{
new DotNetProjectInfo(repositoryName, "G")
new DotNetProjectInfo(repositoryName, "G", true)
}
},
new DotNetProjectInfo(repositoryName, "G")
new DotNetProjectInfo(repositoryName, "G", true)
};
var sortedDependencies = _buildProjectListSorter.SortByDependencies(source, new DotNetProjectInfoEqualityComparer());
var sortedDependencies =
_buildProjectListSorter.SortByDependencies(source, new DotNetProjectInfoEqualityComparer());
sortedDependencies.Count.ShouldBe(6);
sortedDependencies[0].CsProjPath.ShouldBe("G");
sortedDependencies[1].CsProjPath.ShouldBe("C");

18
framework/test/Volo.Abp.Cli.Core.Tests/Volo/Abp/Cli/Build/BuildStatusGenerator_Tests.cs

@ -38,9 +38,9 @@ namespace Volo.Abp.Cli.Build
var changedProjects = new List<DotNetProjectInfo>()
{
new DotNetProjectInfo("volo", "project1.csproj")
new DotNetProjectInfo("volo", "project1.csproj", true)
};
var builtProjects = new List<string>()
{
"project1.csproj"
@ -53,8 +53,8 @@ namespace Volo.Abp.Cli.Build
var status = _buildStatusGenerator.Generate(buildConfig, changedProjects, builtProjects);
status.CommitId.ShouldBe(lastCommitId);
}
[Fact]
public void Should_Set_Repository_CommitId_When_All_Projects_Built_For_Child_Repository()
{
@ -71,9 +71,9 @@ namespace Volo.Abp.Cli.Build
var changedProjects = new List<DotNetProjectInfo>()
{
new DotNetProjectInfo("abp", "project1.csproj")
new DotNetProjectInfo("abp", "project1.csproj", true)
};
var builtProjects = new List<string>()
{
"project1.csproj"
@ -86,7 +86,7 @@ namespace Volo.Abp.Cli.Build
var status = _buildStatusGenerator.Generate(buildConfig, changedProjects, builtProjects);
status.CommitId.ShouldBe(lastCommitId);
}
[Fact]
public void Should_Not_Set_Repository_CommitId_When_Building_Single_Solution()
{
@ -104,9 +104,9 @@ namespace Volo.Abp.Cli.Build
var changedProjects = new List<DotNetProjectInfo>()
{
new DotNetProjectInfo("volo", "project1.csproj")
new DotNetProjectInfo("volo", "project1.csproj", true)
};
var builtProjects = new List<string>()
{
"project1.csproj"

Loading…
Cancel
Save