diff --git a/modules/docs/src/Volo.Docs.Application/Volo/Docs/Documents/DocumentAppService.cs b/modules/docs/src/Volo.Docs.Application/Volo/Docs/Documents/DocumentAppService.cs index 02a7c171bd..430cd9ead2 100644 --- a/modules/docs/src/Volo.Docs.Application/Volo/Docs/Documents/DocumentAppService.cs +++ b/modules/docs/src/Volo.Docs.Application/Volo/Docs/Documents/DocumentAppService.cs @@ -7,8 +7,6 @@ using Microsoft.Extensions.Configuration; using Microsoft.Extensions.Hosting; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; -using Nest; -using Newtonsoft.Json; using Volo.Abp; using Volo.Abp.Caching; using Volo.Docs.Caching; diff --git a/modules/docs/src/Volo.Docs.Domain/Volo/Docs/GitHub/Documents/GithubDocumentSource.cs b/modules/docs/src/Volo.Docs.Domain/Volo/Docs/GitHub/Documents/GithubDocumentSource.cs index a6285389de..ef6f6e1ebb 100644 --- a/modules/docs/src/Volo.Docs.Domain/Volo/Docs/GitHub/Documents/GithubDocumentSource.cs +++ b/modules/docs/src/Volo.Docs.Domain/Volo/Docs/GitHub/Documents/GithubDocumentSource.cs @@ -48,68 +48,155 @@ namespace Volo.Docs.GitHub.Documents fileName = documentName.Substring(documentName.LastIndexOf('/') + 1); } - var fileCommits = await GetFileCommitsAsync(project, version, project.GetGitHubInnerUrl(languageCode, documentName)); - - var documentCreationTime = fileCommits.LastOrDefault()?.Commit.Author.Date.DateTime ?? DateTime.MinValue; - - var lastSignificantUpdateTime = !isNavigationDocument && !isParameterDocument && version == project.LatestVersionBranchName ? - await GetLastSignificantUpdateTime( - fileCommits, - project, - project.GetGitHubInnerUrl(languageCode, documentName), - lastKnownSignificantUpdateTime, - documentCreationTime - ) ?? lastKnownSignificantUpdateTime - : null; + var content = await DownloadWebContentAsStringAsync(rawDocumentUrl, token, userAgent); + var commits = await GetGitHubCommitsOrNull(project, documentName, languageCode, version); + + var documentCreationTime = GetFirstCommitDate(commits); + var lastUpdateTime = GetLastCommitDate(commits); + var lastSignificantUpdateTime = await GetLastKnownSignificantUpdateTime(project, documentName, languageCode, version, lastKnownSignificantUpdateTime, isNavigationDocument, isParameterDocument, commits, documentCreationTime); - var document = new Document(GuidGenerator.Create(), + var document = new Document + ( + GuidGenerator.Create(), project.Id, documentName, version, languageCode, fileName, - await DownloadWebContentAsStringAsync(rawDocumentUrl, token, userAgent), + content, project.Format, editLink, rootUrl, rawRootUrl, localDirectory, documentCreationTime, - fileCommits.FirstOrDefault()?.Commit.Author.Date.DateTime ?? DateTime.MinValue, + lastUpdateTime, DateTime.Now, - lastSignificantUpdateTime); + lastSignificantUpdateTime + ); + + if (isNavigationDocument || isParameterDocument) + { + return document; + } + + var authors = GetAuthors(commits); + foreach (var author in authors) + { + document.AddContributor(author.Login, author.HtmlUrl, author.AvatarUrl); + } - var authors = fileCommits + return document; + } + + private async Task GetLastKnownSignificantUpdateTime( + Project project, + string documentName, + string languageCode, + string version, + DateTime? lastKnownSignificantUpdateTime, + bool isNavigationDocument, + bool isParameterDocument, + IReadOnlyList commits, + DateTime documentCreationTime) + { + return !isNavigationDocument && !isParameterDocument && version == project.LatestVersionBranchName + ? await GetLastSignificantUpdateTime( + commits, + project, + project.GetGitHubInnerUrl(languageCode, documentName), + lastKnownSignificantUpdateTime, + documentCreationTime + ) ?? lastKnownSignificantUpdateTime + : null; + } + + private static List GetAuthors(IReadOnlyList commits) + { + if (commits == null || !commits.Any()) + { + return new List(); + } + + return commits .Where(x => x.Author != null) .Select(x => x.Author) .GroupBy(x => x.Id) .OrderByDescending(x => x.Count()) - .Select(x => x.FirstOrDefault()).ToList(); + .Select(x => x.FirstOrDefault()) + .ToList(); + } + + private static DateTime GetLastCommitDate(IReadOnlyList commits) + { + return GetCommitDate(commits, false); + } - if (!isNavigationDocument && !isParameterDocument) + private static DateTime GetFirstCommitDate(IReadOnlyList commits) + { + return GetCommitDate(commits, true); + } + + private static DateTime GetCommitDate(IReadOnlyList commits, bool isFirstCommit) + { + if (commits == null) { - foreach (var author in authors) - { - document.AddContributor(author.Login, author.HtmlUrl, author.AvatarUrl); - } + return DateTime.MinValue; } - return document; + var gitHubCommit = isFirstCommit ? + commits.LastOrDefault() : //first commit + commits.FirstOrDefault(); //last commit + + if (gitHubCommit == null) + { + return DateTime.MinValue; + } + + if (gitHubCommit.Commit == null) + { + return DateTime.MinValue; + } + + if (gitHubCommit.Commit.Author == null) + { + return DateTime.MinValue; + } + + return gitHubCommit.Commit.Author.Date.DateTime; + } + + private async Task> GetGitHubCommitsOrNull(Project project, string documentName, string languageCode, string version) + { + /* + * Getting file commits usually throws "Resource temporarily unavailable" or "Network is unreachable" + * This is a trival information and running this inside try-catch is safer. + */ + + try + { + return await GetFileCommitsAsync(project, version, project.GetGitHubInnerUrl(languageCode, documentName)); + } + catch (Exception e) + { + Logger.LogError(e.ToString()); + return null; + } } private async Task GetLastSignificantUpdateTime( - IReadOnlyList fileCommits, + IReadOnlyList commits, Project project, string fileName, DateTime? lastKnownSignificantUpdateTime, DateTime documentCreationTime) { - if (!fileCommits.Any()) + if (commits == null || !commits.Any()) { return null; } - var fileCommitsAfterCreation = fileCommits.Take(fileCommits.Count - 1); + var fileCommitsAfterCreation = commits.Take(commits.Count - 1); var commitsToEvaluate = (lastKnownSignificantUpdateTime != null ? fileCommitsAfterCreation.Where(c => c.Commit.Author.Date.DateTime > lastKnownSignificantUpdateTime) @@ -117,7 +204,6 @@ namespace Volo.Docs.GitHub.Documents foreach (var gitHubCommit in commitsToEvaluate) { - var fullCommit = await _githubRepositoryManager.GetSingleCommitsAsync( GetOwnerNameFromUrl(project.GetGitHubUrl()), GetRepositoryNameFromUrl(project.GetGitHubUrl()),