Merge pull request #15157 from abpframework/issue-15130

docs: offline support

modules/docs/src/Volo.Docs.Domain/Volo/Docs/DocsDomainModule.cs

@@ -1,4 +1,5 @@
 using System;
+using System.Collections.Generic;
 using System.Threading.Tasks;
 using Microsoft.Extensions.DependencyInjection;
 using Microsoft.Extensions.Options;
@@ -13,6 +14,7 @@ using Volo.Abp.VirtualFileSystem;
 using Volo.Docs.Documents;
 using Volo.Docs.Documents.FullSearch.Elastic;
 using Volo.Docs.FileSystem.Documents;
+using Volo.Docs.GitHub;
 using Volo.Docs.GitHub.Documents;
 using Volo.Docs.Localization;
 using Volo.Docs.Projects;
@@ -59,6 +61,16 @@ namespace Volo.Docs
                 options.Sources[GithubDocumentSource.Type] = typeof(GithubDocumentSource);
                 options.Sources[FileSystemDocumentSource.Type] = typeof(FileSystemDocumentSource);
             });
+            
+            Configure<DocsGithubLanguageOptions>(options =>
+            {
+                options.DefaultLanguage = new LanguageConfigElement 
+                {
+                    Code = "en", 
+                    DisplayName = "English", 
+                    IsDefault = true
+                };
+            });
 
             context.Services.AddHttpClient(GithubRepositoryManager.HttpClientName, client =>
             {

modules/docs/src/Volo.Docs.Domain/Volo/Docs/Documents/IDocumentSource.cs

@@ -2,7 +2,6 @@ using System;
 using System.Collections.Generic;
 using System.Threading.Tasks;
 using Volo.Abp.Domain.Services;
-using Volo.Docs.GitHub.Documents;
 using Volo.Docs.Projects;
 
 namespace Volo.Docs.Documents

modules/docs/src/Volo.Docs.Domain/Volo/Docs/GitHub/DocsGithubLanguageOptions.cs

@@ -0,0 +1,8 @@
+using Volo.Docs.Documents;
+
+namespace Volo.Docs.GitHub;
+
+public class DocsGithubLanguageOptions
+{
+    public LanguageConfigElement DefaultLanguage { get; set; }
+}

modules/docs/src/Volo.Docs.Domain/Volo/Docs/GitHub/Documents/GithubDocumentSource.cs

@@ -1,8 +1,10 @@
 using System;
 using System.Collections.Generic;
 using System.Linq;
+using System.Net.Http;
 using System.Threading.Tasks;
 using Microsoft.Extensions.Logging;
+using Microsoft.Extensions.Options;
 using Volo.Abp.Domain.Services;
 using Volo.Docs.Documents;
 using Volo.Docs.GitHub.Projects;
@@ -23,20 +25,25 @@ namespace Volo.Docs.GitHub.Documents
 
         private readonly IGithubRepositoryManager _githubRepositoryManager;
         private readonly IGithubPatchAnalyzer _githubPatchAnalyzer;
-
+        private readonly IDocumentRepository _documentRepository;
-        public GithubDocumentSource(IGithubRepositoryManager githubRepositoryManager, IGithubPatchAnalyzer githubPatchAnalyzer)
+        private readonly DocsGithubLanguageOptions _docsGithubLanguageOptions;
+
+        public GithubDocumentSource(
+            IGithubRepositoryManager githubRepositoryManager, 
+            IGithubPatchAnalyzer githubPatchAnalyzer, 
+            IDocumentRepository documentRepository,
+            IOptions<DocsGithubLanguageOptions> docsGithubLanguageOptions)
         {
             _githubRepositoryManager = githubRepositoryManager;
             _githubPatchAnalyzer = githubPatchAnalyzer;
+            _documentRepository = documentRepository;
+            _docsGithubLanguageOptions = docsGithubLanguageOptions.Value;
         }
 
         public virtual async Task<Document> GetDocumentAsync(Project project, string documentName, string languageCode, string version, DateTime? lastKnownSignificantUpdateTime = null)
         {
-            var token = project.GetGitHubAccessTokenOrNull();
             var rootUrl = project.GetGitHubUrl(version);
-            var userAgent = project.GetGithubUserAgentOrNull();
             var rawRootUrl = CalculateRawRootUrlWithLanguageCode(rootUrl, languageCode);
-            var rawDocumentUrl = rawRootUrl + documentName;
             var isNavigationDocument = documentName == project.NavigationDocumentName;
             var isParameterDocument = documentName == project.ParametersDocumentName;
             var editLink = rootUrl.ReplaceFirst("/tree/", "/blob/").EnsureEndsWith('/') + languageCode + "/" + documentName;
@@ -49,12 +56,12 @@ namespace Volo.Docs.GitHub.Documents
                 fileName = documentName.Substring(documentName.LastIndexOf('/') + 1);
             }
 
-            var content = await DownloadWebContentAsStringAsync(rawDocumentUrl, token, userAgent);
+            var content = await DownloadWebContentAsStringAsync(project, documentName, languageCode, version);
             var commits = await GetGitHubCommitsOrNull(project, documentName, languageCode, version);
 
             var documentCreationTime = GetFirstCommitDate(commits);
             var lastUpdateTime = GetLastCommitDate(commits);
-            var lastSignificantUpdateTime = await GetLastKnownSignificantUpdateTime(project, documentName, languageCode, version, lastKnownSignificantUpdateTime, isNavigationDocument, isParameterDocument, commits, documentCreationTime);
+            var lastSignificantUpdateTime = await GetLastKnownSignificantUpdateTime(project, documentName, languageCode, version, lastKnownSignificantUpdateTime, isNavigationDocument, isParameterDocument, commits);
 
             var document = new Document
             (
@@ -100,18 +107,24 @@ namespace Volo.Docs.GitHub.Documents
             DateTime? lastKnownSignificantUpdateTime,
             bool isNavigationDocument,
             bool isParameterDocument,
-            IReadOnlyList<GitHubCommit> commits,
+            IReadOnlyList<GitHubCommit> commits)
-            DateTime documentCreationTime)
         {
-            return !isNavigationDocument && !isParameterDocument && version == project.LatestVersionBranchName
+            try
-                ? await GetLastSignificantUpdateTime(
+            {
-                      commits,
+                return !isNavigationDocument && !isParameterDocument && version == project.LatestVersionBranchName
-                      project,
+                    ? await GetLastSignificantUpdateTime(
-                      project.GetGitHubInnerUrl(languageCode, documentName),
+                        commits,
-                      lastKnownSignificantUpdateTime,
+                        project,
-                      documentCreationTime
+                        project.GetGitHubInnerUrl(languageCode, documentName),
-                  ) ?? lastKnownSignificantUpdateTime
+                        lastKnownSignificantUpdateTime
-                : null;
+                    ) ?? lastKnownSignificantUpdateTime
+                    : null;
+            }
+            catch
+            {
+                Logger.LogWarning("Could not retrieved the last update time from Github.");
+                return null;
+            }
         }
 
         private static List<DocumentAuthor> GetAuthors(IReadOnlyList<GitHubCommit> commits)
@@ -206,8 +219,7 @@ namespace Volo.Docs.GitHub.Documents
             IReadOnlyList<GitHubCommit> commits,
             Project project,
             string fileName,
-            DateTime? lastKnownSignificantUpdateTime,
+            DateTime? lastKnownSignificantUpdateTime)
-            DateTime documentCreationTime)
         {
             if (commits == null || !commits.Any())
             {
@@ -317,17 +329,28 @@ namespace Volo.Docs.GitHub.Documents
             var token = project.GetGitHubAccessTokenOrNull();
             var rootUrl = project.GetGitHubUrl(version);
             var userAgent = project.GetGithubUserAgentOrNull();
-
             var url = CalculateRawRootUrl(rootUrl) + DocsDomainConsts.LanguageConfigFileName;
 
-            var configAsJson = await DownloadWebContentAsStringAsync(url, token, userAgent);
+            try
+            {
+                var configAsJson = await _githubRepositoryManager.GetFileRawStringContentAsync(url, token, userAgent);
+
+                if (!DocsJsonSerializerHelper.TryDeserialize<LanguageConfig>(configAsJson, out var languageConfig))
+                {
+                    throw new UserFriendlyException($"Cannot validate language config file '{DocsDomainConsts.LanguageConfigFileName}' for the project {project.Name} - v{version}.");
+                }
 
-            if (!DocsJsonSerializerHelper.TryDeserialize<LanguageConfig>(configAsJson, out var languageConfig))
+                return languageConfig;
+            }
+            catch
             {
-                throw new UserFriendlyException($"Cannot validate language config file '{DocsDomainConsts.LanguageConfigFileName}' for the project {project.Name} - v{version}.");
+                Logger.LogWarning("Could not retrieved language list from Github. Using the default language from DocsGithubLanguageOptions."); 
+                
+                return new LanguageConfig 
+                {
+                    Languages = new List<LanguageConfigElement> { _docsGithubLanguageOptions.DefaultLanguage }
+                };
             }
-
-            return languageConfig;
         }
 
         private async Task<IReadOnlyList<GitHubCommit>> GetFileCommitsAsync(Project project, string version, string filename)
@@ -372,19 +395,38 @@ namespace Volo.Docs.GitHub.Documents
             }
         }
 
-        private async Task<string> DownloadWebContentAsStringAsync(string rawUrl, string token, string userAgent)
+        private async Task<string> DownloadWebContentAsStringAsync(Project project, string documentName, string languageCode, string version)
         {
+            var token = project.GetGitHubAccessTokenOrNull();
+            var rootUrl = project.GetGitHubUrl(version);
+            var userAgent = project.GetGithubUserAgentOrNull();
+            var rawRootUrl = CalculateRawRootUrlWithLanguageCode(rootUrl, languageCode);
+            var rawDocumentUrl = rawRootUrl + documentName;
+
             try
             {
-                Logger.LogInformation("Downloading content from Github (DownloadWebContentAsStringAsync): " + rawUrl);
+                Logger.LogInformation("Downloading content from Github (DownloadWebContentAsStringAsync): " +
+                                      rawDocumentUrl);
 
-                return await _githubRepositoryManager.GetFileRawStringContentAsync(rawUrl, token, userAgent);
+                return await _githubRepositoryManager.GetFileRawStringContentAsync(rawDocumentUrl, token, userAgent);
+            }
+            catch (HttpRequestException ex)
+            {
+                Logger.LogWarning($"Could not retrieve the document ({rawDocumentUrl}) from Github. Trying to get it from database...");
+                
+                var document = await _documentRepository.FindAsync(project.Id, documentName, languageCode, version);
+                if (document == null)
+                {
+                    throw new DocumentNotFoundException(rawDocumentUrl);
+                }
+                
+                return document.Content;
             }
             catch (Exception ex)
             {
                 //TODO: Only handle when document is really not available
-                Logger.LogWarning($"{ex.Message}: {rawUrl}", ex);
+                Logger.LogWarning($"{ex.Message}: {rawDocumentUrl}", ex);
-                throw new DocumentNotFoundException(rawUrl);
+                throw new DocumentNotFoundException(rawDocumentUrl);
             }
         }