From 943a0892e2bcda99a7729d24c168e8e1c25ca4c0 Mon Sep 17 00:00:00 2001 From: liangshiwei Date: Thu, 24 Feb 2022 13:16:27 +0800 Subject: [PATCH] Docs: Batch re-index --- .../Admin/Projects/ProjectAdminAppService.cs | 19 ++++++------------ .../Elastic/ElasticDocumentFullSearch.cs | 20 ++++++++++++++++++- .../FullSearch/Elastic/IDocumentFullSearch.cs | 2 ++ .../GitHub/Documents/GithubDocumentSource.cs | 2 +- 4 files changed, 28 insertions(+), 15 deletions(-) diff --git a/modules/docs/src/Volo.Docs.Admin.Application/Volo/Docs/Admin/Projects/ProjectAdminAppService.cs b/modules/docs/src/Volo.Docs.Admin.Application/Volo/Docs/Admin/Projects/ProjectAdminAppService.cs index 078288572b..14833cd3e4 100644 --- a/modules/docs/src/Volo.Docs.Admin.Application/Volo/Docs/Admin/Projects/ProjectAdminAppService.cs +++ b/modules/docs/src/Volo.Docs.Admin.Application/Volo/Docs/Admin/Projects/ProjectAdminAppService.cs @@ -1,5 +1,6 @@ using System; using System.Collections.Generic; +using System.Linq; using System.Threading.Tasks; using Microsoft.AspNetCore.Authorization; using Volo.Abp.Application.Dtos; @@ -134,22 +135,14 @@ namespace Volo.Docs.Admin.Projects throw new Exception("Cannot find the project with the Id " + projectId); } - var docs = await _documentRepository.GetListByProjectId(project.Id); + var docs = (await _documentRepository.GetListByProjectId(project.Id)) + .Where(doc => doc.FileName != project.NavigationDocumentName && doc.FileName != project.ParametersDocumentName) + .ToList(); await _elasticSearchService.DeleteAllByProjectIdAsync(project.Id); - foreach (var doc in docs) + if(docs.Any()) { - if (doc.FileName == project.NavigationDocumentName) - { - continue; - } - - if (doc.FileName == project.ParametersDocumentName) - { - continue; - } - - await _elasticSearchService.AddOrUpdateAsync(doc); + await _elasticSearchService.AddOrUpdateManyAsync(docs); } } diff --git a/modules/docs/src/Volo.Docs.Domain/Volo/Docs/Documents/FullSearch/Elastic/ElasticDocumentFullSearch.cs b/modules/docs/src/Volo.Docs.Domain/Volo/Docs/Documents/FullSearch/Elastic/ElasticDocumentFullSearch.cs index 7a47cd7b45..6a6c8cc7e4 100644 --- a/modules/docs/src/Volo.Docs.Domain/Volo/Docs/Documents/FullSearch/Elastic/ElasticDocumentFullSearch.cs +++ b/modules/docs/src/Volo.Docs.Domain/Volo/Docs/Documents/FullSearch/Elastic/ElasticDocumentFullSearch.cs @@ -1,5 +1,6 @@ using System; using System.Collections.Generic; +using System.Linq; using System.Threading; using System.Threading.Tasks; using Elasticsearch.Net; @@ -69,7 +70,24 @@ namespace Volo.Docs.Documents.FullSearch.Elastic Version = NormalizeField(document.Version) }; - await client.IndexAsync(esDocument, x => x.Index(_options.IndexName), cancellationToken); + HandleError(await client.IndexAsync(esDocument, x => x.Index(_options.IndexName), cancellationToken)); + } + + public virtual async Task AddOrUpdateManyAsync(IEnumerable documents, CancellationToken cancellationToken = default) + { + var client = _clientProvider.GetClient(); + + var esDocuments = documents.Select(x => new EsDocument { + Id = NormalizeField(x.Id), + ProjectId = NormalizeField(x.ProjectId), + Name = x.Name, + FileName = x.FileName, + Content = x.Content, + LanguageCode = NormalizeField(x.LanguageCode), + Version = NormalizeField(x.Version) + }); + + HandleError(await client.IndexManyAsync(esDocuments, _options.IndexName, cancellationToken)); } public virtual async Task DeleteAsync(Guid id, CancellationToken cancellationToken = default) diff --git a/modules/docs/src/Volo.Docs.Domain/Volo/Docs/Documents/FullSearch/Elastic/IDocumentFullSearch.cs b/modules/docs/src/Volo.Docs.Domain/Volo/Docs/Documents/FullSearch/Elastic/IDocumentFullSearch.cs index c84df577b7..2c7cf1f3e6 100644 --- a/modules/docs/src/Volo.Docs.Domain/Volo/Docs/Documents/FullSearch/Elastic/IDocumentFullSearch.cs +++ b/modules/docs/src/Volo.Docs.Domain/Volo/Docs/Documents/FullSearch/Elastic/IDocumentFullSearch.cs @@ -11,6 +11,8 @@ namespace Volo.Docs.Documents.FullSearch.Elastic Task AddOrUpdateAsync(Document document, CancellationToken cancellationToken = default); + Task AddOrUpdateManyAsync(IEnumerable documents, CancellationToken cancellationToken = default); + Task DeleteAsync(Guid id, CancellationToken cancellationToken = default); Task DeleteAllAsync(CancellationToken cancellationToken = default); diff --git a/modules/docs/src/Volo.Docs.Domain/Volo/Docs/GitHub/Documents/GithubDocumentSource.cs b/modules/docs/src/Volo.Docs.Domain/Volo/Docs/GitHub/Documents/GithubDocumentSource.cs index d03d8078e4..060c6dcf08 100644 --- a/modules/docs/src/Volo.Docs.Domain/Volo/Docs/GitHub/Documents/GithubDocumentSource.cs +++ b/modules/docs/src/Volo.Docs.Domain/Volo/Docs/GitHub/Documents/GithubDocumentSource.cs @@ -383,7 +383,7 @@ namespace Volo.Docs.GitHub.Documents catch (Exception ex) { //TODO: Only handle when document is really not available - Logger.LogWarning(ex.Message, ex); + Logger.LogWarning($"{ex.Message}: {rawUrl}", ex); throw new DocumentNotFoundException(rawUrl); } }