diff --git a/modules/docs/src/Volo.Docs.Admin.Application/Volo/Docs/Admin/Projects/ProjectAdminAppService.cs b/modules/docs/src/Volo.Docs.Admin.Application/Volo/Docs/Admin/Projects/ProjectAdminAppService.cs index f82f578da5..838e84f9ad 100644 --- a/modules/docs/src/Volo.Docs.Admin.Application/Volo/Docs/Admin/Projects/ProjectAdminAppService.cs +++ b/modules/docs/src/Volo.Docs.Admin.Application/Volo/Docs/Admin/Projects/ProjectAdminAppService.cs @@ -134,15 +134,29 @@ namespace Volo.Docs.Admin.Projects { throw new Exception("Cannot find the project with the Id " + projectId); } - - var docs = (await _documentRepository.GetListByProjectId(project.Id)) - .Where(doc => doc.FileName != project.NavigationDocumentName && doc.FileName != project.ParametersDocumentName) - .ToList(); + await _elasticSearchService.DeleteAllByProjectIdAsync(project.Id); - - if(docs.Any()) + + var docsCount = await _documentRepository.GetUniqueDocumentCountByProjectIdAsync(projectId); + + if (docsCount == 0) + { + return; + } + + const int maxResultCount = 1000; + + var skipCount = 0; + while(skipCount < docsCount) { - await _elasticSearchService.AddOrUpdateManyAsync(docs); + var docs = await _documentRepository.GetUniqueDocumentsByProjectIdPagedAsync(projectId, skipCount, maxResultCount); + docs = docs.Where(doc => doc.FileName != project.NavigationDocumentName && doc.FileName != project.ParametersDocumentName).ToList(); + if (!docs.Any()) + { + return; + } + await _elasticSearchService.AddOrUpdateManyAsync(docs); + skipCount += maxResultCount; } } diff --git a/modules/docs/src/Volo.Docs.Domain/Volo/Docs/Documents/FullSearch/Elastic/ElasticDocumentFullSearch.cs b/modules/docs/src/Volo.Docs.Domain/Volo/Docs/Documents/FullSearch/Elastic/ElasticDocumentFullSearch.cs index c0a5b1295f..22d4bf9ff4 100644 --- a/modules/docs/src/Volo.Docs.Domain/Volo/Docs/Documents/FullSearch/Elastic/ElasticDocumentFullSearch.cs +++ b/modules/docs/src/Volo.Docs.Domain/Volo/Docs/Documents/FullSearch/Elastic/ElasticDocumentFullSearch.cs @@ -58,6 +58,26 @@ namespace Volo.Docs.Documents.FullSearch.Elastic public virtual async Task AddOrUpdateAsync(Document document, CancellationToken cancellationToken = default) { var client = _clientProvider.GetClient(); + + // exist by name, project id, language code and version + var existResponse = await client.SearchAsync(s => s + .Query(q => q + .Bool(b => b + .Must(m => m + .Term(t => t.ProjectId, NormalizeField(document.ProjectId)) + && m.Term(t => t.LanguageCode, NormalizeField(document.LanguageCode)) + && m.Term(t => t.Version, NormalizeField(document.Version)) + && m.Term(t => t.Name, document.Name) + ) + ) + ), cancellationToken); + + HandleError(existResponse); + + if (existResponse.Documents.Count != 0) + { + HandleError(await client.DeleteManyAsync(existResponse.Documents, _options.IndexName, cancellationToken)); + } var esDocument = new EsDocument { diff --git a/modules/docs/src/Volo.Docs.Domain/Volo/Docs/Documents/IDocumentRepository.cs b/modules/docs/src/Volo.Docs.Domain/Volo/Docs/Documents/IDocumentRepository.cs index 7dc8f13147..86fb2cdc7f 100644 --- a/modules/docs/src/Volo.Docs.Domain/Volo/Docs/Documents/IDocumentRepository.cs +++ b/modules/docs/src/Volo.Docs.Domain/Volo/Docs/Documents/IDocumentRepository.cs @@ -13,6 +13,10 @@ namespace Volo.Docs.Documents Task> GetListByProjectId(Guid projectId, CancellationToken cancellationToken = default); + Task> GetUniqueDocumentsByProjectIdPagedAsync(Guid projectId, int skipCount, int maxResultCount, CancellationToken cancellationToken = default); + + Task GetUniqueDocumentCountByProjectIdAsync(Guid projectId, CancellationToken cancellationToken = default); + Task UpdateProjectLastCachedTimeAsync(Guid projectId, DateTime cachedTime, CancellationToken cancellationToken = default); diff --git a/modules/docs/src/Volo.Docs.EntityFrameworkCore/Volo/Docs/Documents/EFCoreDocumentRepository.cs b/modules/docs/src/Volo.Docs.EntityFrameworkCore/Volo/Docs/Documents/EFCoreDocumentRepository.cs index 1120e36d01..a585545bdf 100644 --- a/modules/docs/src/Volo.Docs.EntityFrameworkCore/Volo/Docs/Documents/EFCoreDocumentRepository.cs +++ b/modules/docs/src/Volo.Docs.EntityFrameworkCore/Volo/Docs/Documents/EFCoreDocumentRepository.cs @@ -56,6 +56,27 @@ namespace Volo.Docs.Documents return await (await GetDbSetAsync()).Where(d => d.ProjectId == projectId).ToListAsync(GetCancellationToken(cancellationToken)); } + public virtual async Task> GetUniqueDocumentsByProjectIdPagedAsync(Guid projectId, int skipCount, int maxResultCount, + CancellationToken cancellationToken = default) + { + return await (await GetDbSetAsync()) + .Where(d => d.ProjectId == projectId) + .OrderBy(x => x.LastCachedTime) + .GroupBy(x => new { x.Name, x.LanguageCode, x.Version }) + .Select(group => group.First()) + .Skip(skipCount) + .Take(maxResultCount) + .ToListAsync(cancellationToken); + } + + public virtual async Task GetUniqueDocumentCountByProjectIdAsync(Guid projectId, CancellationToken cancellationToken = default) + { + return await (await GetDbSetAsync()) + .Where(d => d.ProjectId == projectId) + .GroupBy(x => new {x.FileName, x.Version, x.LanguageCode}) + .LongCountAsync(GetCancellationToken(cancellationToken)); + } + public async Task UpdateProjectLastCachedTimeAsync(Guid projectId, DateTime cachedTime, CancellationToken cancellationToken = default) { diff --git a/modules/docs/src/Volo.Docs.MongoDB/Volo/Docs/Documents/MongoDocumentRepository.cs b/modules/docs/src/Volo.Docs.MongoDB/Volo/Docs/Documents/MongoDocumentRepository.cs index 9a3750deb0..c678947f5c 100644 --- a/modules/docs/src/Volo.Docs.MongoDB/Volo/Docs/Documents/MongoDocumentRepository.cs +++ b/modules/docs/src/Volo.Docs.MongoDB/Volo/Docs/Documents/MongoDocumentRepository.cs @@ -54,6 +54,26 @@ namespace Volo.Docs.Documents return await (await GetMongoQueryableAsync(cancellationToken)).Where(d => d.ProjectId == projectId).ToListAsync(GetCancellationToken(cancellationToken)); } + public virtual async Task> GetUniqueDocumentsByProjectIdPagedAsync(Guid projectId, int skipCount, int maxResultCount, + CancellationToken cancellationToken = default) + { + return await (await GetMongoQueryableAsync(cancellationToken)) + .Where(d => d.ProjectId == projectId) + .OrderBy(x => x.LastCachedTime) + .GroupBy(x => new { x.Name, x.LanguageCode, x.Version }) + .Select(group => group.First()) + .Skip(skipCount) + .Take(maxResultCount) + .ToListAsync(cancellationToken); + } + + public virtual async Task GetUniqueDocumentCountByProjectIdAsync(Guid projectId, CancellationToken cancellationToken = default) + { + return await (await GetMongoQueryableAsync(cancellationToken)).Where(d => d.ProjectId == projectId) + .GroupBy(x => new { x.Name, x.LanguageCode, x.Version }) + .LongCountAsync(GetCancellationToken(cancellationToken)); + } + public async Task UpdateProjectLastCachedTimeAsync(Guid projectId, DateTime cachedTime, CancellationToken cancellationToken = default) { diff --git a/modules/docs/test/Volo.Docs.TestBase/Volo/Docs/DocumentRepository_Tests.cs b/modules/docs/test/Volo.Docs.TestBase/Volo/Docs/DocumentRepository_Tests.cs index 742f80fe17..1c0c63bf64 100644 --- a/modules/docs/test/Volo.Docs.TestBase/Volo/Docs/DocumentRepository_Tests.cs +++ b/modules/docs/test/Volo.Docs.TestBase/Volo/Docs/DocumentRepository_Tests.cs @@ -43,5 +43,19 @@ namespace Volo.Docs var documentsAfterClear = await DocumentRepository.GetListByProjectId(DocsTestData.ProjectId); documentsAfterClear.ForEach(d => d.LastCachedTime.ShouldBe(DateTime.MinValue)); } + + [Fact] + public async Task GetUniqueDocumentsByProjectIdPagedAsync() + { + var documents = await DocumentRepository.GetUniqueDocumentsByProjectIdPagedAsync(DocsTestData.ProjectId, 0, 10); + documents.Count.ShouldBe(1); + } + + [Fact] + public async Task GetUniqueDocumentCountByProjectIdAsync() + { + var count = await DocumentRepository.GetUniqueDocumentCountByProjectIdAsync(DocsTestData.ProjectId); + count.ShouldBe(1); + } } }