From ef985630afdd3a96f906da52b078c1772f0f7e40 Mon Sep 17 00:00:00 2001 From: Salih Date: Fri, 3 May 2024 10:36:53 +0300 Subject: [PATCH 1/5] Fix reindex problem --- .../Admin/Projects/ProjectAdminAppService.cs | 28 ++++++++++++++----- .../Docs/Documents/IDocumentRepository.cs | 4 +++ .../Documents/EFCoreDocumentRepository.cs | 11 ++++++++ .../Docs/Documents/MongoDocumentRepository.cs | 15 ++++++++++ 4 files changed, 51 insertions(+), 7 deletions(-) diff --git a/modules/docs/src/Volo.Docs.Admin.Application/Volo/Docs/Admin/Projects/ProjectAdminAppService.cs b/modules/docs/src/Volo.Docs.Admin.Application/Volo/Docs/Admin/Projects/ProjectAdminAppService.cs index f82f578da5..b11548f0cf 100644 --- a/modules/docs/src/Volo.Docs.Admin.Application/Volo/Docs/Admin/Projects/ProjectAdminAppService.cs +++ b/modules/docs/src/Volo.Docs.Admin.Application/Volo/Docs/Admin/Projects/ProjectAdminAppService.cs @@ -134,15 +134,29 @@ namespace Volo.Docs.Admin.Projects { throw new Exception("Cannot find the project with the Id " + projectId); } - - var docs = (await _documentRepository.GetListByProjectId(project.Id)) - .Where(doc => doc.FileName != project.NavigationDocumentName && doc.FileName != project.ParametersDocumentName) - .ToList(); + await _elasticSearchService.DeleteAllByProjectIdAsync(project.Id); - - if(docs.Any()) + + var docsCount = await _documentRepository.GetCountByProjectId(projectId); + + if (docsCount == 0) + { + return; + } + + const int maxResultCount = 1000; + + var skipCount = 0; + while(skipCount < docsCount) { - await _elasticSearchService.AddOrUpdateManyAsync(docs); + var docs = await _documentRepository.GetListByProjectId(projectId, skipCount, maxResultCount); + docs = docs.Where(doc => doc.FileName != project.NavigationDocumentName && doc.FileName != project.ParametersDocumentName).ToList(); + if (!docs.Any()) + { + return; + } + await _elasticSearchService.AddOrUpdateManyAsync(docs); + skipCount += maxResultCount; } } diff --git a/modules/docs/src/Volo.Docs.Domain/Volo/Docs/Documents/IDocumentRepository.cs b/modules/docs/src/Volo.Docs.Domain/Volo/Docs/Documents/IDocumentRepository.cs index 7dc8f13147..2493906a12 100644 --- a/modules/docs/src/Volo.Docs.Domain/Volo/Docs/Documents/IDocumentRepository.cs +++ b/modules/docs/src/Volo.Docs.Domain/Volo/Docs/Documents/IDocumentRepository.cs @@ -13,6 +13,10 @@ namespace Volo.Docs.Documents Task> GetListByProjectId(Guid projectId, CancellationToken cancellationToken = default); + Task> GetListByProjectId(Guid projectId, int skipCount, int maxResultCount, CancellationToken cancellationToken = default); + + Task GetCountByProjectId(Guid projectId, CancellationToken cancellationToken = default); + Task UpdateProjectLastCachedTimeAsync(Guid projectId, DateTime cachedTime, CancellationToken cancellationToken = default); diff --git a/modules/docs/src/Volo.Docs.EntityFrameworkCore/Volo/Docs/Documents/EFCoreDocumentRepository.cs b/modules/docs/src/Volo.Docs.EntityFrameworkCore/Volo/Docs/Documents/EFCoreDocumentRepository.cs index 1120e36d01..f72e9076b0 100644 --- a/modules/docs/src/Volo.Docs.EntityFrameworkCore/Volo/Docs/Documents/EFCoreDocumentRepository.cs +++ b/modules/docs/src/Volo.Docs.EntityFrameworkCore/Volo/Docs/Documents/EFCoreDocumentRepository.cs @@ -56,6 +56,17 @@ namespace Volo.Docs.Documents return await (await GetDbSetAsync()).Where(d => d.ProjectId == projectId).ToListAsync(GetCancellationToken(cancellationToken)); } + public virtual async Task> GetListByProjectId(Guid projectId, int skipCount, int maxResultCount, + CancellationToken cancellationToken = default) + { + return await (await GetDbSetAsync()).Where(d => d.ProjectId == projectId).PageBy(skipCount, maxResultCount).ToListAsync(GetCancellationToken(cancellationToken)); + } + + public virtual async Task GetCountByProjectId(Guid projectId, CancellationToken cancellationToken = default) + { + return await (await GetDbSetAsync()).Where(d => d.ProjectId == projectId).LongCountAsync(GetCancellationToken(cancellationToken)); + } + public async Task UpdateProjectLastCachedTimeAsync(Guid projectId, DateTime cachedTime, CancellationToken cancellationToken = default) { diff --git a/modules/docs/src/Volo.Docs.MongoDB/Volo/Docs/Documents/MongoDocumentRepository.cs b/modules/docs/src/Volo.Docs.MongoDB/Volo/Docs/Documents/MongoDocumentRepository.cs index 9a3750deb0..d127a46907 100644 --- a/modules/docs/src/Volo.Docs.MongoDB/Volo/Docs/Documents/MongoDocumentRepository.cs +++ b/modules/docs/src/Volo.Docs.MongoDB/Volo/Docs/Documents/MongoDocumentRepository.cs @@ -54,6 +54,21 @@ namespace Volo.Docs.Documents return await (await GetMongoQueryableAsync(cancellationToken)).Where(d => d.ProjectId == projectId).ToListAsync(GetCancellationToken(cancellationToken)); } + public virtual async Task> GetListByProjectId(Guid projectId, int skipCount, int maxResultCount, + CancellationToken cancellationToken = default) + { + return await (await GetMongoQueryableAsync(cancellationToken)).Where(d => d.ProjectId == projectId) + .OrderBy(x => x.Name) + .Skip(skipCount) + .Take(maxResultCount) + .ToListAsync(GetCancellationToken(cancellationToken)); + } + + public virtual async Task GetCountByProjectId(Guid projectId, CancellationToken cancellationToken = default) + { + return await (await GetMongoQueryableAsync(cancellationToken)).LongCountAsync(x => x.ProjectId == projectId, GetCancellationToken(cancellationToken)); + } + public async Task UpdateProjectLastCachedTimeAsync(Guid projectId, DateTime cachedTime, CancellationToken cancellationToken = default) { From 7e9f4e43461c0eb02e35df0876d9a674b02b1e25 Mon Sep 17 00:00:00 2001 From: Salih Date: Fri, 3 May 2024 15:14:47 +0300 Subject: [PATCH 2/5] Fix reindex multiple index problem --- .../Admin/Projects/ProjectAdminAppService.cs | 4 ++-- .../Volo/Docs/Documents/IDocumentRepository.cs | 4 ++-- .../Docs/Documents/EFCoreDocumentRepository.cs | 18 ++++++++++++++---- .../Docs/Documents/MongoDocumentRepository.cs | 17 +++++++++++------ .../Volo/Docs/DocumentRepository_Tests.cs | 14 ++++++++++++++ 5 files changed, 43 insertions(+), 14 deletions(-) diff --git a/modules/docs/src/Volo.Docs.Admin.Application/Volo/Docs/Admin/Projects/ProjectAdminAppService.cs b/modules/docs/src/Volo.Docs.Admin.Application/Volo/Docs/Admin/Projects/ProjectAdminAppService.cs index b11548f0cf..838e84f9ad 100644 --- a/modules/docs/src/Volo.Docs.Admin.Application/Volo/Docs/Admin/Projects/ProjectAdminAppService.cs +++ b/modules/docs/src/Volo.Docs.Admin.Application/Volo/Docs/Admin/Projects/ProjectAdminAppService.cs @@ -137,7 +137,7 @@ namespace Volo.Docs.Admin.Projects await _elasticSearchService.DeleteAllByProjectIdAsync(project.Id); - var docsCount = await _documentRepository.GetCountByProjectId(projectId); + var docsCount = await _documentRepository.GetUniqueDocumentCountByProjectIdAsync(projectId); if (docsCount == 0) { @@ -149,7 +149,7 @@ namespace Volo.Docs.Admin.Projects var skipCount = 0; while(skipCount < docsCount) { - var docs = await _documentRepository.GetListByProjectId(projectId, skipCount, maxResultCount); + var docs = await _documentRepository.GetUniqueDocumentsByProjectIdPagedAsync(projectId, skipCount, maxResultCount); docs = docs.Where(doc => doc.FileName != project.NavigationDocumentName && doc.FileName != project.ParametersDocumentName).ToList(); if (!docs.Any()) { diff --git a/modules/docs/src/Volo.Docs.Domain/Volo/Docs/Documents/IDocumentRepository.cs b/modules/docs/src/Volo.Docs.Domain/Volo/Docs/Documents/IDocumentRepository.cs index 2493906a12..86fb2cdc7f 100644 --- a/modules/docs/src/Volo.Docs.Domain/Volo/Docs/Documents/IDocumentRepository.cs +++ b/modules/docs/src/Volo.Docs.Domain/Volo/Docs/Documents/IDocumentRepository.cs @@ -13,9 +13,9 @@ namespace Volo.Docs.Documents Task> GetListByProjectId(Guid projectId, CancellationToken cancellationToken = default); - Task> GetListByProjectId(Guid projectId, int skipCount, int maxResultCount, CancellationToken cancellationToken = default); + Task> GetUniqueDocumentsByProjectIdPagedAsync(Guid projectId, int skipCount, int maxResultCount, CancellationToken cancellationToken = default); - Task GetCountByProjectId(Guid projectId, CancellationToken cancellationToken = default); + Task GetUniqueDocumentCountByProjectIdAsync(Guid projectId, CancellationToken cancellationToken = default); Task UpdateProjectLastCachedTimeAsync(Guid projectId, DateTime cachedTime, CancellationToken cancellationToken = default); diff --git a/modules/docs/src/Volo.Docs.EntityFrameworkCore/Volo/Docs/Documents/EFCoreDocumentRepository.cs b/modules/docs/src/Volo.Docs.EntityFrameworkCore/Volo/Docs/Documents/EFCoreDocumentRepository.cs index f72e9076b0..a585545bdf 100644 --- a/modules/docs/src/Volo.Docs.EntityFrameworkCore/Volo/Docs/Documents/EFCoreDocumentRepository.cs +++ b/modules/docs/src/Volo.Docs.EntityFrameworkCore/Volo/Docs/Documents/EFCoreDocumentRepository.cs @@ -56,15 +56,25 @@ namespace Volo.Docs.Documents return await (await GetDbSetAsync()).Where(d => d.ProjectId == projectId).ToListAsync(GetCancellationToken(cancellationToken)); } - public virtual async Task> GetListByProjectId(Guid projectId, int skipCount, int maxResultCount, + public virtual async Task> GetUniqueDocumentsByProjectIdPagedAsync(Guid projectId, int skipCount, int maxResultCount, CancellationToken cancellationToken = default) { - return await (await GetDbSetAsync()).Where(d => d.ProjectId == projectId).PageBy(skipCount, maxResultCount).ToListAsync(GetCancellationToken(cancellationToken)); + return await (await GetDbSetAsync()) + .Where(d => d.ProjectId == projectId) + .OrderBy(x => x.LastCachedTime) + .GroupBy(x => new { x.Name, x.LanguageCode, x.Version }) + .Select(group => group.First()) + .Skip(skipCount) + .Take(maxResultCount) + .ToListAsync(cancellationToken); } - public virtual async Task GetCountByProjectId(Guid projectId, CancellationToken cancellationToken = default) + public virtual async Task GetUniqueDocumentCountByProjectIdAsync(Guid projectId, CancellationToken cancellationToken = default) { - return await (await GetDbSetAsync()).Where(d => d.ProjectId == projectId).LongCountAsync(GetCancellationToken(cancellationToken)); + return await (await GetDbSetAsync()) + .Where(d => d.ProjectId == projectId) + .GroupBy(x => new {x.FileName, x.Version, x.LanguageCode}) + .LongCountAsync(GetCancellationToken(cancellationToken)); } public async Task UpdateProjectLastCachedTimeAsync(Guid projectId, DateTime cachedTime, diff --git a/modules/docs/src/Volo.Docs.MongoDB/Volo/Docs/Documents/MongoDocumentRepository.cs b/modules/docs/src/Volo.Docs.MongoDB/Volo/Docs/Documents/MongoDocumentRepository.cs index d127a46907..c678947f5c 100644 --- a/modules/docs/src/Volo.Docs.MongoDB/Volo/Docs/Documents/MongoDocumentRepository.cs +++ b/modules/docs/src/Volo.Docs.MongoDB/Volo/Docs/Documents/MongoDocumentRepository.cs @@ -54,19 +54,24 @@ namespace Volo.Docs.Documents return await (await GetMongoQueryableAsync(cancellationToken)).Where(d => d.ProjectId == projectId).ToListAsync(GetCancellationToken(cancellationToken)); } - public virtual async Task> GetListByProjectId(Guid projectId, int skipCount, int maxResultCount, + public virtual async Task> GetUniqueDocumentsByProjectIdPagedAsync(Guid projectId, int skipCount, int maxResultCount, CancellationToken cancellationToken = default) { - return await (await GetMongoQueryableAsync(cancellationToken)).Where(d => d.ProjectId == projectId) - .OrderBy(x => x.Name) + return await (await GetMongoQueryableAsync(cancellationToken)) + .Where(d => d.ProjectId == projectId) + .OrderBy(x => x.LastCachedTime) + .GroupBy(x => new { x.Name, x.LanguageCode, x.Version }) + .Select(group => group.First()) .Skip(skipCount) .Take(maxResultCount) - .ToListAsync(GetCancellationToken(cancellationToken)); + .ToListAsync(cancellationToken); } - public virtual async Task GetCountByProjectId(Guid projectId, CancellationToken cancellationToken = default) + public virtual async Task GetUniqueDocumentCountByProjectIdAsync(Guid projectId, CancellationToken cancellationToken = default) { - return await (await GetMongoQueryableAsync(cancellationToken)).LongCountAsync(x => x.ProjectId == projectId, GetCancellationToken(cancellationToken)); + return await (await GetMongoQueryableAsync(cancellationToken)).Where(d => d.ProjectId == projectId) + .GroupBy(x => new { x.Name, x.LanguageCode, x.Version }) + .LongCountAsync(GetCancellationToken(cancellationToken)); } public async Task UpdateProjectLastCachedTimeAsync(Guid projectId, DateTime cachedTime, diff --git a/modules/docs/test/Volo.Docs.TestBase/Volo/Docs/DocumentRepository_Tests.cs b/modules/docs/test/Volo.Docs.TestBase/Volo/Docs/DocumentRepository_Tests.cs index 742f80fe17..1c0c63bf64 100644 --- a/modules/docs/test/Volo.Docs.TestBase/Volo/Docs/DocumentRepository_Tests.cs +++ b/modules/docs/test/Volo.Docs.TestBase/Volo/Docs/DocumentRepository_Tests.cs @@ -43,5 +43,19 @@ namespace Volo.Docs var documentsAfterClear = await DocumentRepository.GetListByProjectId(DocsTestData.ProjectId); documentsAfterClear.ForEach(d => d.LastCachedTime.ShouldBe(DateTime.MinValue)); } + + [Fact] + public async Task GetUniqueDocumentsByProjectIdPagedAsync() + { + var documents = await DocumentRepository.GetUniqueDocumentsByProjectIdPagedAsync(DocsTestData.ProjectId, 0, 10); + documents.Count.ShouldBe(1); + } + + [Fact] + public async Task GetUniqueDocumentCountByProjectIdAsync() + { + var count = await DocumentRepository.GetUniqueDocumentCountByProjectIdAsync(DocsTestData.ProjectId); + count.ShouldBe(1); + } } } From c816ac70bd4368b46ad0ac50595bc57418dc59ca Mon Sep 17 00:00:00 2001 From: Salih Date: Mon, 6 May 2024 11:53:55 +0300 Subject: [PATCH 3/5] Update ElasticDocumentFullSearch.cs --- .../FullSearch/Elastic/ElasticDocumentFullSearch.cs | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/modules/docs/src/Volo.Docs.Domain/Volo/Docs/Documents/FullSearch/Elastic/ElasticDocumentFullSearch.cs b/modules/docs/src/Volo.Docs.Domain/Volo/Docs/Documents/FullSearch/Elastic/ElasticDocumentFullSearch.cs index c0a5b1295f..1bf5cb873f 100644 --- a/modules/docs/src/Volo.Docs.Domain/Volo/Docs/Documents/FullSearch/Elastic/ElasticDocumentFullSearch.cs +++ b/modules/docs/src/Volo.Docs.Domain/Volo/Docs/Documents/FullSearch/Elastic/ElasticDocumentFullSearch.cs @@ -58,6 +58,12 @@ namespace Volo.Docs.Documents.FullSearch.Elastic public virtual async Task AddOrUpdateAsync(Document document, CancellationToken cancellationToken = default) { var client = _clientProvider.GetClient(); + + var existsResponse = await client.DocumentExistsAsync(DocumentPath.Id(NormalizeField(document.Id)), x => x.Index(_options.IndexName), cancellationToken); + if (existsResponse.Exists) + { + await DeleteAsync(document.Id, cancellationToken); + } var esDocument = new EsDocument { From bfe399d1a24e2975fb3f3c6fe4c18c566e53063a Mon Sep 17 00:00:00 2001 From: Salih Date: Mon, 6 May 2024 12:03:11 +0300 Subject: [PATCH 4/5] Update ElasticDocumentFullSearch.cs --- .../Elastic/ElasticDocumentFullSearch.cs | 22 ++++++++++++++++--- 1 file changed, 19 insertions(+), 3 deletions(-) diff --git a/modules/docs/src/Volo.Docs.Domain/Volo/Docs/Documents/FullSearch/Elastic/ElasticDocumentFullSearch.cs b/modules/docs/src/Volo.Docs.Domain/Volo/Docs/Documents/FullSearch/Elastic/ElasticDocumentFullSearch.cs index 1bf5cb873f..c9a926d10d 100644 --- a/modules/docs/src/Volo.Docs.Domain/Volo/Docs/Documents/FullSearch/Elastic/ElasticDocumentFullSearch.cs +++ b/modules/docs/src/Volo.Docs.Domain/Volo/Docs/Documents/FullSearch/Elastic/ElasticDocumentFullSearch.cs @@ -59,10 +59,26 @@ namespace Volo.Docs.Documents.FullSearch.Elastic { var client = _clientProvider.GetClient(); - var existsResponse = await client.DocumentExistsAsync(DocumentPath.Id(NormalizeField(document.Id)), x => x.Index(_options.IndexName), cancellationToken); - if (existsResponse.Exists) + // exist by name, project id, language code and version + var existResponse = await client.SearchAsync(s => s + .Query(q => q + .Bool(b => b + .Must(m => m + .Term(t => t.ProjectId, NormalizeField(document.ProjectId)) + && m.Term(t => t.LanguageCode, NormalizeField(document.LanguageCode)) + && m.Term(t => t.Version, NormalizeField(document.Version)) + && m.Term(t => t.Name, document.Name) + ) + ) + ), cancellationToken); + + HandleError(existResponse); + + if (existResponse.Documents.Count != 0) { - await DeleteAsync(document.Id, cancellationToken); + // delete many + var ids = existResponse.Documents.Select(x => x.Id).ToList(); + HandleError(await client.DeleteManyAsync(ids, _options.IndexName, cancellationToken)); } var esDocument = new EsDocument From c4188c3a44d45d0590774e17d6440a579fd86d03 Mon Sep 17 00:00:00 2001 From: Salih Date: Mon, 6 May 2024 12:48:45 +0300 Subject: [PATCH 5/5] Update ElasticDocumentFullSearch.cs --- .../Documents/FullSearch/Elastic/ElasticDocumentFullSearch.cs | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/modules/docs/src/Volo.Docs.Domain/Volo/Docs/Documents/FullSearch/Elastic/ElasticDocumentFullSearch.cs b/modules/docs/src/Volo.Docs.Domain/Volo/Docs/Documents/FullSearch/Elastic/ElasticDocumentFullSearch.cs index c9a926d10d..22d4bf9ff4 100644 --- a/modules/docs/src/Volo.Docs.Domain/Volo/Docs/Documents/FullSearch/Elastic/ElasticDocumentFullSearch.cs +++ b/modules/docs/src/Volo.Docs.Domain/Volo/Docs/Documents/FullSearch/Elastic/ElasticDocumentFullSearch.cs @@ -76,9 +76,7 @@ namespace Volo.Docs.Documents.FullSearch.Elastic if (existResponse.Documents.Count != 0) { - // delete many - var ids = existResponse.Documents.Select(x => x.Id).ToList(); - HandleError(await client.DeleteManyAsync(ids, _options.IndexName, cancellationToken)); + HandleError(await client.DeleteManyAsync(existResponse.Documents, _options.IndexName, cancellationToken)); } var esDocument = new EsDocument