Merge pull request #19701 from abpframework/salihozkara/fixDocsReIndexProblem

Fix reindex problem

modules/docs/src/Volo.Docs.Admin.Application/Volo/Docs/Admin/Projects/ProjectAdminAppService.cs

@@ -134,15 +134,29 @@ namespace Volo.Docs.Admin.Projects
             {
                 throw new Exception("Cannot find the project with the Id " + projectId);
             }
-
-            var docs = (await _documentRepository.GetListByProjectId(project.Id))
-                .Where(doc => doc.FileName != project.NavigationDocumentName && doc.FileName != project.ParametersDocumentName)
-                .ToList();
+            
             await _elasticSearchService.DeleteAllByProjectIdAsync(project.Id);
-
-            if(docs.Any())
+            
+            var docsCount = await _documentRepository.GetUniqueDocumentCountByProjectIdAsync(projectId);
+            
+            if (docsCount == 0)
+            {
+                return;
+            }
+            
+            const int maxResultCount = 1000;
+            
+            var skipCount = 0;
+            while(skipCount < docsCount)
             {
-                await _elasticSearchService.AddOrUpdateManyAsync(docs);    
+                var docs = await _documentRepository.GetUniqueDocumentsByProjectIdPagedAsync(projectId, skipCount, maxResultCount);
+                docs = docs.Where(doc => doc.FileName != project.NavigationDocumentName && doc.FileName != project.ParametersDocumentName).ToList();
+                if (!docs.Any())
+                {
+                    return;
+                }
+                await _elasticSearchService.AddOrUpdateManyAsync(docs);
+                skipCount += maxResultCount;
             }
         }
 

modules/docs/src/Volo.Docs.Domain/Volo/Docs/Documents/FullSearch/Elastic/ElasticDocumentFullSearch.cs

@@ -58,6 +58,26 @@ namespace Volo.Docs.Documents.FullSearch.Elastic
         public virtual async Task AddOrUpdateAsync(Document document, CancellationToken cancellationToken = default)
         {
             var client = _clientProvider.GetClient();
+            
+            // exist by name, project id, language code and version
+            var existResponse = await client.SearchAsync<EsDocument>(s => s
+                .Query(q => q
+                    .Bool(b => b
+                        .Must(m => m
+                            .Term(t => t.ProjectId, NormalizeField(document.ProjectId))
+                            && m.Term(t => t.LanguageCode, NormalizeField(document.LanguageCode))
+                            && m.Term(t => t.Version, NormalizeField(document.Version))
+                            && m.Term(t => t.Name, document.Name)
+                        )
+                    )
+                ), cancellationToken);
+            
+            HandleError(existResponse);
+            
+            if (existResponse.Documents.Count != 0)
+            {
+                HandleError(await client.DeleteManyAsync(existResponse.Documents, _options.IndexName, cancellationToken));
+            }
 
             var esDocument = new EsDocument
             {

modules/docs/src/Volo.Docs.Domain/Volo/Docs/Documents/IDocumentRepository.cs

@@ -13,6 +13,10 @@ namespace Volo.Docs.Documents
 
         Task<List<Document>> GetListByProjectId(Guid projectId, CancellationToken cancellationToken = default);
         
+        Task<List<Document>> GetUniqueDocumentsByProjectIdPagedAsync(Guid projectId, int skipCount, int maxResultCount, CancellationToken cancellationToken = default);
+        
+        Task<long> GetUniqueDocumentCountByProjectIdAsync(Guid projectId, CancellationToken cancellationToken = default);
+        
         Task UpdateProjectLastCachedTimeAsync(Guid projectId, DateTime cachedTime,
             CancellationToken cancellationToken = default);
 

modules/docs/src/Volo.Docs.EntityFrameworkCore/Volo/Docs/Documents/EFCoreDocumentRepository.cs

@@ -56,6 +56,27 @@ namespace Volo.Docs.Documents
             return await (await GetDbSetAsync()).Where(d => d.ProjectId == projectId).ToListAsync(GetCancellationToken(cancellationToken));
         }
 
+        public virtual async Task<List<Document>> GetUniqueDocumentsByProjectIdPagedAsync(Guid projectId, int skipCount, int maxResultCount,
+            CancellationToken cancellationToken = default)
+        {
+            return await (await GetDbSetAsync())
+                .Where(d => d.ProjectId == projectId)
+                .OrderBy(x => x.LastCachedTime)
+                .GroupBy(x => new { x.Name, x.LanguageCode, x.Version })
+                .Select(group => group.First())
+                .Skip(skipCount)
+                .Take(maxResultCount)
+                .ToListAsync(cancellationToken);
+        }
+
+        public virtual async Task<long> GetUniqueDocumentCountByProjectIdAsync(Guid projectId, CancellationToken cancellationToken = default)
+        {
+            return await (await GetDbSetAsync())
+                .Where(d => d.ProjectId == projectId)
+                .GroupBy(x => new {x.FileName, x.Version, x.LanguageCode})
+                .LongCountAsync(GetCancellationToken(cancellationToken));
+        }
+
         public async Task UpdateProjectLastCachedTimeAsync(Guid projectId, DateTime cachedTime,
             CancellationToken cancellationToken = default)
         {

modules/docs/src/Volo.Docs.MongoDB/Volo/Docs/Documents/MongoDocumentRepository.cs

@@ -54,6 +54,26 @@ namespace Volo.Docs.Documents
             return await (await GetMongoQueryableAsync(cancellationToken)).Where(d => d.ProjectId == projectId).ToListAsync(GetCancellationToken(cancellationToken));
         }
 
+        public virtual async Task<List<Document>> GetUniqueDocumentsByProjectIdPagedAsync(Guid projectId, int skipCount, int maxResultCount,
+            CancellationToken cancellationToken = default)
+        {
+            return await (await GetMongoQueryableAsync(cancellationToken))
+                .Where(d => d.ProjectId == projectId)
+                .OrderBy(x => x.LastCachedTime)
+                .GroupBy(x => new { x.Name, x.LanguageCode, x.Version })
+                .Select(group => group.First())
+                .Skip(skipCount)
+                .Take(maxResultCount)
+                .ToListAsync(cancellationToken);
+        }
+
+        public virtual async Task<long> GetUniqueDocumentCountByProjectIdAsync(Guid projectId, CancellationToken cancellationToken = default)
+        {
+            return await (await GetMongoQueryableAsync(cancellationToken)).Where(d => d.ProjectId == projectId)
+                .GroupBy(x => new { x.Name, x.LanguageCode, x.Version })
+                .LongCountAsync(GetCancellationToken(cancellationToken));
+        }
+
         public async Task UpdateProjectLastCachedTimeAsync(Guid projectId, DateTime cachedTime,
             CancellationToken cancellationToken = default)
         {

modules/docs/test/Volo.Docs.TestBase/Volo/Docs/DocumentRepository_Tests.cs

@@ -43,5 +43,19 @@ namespace Volo.Docs
             var documentsAfterClear = await DocumentRepository.GetListByProjectId(DocsTestData.ProjectId);
             documentsAfterClear.ForEach(d => d.LastCachedTime.ShouldBe(DateTime.MinValue));
         }
+        
+        [Fact] 
+        public async Task GetUniqueDocumentsByProjectIdPagedAsync()
+        {
+            var documents = await DocumentRepository.GetUniqueDocumentsByProjectIdPagedAsync(DocsTestData.ProjectId, 0, 10);
+            documents.Count.ShouldBe(1);
+        }
+        
+        [Fact]
+        public async Task GetUniqueDocumentCountByProjectIdAsync()
+        {
+            var count = await DocumentRepository.GetUniqueDocumentCountByProjectIdAsync(DocsTestData.ProjectId);
+            count.ShouldBe(1);
+        }
     }
 }