Merge pull request #1556 from colinin/es-index

feat: Optimize es query indexes

aspnet-core/framework/auditing/LINGYUN.Abp.AuditLogging.Elasticsearch/LINGYUN/Abp/AuditLogging/Elasticsearch/AuditLoggingIndexInitializer.cs

@@ -145,31 +145,15 @@ public class AuditLoggingIndexInitializer : IAuditLoggingIndexInitializer, ISing
             });
         }, cancellationToken);
 
-        if (!putTemplateResponse.IsValidResponse)
+        if (putTemplateResponse.TryGetErrorMessage(out var errorMessage))
         {
-            var errorBuilder = new StringBuilder();
-            if (putTemplateResponse.TryGetOriginalException(out var ex) && ex != null)
-            {
-                errorBuilder.AppendLine(ex.Message);
-                Logger.LogWarning(ex, "Failed to initialize index and audit log may not be retrieved.");
-                return;
-            }
-            else if (putTemplateResponse.TryGetElasticsearchServerError(out var error) && error != null)
-            {
-                errorBuilder.AppendLine(error.ToString());
-            }
-            else
-            {
-                errorBuilder.AppendLine(putTemplateResponse.DebugInformation);
-            }
-
             if (_elasticsearchOptions.ThrowIfIndexInitFailed)
             {
-                throw new AbpInitializationException($"Failed to initialize audit log index template, the error: {errorBuilder.ToString()}");
+                throw new AbpInitializationException($"Failed to initialize audit log index template, the error: {errorMessage}");
             }
 
             Logger.LogWarning("Failed to initialize index and audit log may not be retrieved.");
-            Logger.LogWarning("The error: {error}", errorBuilder.ToString());
+            Logger.LogWarning("The error: {error}", errorMessage);
         }
     }
 
@@ -217,31 +201,15 @@ public class AuditLoggingIndexInitializer : IAuditLoggingIndexInitializer, ISing
             });
         }, cancellationToken);
 
-        if (!putTemplateResponse.IsValidResponse)
+        if (putTemplateResponse.TryGetErrorMessage(out var errorMessage))
         {
-            var errorBuilder = new StringBuilder();
-            if (putTemplateResponse.TryGetOriginalException(out var ex) && ex != null)
-            {
-                errorBuilder.AppendLine(ex.Message);
-                Logger.LogWarning(ex, "Failed to initialize index and security log may not be retrieved.");
-                return;
-            }
-            else if (putTemplateResponse.TryGetElasticsearchServerError(out var error) && error != null)
-            {
-                errorBuilder.AppendLine(error.ToString());
-            }
-            else
-            {
-                errorBuilder.AppendLine(putTemplateResponse.DebugInformation);
-            }
-
             if (_elasticsearchOptions.ThrowIfIndexInitFailed)
             {
-                throw new AbpInitializationException($"Failed to initialize security log index template, the error: {errorBuilder.ToString()}");
+                throw new AbpInitializationException($"Failed to initialize security log index template, the error: {errorMessage}");
             }
 
             Logger.LogWarning("Failed to initialize index and security log may not be retrieved.");
-            Logger.LogWarning("The error: {error}", errorBuilder.ToString());
+            Logger.LogWarning("The error: {error}", errorMessage);
         }
     }
 }

aspnet-core/framework/auditing/LINGYUN.Abp.AuditLogging.Elasticsearch/LINGYUN/Abp/AuditLogging/Elasticsearch/ElasticsearchAuditLogManager.cs

@@ -145,6 +145,11 @@ public class ElasticsearchAuditLogManager : IAuditLogManager, ITransientDependen
                }),
             cancellationToken);
 
+        if (response.TryGetErrorMessage(out var errorMessage))
+        {
+            Logger.LogWarning("Query audit log count failed: {errorMessage}", errorMessage);
+        }
+
         return response.Count;
     }
 
@@ -407,8 +412,9 @@ public class ElasticsearchAuditLogManager : IAuditLogManager, ITransientDependen
             }
         }, cancellationToken);
 
-        if (!searchResponse.IsSuccess())
+        if (searchResponse.TryGetErrorMessage(out var errorMessage))
         {
+            Logger.LogWarning("Query audit log failed: {errorMessage}", errorMessage);
             return [];
         }
 
@@ -456,8 +462,9 @@ public class ElasticsearchAuditLogManager : IAuditLogManager, ITransientDependen
             }
         }, cancellationToken);
 
-        if (!searchResponse.IsSuccess())
+        if (searchResponse.TryGetErrorMessage(out var errorMessage))
         {
+            Logger.LogWarning("Query audit log failed: {errorMessage}", errorMessage);
             return [];
         }
 
@@ -514,15 +521,15 @@ public class ElasticsearchAuditLogManager : IAuditLogManager, ITransientDependen
             return null;
         }
 
-        var remaining = skipCount - 10000;
         // 获取skipCount最近一条数据作为searchAfter
         var secondResponse = await client.SearchAsync<AuditLog>(
             dsl => dsl.Indices(indexName)
                     .Query(query)
-                    .Sort(sorts)
+                    // 反转排序取第一个数据作为起始索引
+                    .Sort(sorts.Select(x => x).Reverse().ToArray())
                     .SourceIncludes(x => x.Id)
                     .SearchAfter(firstHit.Sort.ToList())
-                    .Size(remaining),
+                    .Size(1),
             cancellationToken);
 
         if (!secondResponse.IsSuccess() || secondResponse.Hits == null || !secondResponse.Hits.Any())
@@ -530,11 +537,6 @@ public class ElasticsearchAuditLogManager : IAuditLogManager, ITransientDependen
             return null;
         }
 
-        if (secondResponse.Hits.Count < remaining)
-        {
-            return null;
-        }
-
         var lastHit = secondResponse.Hits.LastOrDefault();
         if (lastHit?.Sort == null || !lastHit.Sort.Any())
         {
@@ -558,29 +560,26 @@ public class ElasticsearchAuditLogManager : IAuditLogManager, ITransientDependen
             : nameof(AuditLog.ExecutionTime);
 
         SortOptions[]? sorts = null;
-        if (sorting.IsNullOrWhiteSpace())
+        var sortingFieldMap = indexMappingInfo.Fields
-        {
-            var sortingFieldMap = indexMappingInfo.Fields
                     .Where(x => x.Key.Equals(sorting, StringComparison.CurrentCultureIgnoreCase))
                     .Select(x => x.Value)
                     .FirstOrDefault();
-            if (sortingFieldMap != null)
+        if (sortingFieldMap != null)
-            {
+        {
-                sorting = sortingFieldMap.Path;
+            sorting = sortingFieldMap.Path;
-            }
+        }
-            if (!sorting.IsNullOrWhiteSpace())
+        if (!sorting.IsNullOrWhiteSpace())
+        {
+            sorts = new SortOptions[1]
             {
-                sorts = new SortOptions[1]
+                new SortOptions
                 {
-                    new SortOptions
+                    Field = new FieldSort(new Field(sorting))
                     {
-                        Field = new FieldSort(new Field(sorting))
+                        Order = sortOrder,
-                        {
+                    },
-                            Order = sortOrder,
+                }
-                        },
+            };
-                    }
-                };
-            }
         }
 
         return sorts;

aspnet-core/framework/auditing/LINGYUN.Abp.AuditLogging.Elasticsearch/LINGYUN/Abp/AuditLogging/Elasticsearch/ElasticsearchAuditLogWriter.cs

@@ -1,6 +1,5 @@
 using Elastic.Clients.Elasticsearch;
 using Elastic.Clients.Elasticsearch.Core.Bulk;
-using Elastic.Transport.Products.Elasticsearch;
 using LINGYUN.Abp.Elasticsearch;
 using Microsoft.Extensions.Logging;
 using System;
@@ -44,17 +43,10 @@ public class ElasticsearchAuditLogWriter : IAuditLogWriter, ITransientDependency
                       .Id(auditLog.Id),
             cancellationToken);
 
-        if (!response.IsSuccess())
+        if (response.TryGetErrorMessage(out var errorMessage))
         {
             _logger.LogWarning("Could not save the audit log object: " + Environment.NewLine + auditLog.ToString());
-            if (response.TryGetOriginalException(out var ex) && ex != null)
+            _logger.LogWarning(errorMessage);
-            {
-                _logger.LogWarning(ex, ex.Message);
-            }
-            else if (response.ElasticsearchServerError != null)
-            {
-                _logger.LogWarning(response.ElasticsearchServerError.ToString());
-            }
             return "";
         }
 

aspnet-core/framework/auditing/LINGYUN.Abp.AuditLogging.Elasticsearch/LINGYUN/Abp/AuditLogging/Elasticsearch/ElasticsearchSecurityLogWriter.cs

@@ -1,5 +1,6 @@
 using Elastic.Clients.Elasticsearch;
 using Elastic.Clients.Elasticsearch.Core.Bulk;
+using Elastic.Clients.Elasticsearch.IndexManagement;
 using LINGYUN.Abp.Elasticsearch;
 using Microsoft.Extensions.Logging;
 using System;
@@ -48,17 +49,10 @@ public class ElasticsearchSecurityLogWriter : ISecurityLogWriter, ITransientDepe
                       .Id(securityLog.Id),
             cancellationToken);
 
-        if (!response.IsValidResponse)
+        if (response.TryGetErrorMessage(out var errorMessage))
         {
             _logger.LogWarning("Could not save the security log object: " + Environment.NewLine + securityLog.ToString());
-            if (response.TryGetOriginalException(out var ex) && ex != null)
+            _logger.LogWarning(errorMessage);
-            {
-                _logger.LogWarning(ex, ex.Message);
-            }
-            else if (response.ElasticsearchServerError != null)
-            {
-                _logger.LogWarning(response.ElasticsearchServerError.ToString());
-            }
         }
     }
 

aspnet-core/framework/elasticsearch/LINGYUN.Abp.Elasticsearch/Elastic/Clients/Elasticsearch/ElasticsearchResponseErrorExtensions.cs

@@ -0,0 +1,32 @@
+using Elastic.Transport.Products.Elasticsearch;
+using System.Text;
+
+namespace Elastic.Clients.Elasticsearch;
+
+public static class ElasticsearchResponseErrorExtensions
+{
+    public static bool TryGetErrorMessage(this ElasticsearchResponse response, out string? errorMessage)
+    {
+        if (!response.IsSuccess())
+        {
+            var errorBuilder = new StringBuilder();
+            if (response.TryGetOriginalException(out var ex) && ex != null)
+            {
+                errorBuilder.AppendLine(ex.Message);
+            }
+            if (response.TryGetElasticsearchServerError(out var error) && error != null)
+            {
+                errorBuilder.AppendLine(error.ToString());
+            }
+
+            if (errorBuilder.Length == 0)
+            {
+                errorBuilder.AppendLine(response.DebugInformation);
+            }
+            errorMessage = errorBuilder.ToString();
+            return true;
+        }
+        errorMessage = null;
+        return false;
+    }
+}

aspnet-core/framework/elasticsearch/LINGYUN.Abp.Elasticsearch/LINGYUN/Abp/Elasticsearch/ElasticsearchIndexMappingProvider.cs

@@ -1,13 +1,14 @@
 using Elastic.Clients.Elasticsearch;
+using Elastic.Clients.Elasticsearch.IndexManagement;
 using Elastic.Clients.Elasticsearch.Mapping;
-using Elastic.Transport.Products.Elasticsearch;
 using Microsoft.Extensions.Caching.Memory;
+using Microsoft.Extensions.FileSystemGlobbing.Internal;
 using System;
 using System.Collections.Generic;
 using System.Linq;
-using System.Text;
 using System.Threading;
 using System.Threading.Tasks;
+using Volo.Abp;
 using Volo.Abp.DependencyInjection;
 
 namespace LINGYUN.Abp.Elasticsearch;
@@ -26,40 +27,46 @@ public class ElasticsearchIndexMappingProvider : IIndexMappingProvider, ITransie
         _cache = cache;
     }
 
-    public async Task<IndexMappingInfo> GetMappingAsync(string indexName, CancellationToken cancellationToken = default)
+    public async Task<IndexMappingInfo> GetMappingAsync(string indexPattern, CancellationToken cancellationToken = default)
     {
-        var cacheKey = $"es_mapping_{indexName}";
+        var cacheKey = $"es_mapping_{indexPattern}";
 
         var cacheItem = _cache.Get<IndexMappingInfo>(cacheKey);
         if (cacheItem == null)
         {
             var client = _clientFactory.Create();
-            var response = await client.Indices.GetMappingAsync(indexName, cancellationToken);
+            var request = new GetMappingRequest(indexPattern)
+            {
+                IgnoreUnavailable = true,
+                AllowNoIndices = true,
+                ExpandWildcards = [ExpandWildcard.Open]
+            };
+            var response = await client.Indices.GetMappingAsync(request, cancellationToken);
 
-            if (!response.IsSuccess())
+            if (response.TryGetErrorMessage(out var errorMessage))
             {
-                var errorBuilder = new StringBuilder();
+                throw new AbpException($"Failed to get mapping for index {indexPattern}: {errorMessage}");
-                if (response.TryGetOriginalException(out var ex) && ex != null)
+            }
-                {
+
-                    errorBuilder.AppendLine(ex.Message);
+            var indexName = indexPattern.EndsWith("*") ? indexPattern.Substring(0, indexPattern.Length - 1) : indexPattern;
-                }
+            var indexMappings = response.GetMappingFor(indexName);
-                else if (response.TryGetElasticsearchServerError(out var error) && error != null)
+            if (indexMappings == null)
+            {
+                foreach (var indexMappingRecord in response.Mappings)
                 {
-                    errorBuilder.AppendLine(error.ToString());
+                    if (indexMappingRecord.Key.StartsWith(indexName))
+                    {
+                        indexMappings = indexMappingRecord.Value.Mappings;
+                        break;
+                    }
                 }
-                else
+                if (indexMappings == null)
                 {
-                    errorBuilder.AppendLine(response.DebugInformation);
+                    throw new AbpException($"Index {indexPattern} not found in response");
                 }
-                throw new Exception($"Failed to get mapping for index {indexName}: {errorBuilder.ToString()}");
-            }
-
-            if (!response.Mappings.TryGetValue(indexName, out var indexMappingRecord))
-            {
-                throw new Exception($"Index {indexName} not found in response");
             }
 
-            cacheItem = ParseMapping(indexMappingRecord.Mappings, indexName);
+            cacheItem = ParseMapping(indexMappings, indexPattern);
 
             _cache.Set(cacheKey, cacheItem, _cacheDuration);
         }

aspnet-core/framework/elasticsearch/LINGYUN.Abp.Elasticsearch/LINGYUN/Abp/Elasticsearch/ExpressionQueryService.cs

@@ -241,15 +241,15 @@ public class ExpressionQueryService : IExpressionQueryService, ITransientDepende
             return null;
         }
 
-        var remaining = skipCount - 10000;
         // 获取skipCount最近一条数据作为searchAfter
         var secondResponse = await client.SearchAsync<TDocument>(
             dsl => dsl.Indices(indexName)
                     .Query(query)
-                    .Sort(sorts)
+                    // 反转排序取第一个数据作为起始索引
+                    .Sort(sorts.Select(x => x).Reverse().ToArray())
                     .SourceIncludes([])
                     .SearchAfter(firstHit.Sort.ToList())
-                    .Size(remaining),
+                    .Size(1),
             cancellationToken);
 
         if (!secondResponse.IsSuccess() || secondResponse.Hits == null || !secondResponse.Hits.Any())
@@ -257,11 +257,6 @@ public class ExpressionQueryService : IExpressionQueryService, ITransientDepende
             return null;
         }
 
-        if (secondResponse.Hits.Count < remaining)
-        {
-            return null;
-        }
-
         var lastHit = secondResponse.Hits.LastOrDefault();
         if (lastHit?.Sort == null || !lastHit.Sort.Any())
         {

aspnet-core/framework/elasticsearch/LINGYUN.Abp.Elasticsearch/LINGYUN/Abp/Elasticsearch/IIndexMappingProvider.cs

@@ -5,5 +5,7 @@ namespace LINGYUN.Abp.Elasticsearch;
 
 public interface IIndexMappingProvider
 {
-    Task<IndexMappingInfo> GetMappingAsync(string indexName, CancellationToken cancellationToken = default);
+    Task<IndexMappingInfo> GetMappingAsync(
+        string indexPattern, 
+        CancellationToken cancellationToken = default);
 }

aspnet-core/framework/logging/LINGYUN.Abp.Logging.Serilog.Elasticsearch/LINGYUN/Abp/AuditLogging/Serilog/Elasticsearch/SerilogElasticsearchLoggingManager.cs

@@ -6,6 +6,7 @@ using Microsoft.Extensions.Logging;
 using Microsoft.Extensions.Logging.Abstractions;
 using Microsoft.Extensions.Options;
 using Serilog.Events;
+using Serilog.Formatting.Elasticsearch;
 using System;
 using System.Collections.Generic;
 using System.Linq;
@@ -195,6 +196,10 @@ public class SerilogElasticsearchLoggingManager : ILoggingManager, ISingletonDep
                                         (t) => t.Field(GetField(indexMapping, "fields.UniqueId")).Value(id)))))
                        .Size(1),
                 cancellationToken);
+            if (response.TryGetErrorMessage(out var errorMessage))
+            {
+                Logger.LogWarning("Query logs failed: {errorMessage}", errorMessage);
+            }
         }
 
         return _objectMapper.Map<SerilogInfo?, LogInfo?>(response.Documents.FirstOrDefault());
@@ -240,6 +245,10 @@ public class SerilogElasticsearchLoggingManager : ILoggingManager, ISingletonDep
             dsl.Indices(indexName)
                .Query(log => log.Bool(b => b.Must(querys.ToArray()))),
             cancellationToken);
+        if (response.TryGetErrorMessage(out var errorMessage))
+        {
+            Logger.LogWarning("Query log count failed: {errorMessage}", errorMessage);
+        }
 
         return response.Count;
     }
@@ -469,8 +478,9 @@ public class SerilogElasticsearchLoggingManager : ILoggingManager, ISingletonDep
             }
         }, cancellationToken);
 
-        if (!searchResponse.IsSuccess())
+        if (searchResponse.TryGetErrorMessage(out var errorMessage))
         {
+            Logger.LogWarning("Query log failed: {errorMessage}", errorMessage);
             return [];
         }
 
@@ -506,8 +516,9 @@ public class SerilogElasticsearchLoggingManager : ILoggingManager, ISingletonDep
                 .SearchAfter(searchAfter);
         }, cancellationToken);
 
-        if (!searchResponse.IsSuccess())
+        if (searchResponse.TryGetErrorMessage(out var errorMessage))
         {
+            Logger.LogWarning("Query log failed: {errorMessage}", errorMessage);
             return [];
         }
 
@@ -563,15 +574,15 @@ public class SerilogElasticsearchLoggingManager : ILoggingManager, ISingletonDep
             return null;
         }
 
-        var remaining = skipCount - 10000;
         // 获取skipCount最近一条数据作为searchAfter
         var secondResponse = await client.SearchAsync<SerilogInfo>(
             dsl => dsl.Indices(CreateIndex())
                     .Query(query)
-                    .Sort(sorts)
+                    // 反转排序取第一个数据作为起始索引
+                    .Sort(sorts.Select(x => x).Reverse().ToArray())
                     .SourceIncludes(x => x.Level)
                     .SearchAfter(firstHit.Sort.ToList())
-                    .Size(remaining),
+                    .Size(1),
             cancellationToken);
 
         if (!secondResponse.IsSuccess() || secondResponse.Hits == null || !secondResponse.Hits.Any())
@@ -579,11 +590,6 @@ public class SerilogElasticsearchLoggingManager : ILoggingManager, ISingletonDep
             return null;
         }
 
-        if (secondResponse.Hits.Count < remaining)
-        {
-            return null;
-        }
-
         var lastHit = secondResponse.Hits.LastOrDefault();
         if (lastHit?.Sort == null || !lastHit.Sort.Any())
         {
@@ -621,32 +627,29 @@ public class SerilogElasticsearchLoggingManager : ILoggingManager, ISingletonDep
             ? SortOrder.Asc : SortOrder.Desc;
         sorting = !sorting.IsNullOrWhiteSpace()
             ? sorting.Split()[0]
-            : nameof(SerilogInfo.TimeStamp);
+            : ElasticsearchJsonFormatter.TimestampPropertyName;
 
         SortOptions[]? sorts = null;
-        if (sorting.IsNullOrWhiteSpace())
+        var sortingFieldMap = indexMappingInfo.Fields
+            .Where(x => x.Key.Equals(sorting, StringComparison.CurrentCultureIgnoreCase))
+            .Select(x => x.Value)
+            .FirstOrDefault();
+        if (sortingFieldMap != null)
         {
-            var sortingFieldMap = indexMappingInfo.Fields
+            sorting = sortingFieldMap.Path;
-                    .Where(x => x.Key.Equals(sorting, StringComparison.CurrentCultureIgnoreCase))
+        }
-                    .Select(x => x.Value)
+        if (!sorting.IsNullOrWhiteSpace())
-                    .FirstOrDefault();
+        {
-            if (sortingFieldMap != null)
+            sorts = new SortOptions[1]
-            {
-                sorting = sortingFieldMap.Path;
-            }
-            if (!sorting.IsNullOrWhiteSpace())
             {
-                sorts = new SortOptions[1]
+                new SortOptions
                 {
-                    new SortOptions
+                    Field = new FieldSort(new Field(sorting))
                     {
-                        Field = new FieldSort(new Field(sorting))
+                        Order = sortOrder,
-                        {
+                    },
-                            Order = sortOrder,
+                }
-                        },
+            };
-                    }
-                };
-            }
         }
 
         return sorts;