diff --git a/aspnet-core/framework/auditing/LINGYUN.Abp.AuditLogging.Elasticsearch/LINGYUN/Abp/AuditLogging/Elasticsearch/AuditLoggingIndexInitializer.cs b/aspnet-core/framework/auditing/LINGYUN.Abp.AuditLogging.Elasticsearch/LINGYUN/Abp/AuditLogging/Elasticsearch/AuditLoggingIndexInitializer.cs index 16c1c2aea..894418898 100644 --- a/aspnet-core/framework/auditing/LINGYUN.Abp.AuditLogging.Elasticsearch/LINGYUN/Abp/AuditLogging/Elasticsearch/AuditLoggingIndexInitializer.cs +++ b/aspnet-core/framework/auditing/LINGYUN.Abp.AuditLogging.Elasticsearch/LINGYUN/Abp/AuditLogging/Elasticsearch/AuditLoggingIndexInitializer.cs @@ -145,31 +145,15 @@ public class AuditLoggingIndexInitializer : IAuditLoggingIndexInitializer, ISing }); }, cancellationToken); - if (!putTemplateResponse.IsValidResponse) + if (putTemplateResponse.TryGetErrorMessage(out var errorMessage)) { - var errorBuilder = new StringBuilder(); - if (putTemplateResponse.TryGetOriginalException(out var ex) && ex != null) - { - errorBuilder.AppendLine(ex.Message); - Logger.LogWarning(ex, "Failed to initialize index and audit log may not be retrieved."); - return; - } - else if (putTemplateResponse.TryGetElasticsearchServerError(out var error) && error != null) - { - errorBuilder.AppendLine(error.ToString()); - } - else - { - errorBuilder.AppendLine(putTemplateResponse.DebugInformation); - } - if (_elasticsearchOptions.ThrowIfIndexInitFailed) { - throw new AbpInitializationException($"Failed to initialize audit log index template, the error: {errorBuilder.ToString()}"); + throw new AbpInitializationException($"Failed to initialize audit log index template, the error: {errorMessage}"); } Logger.LogWarning("Failed to initialize index and audit log may not be retrieved."); - Logger.LogWarning("The error: {error}", errorBuilder.ToString()); + Logger.LogWarning("The error: {error}", errorMessage); } } @@ -217,31 +201,15 @@ public class AuditLoggingIndexInitializer : IAuditLoggingIndexInitializer, ISing }); }, cancellationToken); - if (!putTemplateResponse.IsValidResponse) + if (putTemplateResponse.TryGetErrorMessage(out var errorMessage)) { - var errorBuilder = new StringBuilder(); - if (putTemplateResponse.TryGetOriginalException(out var ex) && ex != null) - { - errorBuilder.AppendLine(ex.Message); - Logger.LogWarning(ex, "Failed to initialize index and security log may not be retrieved."); - return; - } - else if (putTemplateResponse.TryGetElasticsearchServerError(out var error) && error != null) - { - errorBuilder.AppendLine(error.ToString()); - } - else - { - errorBuilder.AppendLine(putTemplateResponse.DebugInformation); - } - if (_elasticsearchOptions.ThrowIfIndexInitFailed) { - throw new AbpInitializationException($"Failed to initialize security log index template, the error: {errorBuilder.ToString()}"); + throw new AbpInitializationException($"Failed to initialize security log index template, the error: {errorMessage}"); } Logger.LogWarning("Failed to initialize index and security log may not be retrieved."); - Logger.LogWarning("The error: {error}", errorBuilder.ToString()); + Logger.LogWarning("The error: {error}", errorMessage); } } } diff --git a/aspnet-core/framework/auditing/LINGYUN.Abp.AuditLogging.Elasticsearch/LINGYUN/Abp/AuditLogging/Elasticsearch/ElasticsearchAuditLogManager.cs b/aspnet-core/framework/auditing/LINGYUN.Abp.AuditLogging.Elasticsearch/LINGYUN/Abp/AuditLogging/Elasticsearch/ElasticsearchAuditLogManager.cs index 1cbff1dfd..22e037522 100644 --- a/aspnet-core/framework/auditing/LINGYUN.Abp.AuditLogging.Elasticsearch/LINGYUN/Abp/AuditLogging/Elasticsearch/ElasticsearchAuditLogManager.cs +++ b/aspnet-core/framework/auditing/LINGYUN.Abp.AuditLogging.Elasticsearch/LINGYUN/Abp/AuditLogging/Elasticsearch/ElasticsearchAuditLogManager.cs @@ -145,6 +145,11 @@ public class ElasticsearchAuditLogManager : IAuditLogManager, ITransientDependen }), cancellationToken); + if (response.TryGetErrorMessage(out var errorMessage)) + { + Logger.LogWarning("Query audit log count failed: {errorMessage}", errorMessage); + } + return response.Count; } @@ -407,8 +412,9 @@ public class ElasticsearchAuditLogManager : IAuditLogManager, ITransientDependen } }, cancellationToken); - if (!searchResponse.IsSuccess()) + if (searchResponse.TryGetErrorMessage(out var errorMessage)) { + Logger.LogWarning("Query audit log failed: {errorMessage}", errorMessage); return []; } @@ -456,8 +462,9 @@ public class ElasticsearchAuditLogManager : IAuditLogManager, ITransientDependen } }, cancellationToken); - if (!searchResponse.IsSuccess()) + if (searchResponse.TryGetErrorMessage(out var errorMessage)) { + Logger.LogWarning("Query audit log failed: {errorMessage}", errorMessage); return []; } @@ -514,15 +521,15 @@ public class ElasticsearchAuditLogManager : IAuditLogManager, ITransientDependen return null; } - var remaining = skipCount - 10000; // 获取skipCount最近一条数据作为searchAfter var secondResponse = await client.SearchAsync( dsl => dsl.Indices(indexName) .Query(query) - .Sort(sorts) + // 反转排序取第一个数据作为起始索引 + .Sort(sorts.Select(x => x).Reverse().ToArray()) .SourceIncludes(x => x.Id) .SearchAfter(firstHit.Sort.ToList()) - .Size(remaining), + .Size(1), cancellationToken); if (!secondResponse.IsSuccess() || secondResponse.Hits == null || !secondResponse.Hits.Any()) @@ -530,11 +537,6 @@ public class ElasticsearchAuditLogManager : IAuditLogManager, ITransientDependen return null; } - if (secondResponse.Hits.Count < remaining) - { - return null; - } - var lastHit = secondResponse.Hits.LastOrDefault(); if (lastHit?.Sort == null || !lastHit.Sort.Any()) { @@ -558,29 +560,26 @@ public class ElasticsearchAuditLogManager : IAuditLogManager, ITransientDependen : nameof(AuditLog.ExecutionTime); SortOptions[]? sorts = null; - if (sorting.IsNullOrWhiteSpace()) - { - var sortingFieldMap = indexMappingInfo.Fields + var sortingFieldMap = indexMappingInfo.Fields .Where(x => x.Key.Equals(sorting, StringComparison.CurrentCultureIgnoreCase)) .Select(x => x.Value) .FirstOrDefault(); - if (sortingFieldMap != null) - { - sorting = sortingFieldMap.Path; - } - if (!sorting.IsNullOrWhiteSpace()) + if (sortingFieldMap != null) + { + sorting = sortingFieldMap.Path; + } + if (!sorting.IsNullOrWhiteSpace()) + { + sorts = new SortOptions[1] { - sorts = new SortOptions[1] + new SortOptions { - new SortOptions + Field = new FieldSort(new Field(sorting)) { - Field = new FieldSort(new Field(sorting)) - { - Order = sortOrder, - }, - } - }; - } + Order = sortOrder, + }, + } + }; } return sorts; diff --git a/aspnet-core/framework/auditing/LINGYUN.Abp.AuditLogging.Elasticsearch/LINGYUN/Abp/AuditLogging/Elasticsearch/ElasticsearchAuditLogWriter.cs b/aspnet-core/framework/auditing/LINGYUN.Abp.AuditLogging.Elasticsearch/LINGYUN/Abp/AuditLogging/Elasticsearch/ElasticsearchAuditLogWriter.cs index 7a86cb146..f0063da02 100644 --- a/aspnet-core/framework/auditing/LINGYUN.Abp.AuditLogging.Elasticsearch/LINGYUN/Abp/AuditLogging/Elasticsearch/ElasticsearchAuditLogWriter.cs +++ b/aspnet-core/framework/auditing/LINGYUN.Abp.AuditLogging.Elasticsearch/LINGYUN/Abp/AuditLogging/Elasticsearch/ElasticsearchAuditLogWriter.cs @@ -1,6 +1,5 @@ using Elastic.Clients.Elasticsearch; using Elastic.Clients.Elasticsearch.Core.Bulk; -using Elastic.Transport.Products.Elasticsearch; using LINGYUN.Abp.Elasticsearch; using Microsoft.Extensions.Logging; using System; @@ -44,17 +43,10 @@ public class ElasticsearchAuditLogWriter : IAuditLogWriter, ITransientDependency .Id(auditLog.Id), cancellationToken); - if (!response.IsSuccess()) + if (response.TryGetErrorMessage(out var errorMessage)) { _logger.LogWarning("Could not save the audit log object: " + Environment.NewLine + auditLog.ToString()); - if (response.TryGetOriginalException(out var ex) && ex != null) - { - _logger.LogWarning(ex, ex.Message); - } - else if (response.ElasticsearchServerError != null) - { - _logger.LogWarning(response.ElasticsearchServerError.ToString()); - } + _logger.LogWarning(errorMessage); return ""; } diff --git a/aspnet-core/framework/auditing/LINGYUN.Abp.AuditLogging.Elasticsearch/LINGYUN/Abp/AuditLogging/Elasticsearch/ElasticsearchSecurityLogWriter.cs b/aspnet-core/framework/auditing/LINGYUN.Abp.AuditLogging.Elasticsearch/LINGYUN/Abp/AuditLogging/Elasticsearch/ElasticsearchSecurityLogWriter.cs index fc087a511..a7fe96781 100644 --- a/aspnet-core/framework/auditing/LINGYUN.Abp.AuditLogging.Elasticsearch/LINGYUN/Abp/AuditLogging/Elasticsearch/ElasticsearchSecurityLogWriter.cs +++ b/aspnet-core/framework/auditing/LINGYUN.Abp.AuditLogging.Elasticsearch/LINGYUN/Abp/AuditLogging/Elasticsearch/ElasticsearchSecurityLogWriter.cs @@ -1,5 +1,6 @@ using Elastic.Clients.Elasticsearch; using Elastic.Clients.Elasticsearch.Core.Bulk; +using Elastic.Clients.Elasticsearch.IndexManagement; using LINGYUN.Abp.Elasticsearch; using Microsoft.Extensions.Logging; using System; @@ -48,17 +49,10 @@ public class ElasticsearchSecurityLogWriter : ISecurityLogWriter, ITransientDepe .Id(securityLog.Id), cancellationToken); - if (!response.IsValidResponse) + if (response.TryGetErrorMessage(out var errorMessage)) { _logger.LogWarning("Could not save the security log object: " + Environment.NewLine + securityLog.ToString()); - if (response.TryGetOriginalException(out var ex) && ex != null) - { - _logger.LogWarning(ex, ex.Message); - } - else if (response.ElasticsearchServerError != null) - { - _logger.LogWarning(response.ElasticsearchServerError.ToString()); - } + _logger.LogWarning(errorMessage); } } diff --git a/aspnet-core/framework/elasticsearch/LINGYUN.Abp.Elasticsearch/Elastic/Clients/Elasticsearch/ElasticsearchResponseErrorExtensions.cs b/aspnet-core/framework/elasticsearch/LINGYUN.Abp.Elasticsearch/Elastic/Clients/Elasticsearch/ElasticsearchResponseErrorExtensions.cs new file mode 100644 index 000000000..7c5550408 --- /dev/null +++ b/aspnet-core/framework/elasticsearch/LINGYUN.Abp.Elasticsearch/Elastic/Clients/Elasticsearch/ElasticsearchResponseErrorExtensions.cs @@ -0,0 +1,32 @@ +using Elastic.Transport.Products.Elasticsearch; +using System.Text; + +namespace Elastic.Clients.Elasticsearch; + +public static class ElasticsearchResponseErrorExtensions +{ + public static bool TryGetErrorMessage(this ElasticsearchResponse response, out string? errorMessage) + { + if (!response.IsSuccess()) + { + var errorBuilder = new StringBuilder(); + if (response.TryGetOriginalException(out var ex) && ex != null) + { + errorBuilder.AppendLine(ex.Message); + } + if (response.TryGetElasticsearchServerError(out var error) && error != null) + { + errorBuilder.AppendLine(error.ToString()); + } + + if (errorBuilder.Length == 0) + { + errorBuilder.AppendLine(response.DebugInformation); + } + errorMessage = errorBuilder.ToString(); + return true; + } + errorMessage = null; + return false; + } +} diff --git a/aspnet-core/framework/elasticsearch/LINGYUN.Abp.Elasticsearch/LINGYUN/Abp/Elasticsearch/ElasticsearchIndexMappingProvider.cs b/aspnet-core/framework/elasticsearch/LINGYUN.Abp.Elasticsearch/LINGYUN/Abp/Elasticsearch/ElasticsearchIndexMappingProvider.cs index 38000d182..c3e32e898 100644 --- a/aspnet-core/framework/elasticsearch/LINGYUN.Abp.Elasticsearch/LINGYUN/Abp/Elasticsearch/ElasticsearchIndexMappingProvider.cs +++ b/aspnet-core/framework/elasticsearch/LINGYUN.Abp.Elasticsearch/LINGYUN/Abp/Elasticsearch/ElasticsearchIndexMappingProvider.cs @@ -1,13 +1,14 @@ using Elastic.Clients.Elasticsearch; +using Elastic.Clients.Elasticsearch.IndexManagement; using Elastic.Clients.Elasticsearch.Mapping; -using Elastic.Transport.Products.Elasticsearch; using Microsoft.Extensions.Caching.Memory; +using Microsoft.Extensions.FileSystemGlobbing.Internal; using System; using System.Collections.Generic; using System.Linq; -using System.Text; using System.Threading; using System.Threading.Tasks; +using Volo.Abp; using Volo.Abp.DependencyInjection; namespace LINGYUN.Abp.Elasticsearch; @@ -26,40 +27,46 @@ public class ElasticsearchIndexMappingProvider : IIndexMappingProvider, ITransie _cache = cache; } - public async Task GetMappingAsync(string indexName, CancellationToken cancellationToken = default) + public async Task GetMappingAsync(string indexPattern, CancellationToken cancellationToken = default) { - var cacheKey = $"es_mapping_{indexName}"; + var cacheKey = $"es_mapping_{indexPattern}"; var cacheItem = _cache.Get(cacheKey); if (cacheItem == null) { var client = _clientFactory.Create(); - var response = await client.Indices.GetMappingAsync(indexName, cancellationToken); + var request = new GetMappingRequest(indexPattern) + { + IgnoreUnavailable = true, + AllowNoIndices = true, + ExpandWildcards = [ExpandWildcard.Open] + }; + var response = await client.Indices.GetMappingAsync(request, cancellationToken); - if (!response.IsSuccess()) + if (response.TryGetErrorMessage(out var errorMessage)) { - var errorBuilder = new StringBuilder(); - if (response.TryGetOriginalException(out var ex) && ex != null) - { - errorBuilder.AppendLine(ex.Message); - } - else if (response.TryGetElasticsearchServerError(out var error) && error != null) + throw new AbpException($"Failed to get mapping for index {indexPattern}: {errorMessage}"); + } + + var indexName = indexPattern.EndsWith("*") ? indexPattern.Substring(0, indexPattern.Length - 1) : indexPattern; + var indexMappings = response.GetMappingFor(indexName); + if (indexMappings == null) + { + foreach (var indexMappingRecord in response.Mappings) { - errorBuilder.AppendLine(error.ToString()); + if (indexMappingRecord.Key.StartsWith(indexName)) + { + indexMappings = indexMappingRecord.Value.Mappings; + break; + } } - else + if (indexMappings == null) { - errorBuilder.AppendLine(response.DebugInformation); + throw new AbpException($"Index {indexPattern} not found in response"); } - throw new Exception($"Failed to get mapping for index {indexName}: {errorBuilder.ToString()}"); - } - - if (!response.Mappings.TryGetValue(indexName, out var indexMappingRecord)) - { - throw new Exception($"Index {indexName} not found in response"); } - cacheItem = ParseMapping(indexMappingRecord.Mappings, indexName); + cacheItem = ParseMapping(indexMappings, indexPattern); _cache.Set(cacheKey, cacheItem, _cacheDuration); } diff --git a/aspnet-core/framework/elasticsearch/LINGYUN.Abp.Elasticsearch/LINGYUN/Abp/Elasticsearch/ExpressionQueryService.cs b/aspnet-core/framework/elasticsearch/LINGYUN.Abp.Elasticsearch/LINGYUN/Abp/Elasticsearch/ExpressionQueryService.cs index e51b09fa8..afef44b90 100644 --- a/aspnet-core/framework/elasticsearch/LINGYUN.Abp.Elasticsearch/LINGYUN/Abp/Elasticsearch/ExpressionQueryService.cs +++ b/aspnet-core/framework/elasticsearch/LINGYUN.Abp.Elasticsearch/LINGYUN/Abp/Elasticsearch/ExpressionQueryService.cs @@ -241,15 +241,15 @@ public class ExpressionQueryService : IExpressionQueryService, ITransientDepende return null; } - var remaining = skipCount - 10000; // 获取skipCount最近一条数据作为searchAfter var secondResponse = await client.SearchAsync( dsl => dsl.Indices(indexName) .Query(query) - .Sort(sorts) + // 反转排序取第一个数据作为起始索引 + .Sort(sorts.Select(x => x).Reverse().ToArray()) .SourceIncludes([]) .SearchAfter(firstHit.Sort.ToList()) - .Size(remaining), + .Size(1), cancellationToken); if (!secondResponse.IsSuccess() || secondResponse.Hits == null || !secondResponse.Hits.Any()) @@ -257,11 +257,6 @@ public class ExpressionQueryService : IExpressionQueryService, ITransientDepende return null; } - if (secondResponse.Hits.Count < remaining) - { - return null; - } - var lastHit = secondResponse.Hits.LastOrDefault(); if (lastHit?.Sort == null || !lastHit.Sort.Any()) { diff --git a/aspnet-core/framework/elasticsearch/LINGYUN.Abp.Elasticsearch/LINGYUN/Abp/Elasticsearch/IIndexMappingProvider.cs b/aspnet-core/framework/elasticsearch/LINGYUN.Abp.Elasticsearch/LINGYUN/Abp/Elasticsearch/IIndexMappingProvider.cs index fbbfbd86b..0a017cde2 100644 --- a/aspnet-core/framework/elasticsearch/LINGYUN.Abp.Elasticsearch/LINGYUN/Abp/Elasticsearch/IIndexMappingProvider.cs +++ b/aspnet-core/framework/elasticsearch/LINGYUN.Abp.Elasticsearch/LINGYUN/Abp/Elasticsearch/IIndexMappingProvider.cs @@ -5,5 +5,7 @@ namespace LINGYUN.Abp.Elasticsearch; public interface IIndexMappingProvider { - Task GetMappingAsync(string indexName, CancellationToken cancellationToken = default); + Task GetMappingAsync( + string indexPattern, + CancellationToken cancellationToken = default); } diff --git a/aspnet-core/framework/logging/LINGYUN.Abp.Logging.Serilog.Elasticsearch/LINGYUN/Abp/AuditLogging/Serilog/Elasticsearch/SerilogElasticsearchLoggingManager.cs b/aspnet-core/framework/logging/LINGYUN.Abp.Logging.Serilog.Elasticsearch/LINGYUN/Abp/AuditLogging/Serilog/Elasticsearch/SerilogElasticsearchLoggingManager.cs index c1b1005d0..984c0caac 100644 --- a/aspnet-core/framework/logging/LINGYUN.Abp.Logging.Serilog.Elasticsearch/LINGYUN/Abp/AuditLogging/Serilog/Elasticsearch/SerilogElasticsearchLoggingManager.cs +++ b/aspnet-core/framework/logging/LINGYUN.Abp.Logging.Serilog.Elasticsearch/LINGYUN/Abp/AuditLogging/Serilog/Elasticsearch/SerilogElasticsearchLoggingManager.cs @@ -6,6 +6,7 @@ using Microsoft.Extensions.Logging; using Microsoft.Extensions.Logging.Abstractions; using Microsoft.Extensions.Options; using Serilog.Events; +using Serilog.Formatting.Elasticsearch; using System; using System.Collections.Generic; using System.Linq; @@ -195,6 +196,10 @@ public class SerilogElasticsearchLoggingManager : ILoggingManager, ISingletonDep (t) => t.Field(GetField(indexMapping, "fields.UniqueId")).Value(id))))) .Size(1), cancellationToken); + if (response.TryGetErrorMessage(out var errorMessage)) + { + Logger.LogWarning("Query logs failed: {errorMessage}", errorMessage); + } } return _objectMapper.Map(response.Documents.FirstOrDefault()); @@ -240,6 +245,10 @@ public class SerilogElasticsearchLoggingManager : ILoggingManager, ISingletonDep dsl.Indices(indexName) .Query(log => log.Bool(b => b.Must(querys.ToArray()))), cancellationToken); + if (response.TryGetErrorMessage(out var errorMessage)) + { + Logger.LogWarning("Query log count failed: {errorMessage}", errorMessage); + } return response.Count; } @@ -469,8 +478,9 @@ public class SerilogElasticsearchLoggingManager : ILoggingManager, ISingletonDep } }, cancellationToken); - if (!searchResponse.IsSuccess()) + if (searchResponse.TryGetErrorMessage(out var errorMessage)) { + Logger.LogWarning("Query log failed: {errorMessage}", errorMessage); return []; } @@ -506,8 +516,9 @@ public class SerilogElasticsearchLoggingManager : ILoggingManager, ISingletonDep .SearchAfter(searchAfter); }, cancellationToken); - if (!searchResponse.IsSuccess()) + if (searchResponse.TryGetErrorMessage(out var errorMessage)) { + Logger.LogWarning("Query log failed: {errorMessage}", errorMessage); return []; } @@ -563,15 +574,15 @@ public class SerilogElasticsearchLoggingManager : ILoggingManager, ISingletonDep return null; } - var remaining = skipCount - 10000; // 获取skipCount最近一条数据作为searchAfter var secondResponse = await client.SearchAsync( dsl => dsl.Indices(CreateIndex()) .Query(query) - .Sort(sorts) + // 反转排序取第一个数据作为起始索引 + .Sort(sorts.Select(x => x).Reverse().ToArray()) .SourceIncludes(x => x.Level) .SearchAfter(firstHit.Sort.ToList()) - .Size(remaining), + .Size(1), cancellationToken); if (!secondResponse.IsSuccess() || secondResponse.Hits == null || !secondResponse.Hits.Any()) @@ -579,11 +590,6 @@ public class SerilogElasticsearchLoggingManager : ILoggingManager, ISingletonDep return null; } - if (secondResponse.Hits.Count < remaining) - { - return null; - } - var lastHit = secondResponse.Hits.LastOrDefault(); if (lastHit?.Sort == null || !lastHit.Sort.Any()) { @@ -621,32 +627,29 @@ public class SerilogElasticsearchLoggingManager : ILoggingManager, ISingletonDep ? SortOrder.Asc : SortOrder.Desc; sorting = !sorting.IsNullOrWhiteSpace() ? sorting.Split()[0] - : nameof(SerilogInfo.TimeStamp); + : ElasticsearchJsonFormatter.TimestampPropertyName; SortOptions[]? sorts = null; - if (sorting.IsNullOrWhiteSpace()) + var sortingFieldMap = indexMappingInfo.Fields + .Where(x => x.Key.Equals(sorting, StringComparison.CurrentCultureIgnoreCase)) + .Select(x => x.Value) + .FirstOrDefault(); + if (sortingFieldMap != null) { - var sortingFieldMap = indexMappingInfo.Fields - .Where(x => x.Key.Equals(sorting, StringComparison.CurrentCultureIgnoreCase)) - .Select(x => x.Value) - .FirstOrDefault(); - if (sortingFieldMap != null) - { - sorting = sortingFieldMap.Path; - } - if (!sorting.IsNullOrWhiteSpace()) + sorting = sortingFieldMap.Path; + } + if (!sorting.IsNullOrWhiteSpace()) + { + sorts = new SortOptions[1] { - sorts = new SortOptions[1] + new SortOptions { - new SortOptions + Field = new FieldSort(new Field(sorting)) { - Field = new FieldSort(new Field(sorting)) - { - Order = sortOrder, - }, - } - }; - } + Order = sortOrder, + }, + } + }; } return sorts;