diff --git a/aspnet-core/framework/auditing/LINGYUN.Abp.AuditLogging.Elasticsearch/LINGYUN/Abp/AuditLogging/Elasticsearch/ElasticsearchAuditLogManager.cs b/aspnet-core/framework/auditing/LINGYUN.Abp.AuditLogging.Elasticsearch/LINGYUN/Abp/AuditLogging/Elasticsearch/ElasticsearchAuditLogManager.cs index 4368a4cba..b38a1e4c0 100644 --- a/aspnet-core/framework/auditing/LINGYUN.Abp.AuditLogging.Elasticsearch/LINGYUN/Abp/AuditLogging/Elasticsearch/ElasticsearchAuditLogManager.cs +++ b/aspnet-core/framework/auditing/LINGYUN.Abp.AuditLogging.Elasticsearch/LINGYUN/Abp/AuditLogging/Elasticsearch/ElasticsearchAuditLogManager.cs @@ -6,6 +6,7 @@ using Microsoft.Extensions.Logging.Abstractions; using System; using System.Collections.Generic; using System.Linq; +using System.Linq.Expressions; using System.Net; using System.Threading; using System.Threading.Tasks; @@ -67,7 +68,7 @@ public class ElasticsearchAuditLogManager : IAuditLogManager, ITransientDependen var sortingField = sorting; if (sortingField.IsNullOrWhiteSpace()) { - var indexMapping = await _indexMappingProvider.GetMappingAsync(indexName, cancellationToken); + var indexMapping = await _indexMappingProvider.GetMappingAsync(indexName, cancellationToken); if (indexMapping != null) { var sortingFieldMap = indexMapping.Fields @@ -87,13 +88,13 @@ public class ElasticsearchAuditLogManager : IAuditLogManager, ITransientDependen sortingField, maxResultCount, skipCount, - sourceExcludes: includeDetails == true + sourceExcludes: includeDetails == false ? Fields.FromFields( [ - new Field("Actions"), - new Field("Comments"), - new Field("EntityChanges"), - new Field("Exceptions"), + new Field(nameof(AuditLog.Actions)), + new Field(nameof(AuditLog.Comments)), + new Field(nameof(AuditLog.EntityChanges)), + new Field(nameof(AuditLog.Exceptions)), ]) : null, cancellationToken: cancellationToken); @@ -116,41 +117,31 @@ public class ElasticsearchAuditLogManager : IAuditLogManager, ITransientDependen HttpStatusCode? httpStatusCode = null, CancellationToken cancellationToken = default) { - var indexName = CreateIndex(); var client = _clientFactory.Create(); - var indexMapping = await _indexMappingProvider.GetMappingAsync(indexName, cancellationToken); - var querys = BuildQueryDescriptor( - indexMapping, - startTime, - endTime, - httpMethod, - url, - userId, - userName, - applicationName, - correlationId, - clientId, - clientIpAddress, - maxExecutionDuration, - minExecutionDuration, - hasException, - httpStatusCode); + Expression> expression = _ => true; + + expression = expression + .AndIf(startTime.HasValue, x => x.ExecutionTime >= _clock.Normalize(startTime!.Value)) + .AndIf(endTime.HasValue, x => x.ExecutionTime <= _clock.Normalize(endTime!.Value)) + .AndIf(!httpMethod.IsNullOrWhiteSpace(), x => x.HttpMethod == httpMethod) + .AndIf(!url.IsNullOrWhiteSpace(), x => x.Url!.Contains(url!)) + .AndIf(userId.HasValue, x => x.UserId == userId) + .AndIf(!userName.IsNullOrWhiteSpace(), x => x.UserName == userName) + .AndIf(!applicationName.IsNullOrWhiteSpace(), x => x.ApplicationName == applicationName) + .AndIf(!correlationId.IsNullOrWhiteSpace(), x => x.CorrelationId == correlationId) + .AndIf(!clientId.IsNullOrWhiteSpace(), x => x.ClientId == clientId) + .AndIf(!clientIpAddress.IsNullOrWhiteSpace(), x => x.ClientIpAddress == clientIpAddress) + .AndIf(maxExecutionDuration.HasValue, x => x.ExecutionDuration >= maxExecutionDuration) + .AndIf(minExecutionDuration.HasValue, x => x.ExecutionDuration <= minExecutionDuration) + .AndIf(hasException == true, x => x.Exceptions != null) + .AndIf(hasException == false, x => x.Exceptions == null) + .AndIf(httpStatusCode.HasValue, x => x.HttpStatusCode == (int)httpStatusCode!); - var response = await client.CountAsync(dsl => - dsl.Indices(indexName) - .Query(new BoolQuery - { - Must = querys - }), + return await _expressionQueryService.GetCountAsync( + CreateIndex(), + expression, cancellationToken); - - if (response.TryGetErrorMessage(out var errorMessage)) - { - Logger.LogWarning("Query audit log count failed: {errorMessage}", errorMessage); - } - - return response.Count; } public async virtual Task> GetListAsync( @@ -174,51 +165,47 @@ public class ElasticsearchAuditLogManager : IAuditLogManager, ITransientDependen bool includeDetails = false, CancellationToken cancellationToken = default) { - var indexName = CreateIndex(); var client = _clientFactory.Create(); - var indexMapping = await _indexMappingProvider.GetMappingAsync(indexName, cancellationToken); - - var sorts = GetOrDefaultSort(indexMapping, sorting); - - var querys = BuildQueryDescriptor( - indexMapping, - startTime, - endTime, - httpMethod, - url, - userId, - userName, - applicationName, - correlationId, - clientId, - clientIpAddress, - maxExecutionDuration, - minExecutionDuration, - hasException, - httpStatusCode); + if (sorting.IsNullOrWhiteSpace()) + { + sorting = $"{nameof(AuditLog.ExecutionTime)} DESC"; + } - var query = new BoolQuery { Must = querys }; + Expression> expression = _ => true; + + expression = expression + .AndIf(startTime.HasValue, x => x.ExecutionTime >= _clock.Normalize(startTime!.Value)) + .AndIf(endTime.HasValue, x => x.ExecutionTime <= _clock.Normalize(endTime!.Value)) + .AndIf(!httpMethod.IsNullOrWhiteSpace(), x => x.HttpMethod == httpMethod) + .AndIf(!url.IsNullOrWhiteSpace(), x => x.Url!.Contains(url!)) + .AndIf(userId.HasValue, x => x.UserId == userId) + .AndIf(!userName.IsNullOrWhiteSpace(), x => x.UserName == userName) + .AndIf(!applicationName.IsNullOrWhiteSpace(), x => x.ApplicationName == applicationName) + .AndIf(!correlationId.IsNullOrWhiteSpace(), x => x.CorrelationId == correlationId) + .AndIf(!clientId.IsNullOrWhiteSpace(), x => x.ClientId == clientId) + .AndIf(!clientIpAddress.IsNullOrWhiteSpace(), x => x.ClientIpAddress == clientIpAddress) + .AndIf(maxExecutionDuration.HasValue, x => x.ExecutionDuration >= maxExecutionDuration) + .AndIf(minExecutionDuration.HasValue, x => x.ExecutionDuration <= minExecutionDuration) + .AndIf(hasException == true, x => x.Exceptions != null) + .AndIf(hasException == false, x => x.Exceptions == null) + .AndIf(httpStatusCode.HasValue, x => x.HttpStatusCode == (int)httpStatusCode!); - // ES最大支持10000, 超出这个长度后升级为使用Search_After方案 - return skipCount >= 10000 && sorts != null - ? await SearchAfterAuditLogs( - client, - indexName, - query, - sorts, - maxResultCount, - skipCount, - includeDetails, - cancellationToken) - : await SearchFromSizeAuditLogs( - client, - indexName, - query, - sorts, - maxResultCount, - skipCount, - includeDetails, - cancellationToken); + return await _expressionQueryService.GetListAsync( + CreateIndex(), + expression, + sorting: sorting, + maxResultCount: maxResultCount, + skipCount: skipCount, + sourceExcludes: includeDetails == false + ? Fields.FromFields( + [ + new Field(nameof(AuditLog.Actions)), + new Field(nameof(AuditLog.Comments)), + new Field(nameof(AuditLog.EntityChanges)), + new Field(nameof(AuditLog.Exceptions)), + ]) + : null, + cancellationToken: cancellationToken); } public async virtual Task GetAsync( @@ -271,322 +258,8 @@ public class ElasticsearchAuditLogManager : IAuditLogManager, ITransientDependen cancellationToken); } - protected virtual List BuildQueryDescriptor( - IndexMappingInfo indexMappingInfo, - DateTime? startTime = null, - DateTime? endTime = null, - string? httpMethod = null, - string? url = null, - Guid? userId = null, - string? userName = null, - string? applicationName = null, - string? correlationId = null, - string? clientId = null, - string? clientIpAddress = null, - int? maxExecutionDuration = null, - int? minExecutionDuration = null, - bool? hasException = null, - HttpStatusCode? httpStatusCode = null) - { - var queries = new List(); - - if (startTime.HasValue) - { - queries.Add(new DateRangeQuery(GetField(indexMappingInfo, nameof(AuditLog.ExecutionTime))) - { - Gte = _clock.Normalize(startTime.Value) - }); - } - if (endTime.HasValue) - { - queries.Add(new DateRangeQuery(GetField(indexMappingInfo, nameof(AuditLog.ExecutionTime))) - { - Lte = _clock.Normalize(endTime.Value) - }); - } - if (!httpMethod.IsNullOrWhiteSpace()) - { - queries.Add(new TermQuery(GetField(indexMappingInfo, nameof(AuditLog.HttpMethod)), httpMethod)); - } - if (!url.IsNullOrWhiteSpace()) - { - queries.Add(new WildcardQuery(GetField(indexMappingInfo, nameof(AuditLog.Url))) - { - Value = $"*{url}*" - }); - } - if (userId.HasValue) - { - queries.Add(new TermQuery(GetField(indexMappingInfo, nameof(AuditLog.UserId)), userId.Value.ToString())); - } - if (!userName.IsNullOrWhiteSpace()) - { - queries.Add(new TermQuery(GetField(indexMappingInfo, nameof(AuditLog.UserName)), userName)); - } - if (!applicationName.IsNullOrWhiteSpace()) - { - queries.Add(new TermQuery(GetField(indexMappingInfo, nameof(AuditLog.ApplicationName)), applicationName)); - } - if (!correlationId.IsNullOrWhiteSpace()) - { - queries.Add(new TermQuery(GetField(indexMappingInfo, nameof(AuditLog.CorrelationId)), correlationId)); - } - if (!clientId.IsNullOrWhiteSpace()) - { - queries.Add(new TermQuery(GetField(indexMappingInfo, nameof(AuditLog.ClientId)), clientId)); - } - if (!clientIpAddress.IsNullOrWhiteSpace()) - { - queries.Add(new TermQuery(GetField(indexMappingInfo, nameof(AuditLog.ClientIpAddress)), clientIpAddress)); - } - if (maxExecutionDuration.HasValue) - { - queries.Add(new NumberRangeQuery(GetField(indexMappingInfo, nameof(AuditLog.ExecutionDuration))) - { - Lte = maxExecutionDuration.Value - }); - } - if (minExecutionDuration.HasValue) - { - queries.Add(new NumberRangeQuery(GetField(indexMappingInfo, nameof(AuditLog.ExecutionDuration))) - { - Gte = minExecutionDuration.Value - }); - } - - - if (hasException.HasValue) - { - if (hasException.Value) - { - queries.Add(new ExistsQuery(GetField(indexMappingInfo, nameof(AuditLog.Exceptions)))); - } - else - { - queries.Add(new BoolQuery - { - MustNot = new List - { - new ExistsQuery(GetField(indexMappingInfo, nameof(AuditLog.Exceptions))) - } - }); - } - } - - if (httpStatusCode.HasValue) - { - queries.Add(new TermQuery(GetField(indexMappingInfo, nameof(AuditLog.HttpStatusCode)), ((int)httpStatusCode.Value).ToString())); - } - - return queries; - } - - private async Task> SearchFromSizeAuditLogs( - ElasticsearchClient client, - string indexName, - Query query, - SortOptions[]? sorts = null, - int maxResultCount = 50, - int skipCount = 0, - bool includeDetails = false, - CancellationToken cancellationToken = default) - { - var searchResponse = await client.SearchAsync(dsl => - { - dsl.Indices(indexName) - .Query(query) - .From(skipCount) - .Size(maxResultCount); - if (sorts != null) - { - dsl.Sort(sorts); - } - - if (!includeDetails) - { - dsl.SourceExcludes( - ex => ex.Actions, - ex => ex.Comments, - ex => ex.Exceptions, - ex => ex.EntityChanges); - } - }, cancellationToken); - - if (searchResponse.TryGetErrorMessage(out var errorMessage)) - { - Logger.LogWarning("Query audit log failed: {errorMessage}", errorMessage); - return []; - } - - return searchResponse.Documents.ToList(); - } - - private async Task> SearchAfterAuditLogs( - ElasticsearchClient client, - string indexName, - Query query, - SortOptions[] sorts, - int maxResultCount = 50, - int skipCount = 0, - bool includeDetails = false, - CancellationToken cancellationToken = default) - { - var searchAfter = await GetSearchAfterValue( - client, - indexName, - query, - sorts, - skipCount, - cancellationToken); - - if (searchAfter == null || !searchAfter.Any()) - { - return []; - } - - var searchResponse = await client.SearchAsync(dsl => - { - dsl.Indices(indexName) - .Query(query) - .Sort(sorts) - .Size(maxResultCount) - .SearchAfter(searchAfter); - - if (!includeDetails) - { - dsl.SourceExcludes( - ex => ex.Actions, - ex => ex.Comments, - ex => ex.Exceptions, - ex => ex.EntityChanges); - } - }, cancellationToken); - - if (searchResponse.TryGetErrorMessage(out var errorMessage)) - { - Logger.LogWarning("Query audit log failed: {errorMessage}", errorMessage); - return []; - } - - return searchResponse.Documents.ToList(); - } - - private async Task?> GetSearchAfterValue( - ElasticsearchClient client, - string indexName, - Query query, - SortOptions[] sorts, - int skipCount, - CancellationToken cancellationToken = default) - { - // 10000以内直接取最后一条数据 - if (skipCount < 10000) - { - var response = await client.SearchAsync( - dsl => dsl.Indices(indexName) - .Query(query) - .Sort(sorts) - .SourceIncludes(x => x.Id) - .From(skipCount) - .Size(1), - cancellationToken); - - if (!response.IsSuccess() || response.Hits == null || !response.Hits.Any()) - { - return null; - } - - var hit = response.Hits.FirstOrDefault(); - return hit?.Sort?.ToList(); - } - - // 获取第9999条数据Hits作为searchAfter - var firstResponse = await client.SearchAsync( - dsl => dsl.Indices(indexName) - .Query(query) - .Sort(sorts) - .SourceIncludes(x => x.Id) - .From(9999) - .Size(1), - cancellationToken); - - if (!firstResponse.IsSuccess() || firstResponse.Hits == null || !firstResponse.Hits.Any()) - { - return null; - } - - var firstHit = firstResponse.Hits.FirstOrDefault(); - if (firstHit?.Sort == null || !firstHit.Sort.Any()) - { - return null; - } - - // 获取skipCount最近一条数据作为searchAfter - var secondResponse = await client.SearchAsync( - dsl => dsl.Indices(indexName) - .Query(query) - // 反转排序取第一个数据作为起始索引 - .Sort(sorts.ReverseSort()!.ToArray()) - .SourceIncludes(x => x.Id) - .SearchAfter(firstHit.Sort.ToList()) - .Size(1), - cancellationToken); - - if (!secondResponse.IsSuccess() || secondResponse.Hits == null || !secondResponse.Hits.Any()) - { - return null; - } - - var lastHit = secondResponse.Hits.LastOrDefault(); - if (lastHit?.Sort == null || !lastHit.Sort.Any()) - { - return null; - } - - return lastHit.Sort.ToList(); - } - protected virtual string CreateIndex() { return _indexNameNormalizer.NormalizeIndex("audit-log"); } - - private static SortOptions[]? GetOrDefaultSort(IndexMappingInfo indexMappingInfo, string? sorting = null) - { - var sortOrder = !sorting.IsNullOrWhiteSpace() && sorting.EndsWith("asc", StringComparison.InvariantCultureIgnoreCase) - ? SortOrder.Asc : SortOrder.Desc; - sorting = !sorting.IsNullOrWhiteSpace() - ? sorting.Split()[0] - : nameof(AuditLog.ExecutionTime); - - SortOptions[]? sorts = null; - var sortingFieldMap = indexMappingInfo.Fields - .Where(x => x.Key.Equals(sorting, StringComparison.CurrentCultureIgnoreCase)) - .Select(x => x.Value) - .FirstOrDefault(); - if (sortingFieldMap != null) - { - sorting = sortingFieldMap.Path; - } - if (!sorting.IsNullOrWhiteSpace()) - { - sorts = new SortOptions[1] - { - new SortOptions - { - Field = new FieldSort(new Field(sorting)) - { - Order = sortOrder, - }, - } - }; - } - - return sorts; - } - - private static string GetField(IndexMappingInfo indexMappingInfo, string fieldFullPath) - { - return indexMappingInfo.GetExactFieldPath(fieldFullPath); - } } diff --git a/aspnet-core/framework/auditing/LINGYUN.Abp.AuditLogging.Elasticsearch/System/Linq/Expressions/ExpressionFuncExtensions.cs b/aspnet-core/framework/auditing/LINGYUN.Abp.AuditLogging.Elasticsearch/System/Linq/Expressions/ExpressionFuncExtensions.cs new file mode 100644 index 000000000..fd7a017ea --- /dev/null +++ b/aspnet-core/framework/auditing/LINGYUN.Abp.AuditLogging.Elasticsearch/System/Linq/Expressions/ExpressionFuncExtensions.cs @@ -0,0 +1,32 @@ +using Volo.Abp.Specifications; + +namespace System.Linq.Expressions; + +internal static class ExpressionFuncExtensions +{ + public static Expression> AndIf( + this Expression> first, + bool condition, + Expression> second) + { + if (condition) + { + return ExpressionFuncExtender.And(first, second); + } + + return first; + } + + public static Expression> OrIf( + this Expression> first, + bool condition, + Expression> second) + { + if (condition) + { + return ExpressionFuncExtender.Or(first, second); + } + + return first; + } +} diff --git a/aspnet-core/framework/elasticsearch/LINGYUN.Abp.Elasticsearch/Elastic/Clients/Elasticsearch/SortOptionsExtenssions.cs b/aspnet-core/framework/elasticsearch/LINGYUN.Abp.Elasticsearch/Elastic/Clients/Elasticsearch/SortOptionsExtenssions.cs index fa7403661..8328d9cdc 100644 --- a/aspnet-core/framework/elasticsearch/LINGYUN.Abp.Elasticsearch/Elastic/Clients/Elasticsearch/SortOptionsExtenssions.cs +++ b/aspnet-core/framework/elasticsearch/LINGYUN.Abp.Elasticsearch/Elastic/Clients/Elasticsearch/SortOptionsExtenssions.cs @@ -12,7 +12,7 @@ public static class SortOptionsExtenssions { return sortOptions; } - var newSort = sortOptions.Select(sort => + return sortOptions.Select(sort => { if (sort.Field != null) { @@ -21,10 +21,6 @@ public static class SortOptionsExtenssions : SortOrder.Asc; } return sort; - }).ToArray(); - - newSort.Reverse(); - - return newSort; + }).Reverse(); } } diff --git a/aspnet-core/framework/elasticsearch/LINGYUN.Abp.Elasticsearch/LINGYUN/Abp/Elasticsearch/ElasticsearchIndexMappingProvider.cs b/aspnet-core/framework/elasticsearch/LINGYUN.Abp.Elasticsearch/LINGYUN/Abp/Elasticsearch/ElasticsearchIndexMappingProvider.cs index c3e32e898..aa431044c 100644 --- a/aspnet-core/framework/elasticsearch/LINGYUN.Abp.Elasticsearch/LINGYUN/Abp/Elasticsearch/ElasticsearchIndexMappingProvider.cs +++ b/aspnet-core/framework/elasticsearch/LINGYUN.Abp.Elasticsearch/LINGYUN/Abp/Elasticsearch/ElasticsearchIndexMappingProvider.cs @@ -2,10 +2,11 @@ using Elastic.Clients.Elasticsearch.IndexManagement; using Elastic.Clients.Elasticsearch.Mapping; using Microsoft.Extensions.Caching.Memory; -using Microsoft.Extensions.FileSystemGlobbing.Internal; using System; using System.Collections.Generic; using System.Linq; +using System.Reflection; +using System.Text.Json.Serialization; using System.Threading; using System.Threading.Tasks; using Volo.Abp; @@ -27,9 +28,26 @@ public class ElasticsearchIndexMappingProvider : IIndexMappingProvider, ITransie _cache = cache; } - public async Task GetMappingAsync(string indexPattern, CancellationToken cancellationToken = default) + public async virtual Task GetMappingAsync( + string indexPattern, + CancellationToken cancellationToken = default) { - var cacheKey = $"es_mapping_{indexPattern}"; + return await GetMappingAsync(indexPattern, typeof(TDocument), cancellationToken); + } + + public async virtual Task GetMappingAsync(string indexPattern, CancellationToken cancellationToken = default) + { + return await GetMappingAsync(indexPattern, null, cancellationToken); + } + + private async Task GetMappingAsync( + string indexPattern, + Type? documentType, + CancellationToken cancellationToken = default) + { + var cacheKey = documentType == null + ? $"es_mapping_{indexPattern}" + : $"es_mapping_{indexPattern}_{documentType.FullName}"; var cacheItem = _cache.Get(cacheKey); if (cacheItem == null) @@ -48,7 +66,10 @@ public class ElasticsearchIndexMappingProvider : IIndexMappingProvider, ITransie throw new AbpException($"Failed to get mapping for index {indexPattern}: {errorMessage}"); } - var indexName = indexPattern.EndsWith("*") ? indexPattern.Substring(0, indexPattern.Length - 1) : indexPattern; + var indexName = indexPattern.EndsWith("*") + ? indexPattern.Substring(0, indexPattern.Length - 1) + : indexPattern; + var indexMappings = response.GetMappingFor(indexName); if (indexMappings == null) { @@ -66,7 +87,7 @@ public class ElasticsearchIndexMappingProvider : IIndexMappingProvider, ITransie } } - cacheItem = ParseMapping(indexMappings, indexPattern); + cacheItem = ParseMapping(indexMappings, indexPattern, documentType); _cache.Set(cacheKey, cacheItem, _cacheDuration); } @@ -74,35 +95,68 @@ public class ElasticsearchIndexMappingProvider : IIndexMappingProvider, ITransie return cacheItem; } - private IndexMappingInfo ParseMapping(TypeMapping mappings, string indexName) + private static IndexMappingInfo ParseMapping( + TypeMapping mappings, + string indexName, + Type? documentType = null) { - var mappingInfo = new IndexMappingInfo { IndexName = indexName }; + var mappingInfo = new IndexMappingInfo + { + IndexName = indexName, + DocumentType = documentType + }; if (mappings?.Properties != null) { - ParseProperties(mappings.Properties, mappingInfo, string.Empty); + ParseProperties( + mappings.Properties, + mappingInfo, + string.Empty, + documentType, + string.Empty); } return mappingInfo; } - private void ParseProperties(Properties? properties, IndexMappingInfo mappingInfo, string parentPath) + private static void ParseProperties( + Properties? properties, + IndexMappingInfo mappingInfo, + string esParentPath, + Type? parentClrType, + string clrParentPath) { - if (properties == null) return; + if (properties == null) + { + return; + } foreach (var kvp in properties) { var propertyName = kvp.Key.ToString(); var property = kvp.Value; - var fullPath = string.IsNullOrEmpty(parentPath) + + // ES 完整路径 + var esFullPath = string.IsNullOrEmpty(esParentPath) + ? propertyName + : $"{esParentPath}.{propertyName}"; + + // CLR 完整路径 + var clrFullPath = string.IsNullOrEmpty(clrParentPath) ? propertyName - : $"{parentPath}.{propertyName}"; + : $"{clrParentPath}.{propertyName}"; + + // 解析 CLR 类型 + var clrType = ResolveClrType(parentClrType, propertyName); + var fieldInfo = new FieldMappingInfo { - Path = fullPath, + Path = esFullPath, Name = propertyName, - Type = GetPropertyType(property) + Type = GetPropertyType(property), + ClrType = clrType, + ClrPath = clrFullPath }; switch (property) @@ -110,13 +164,13 @@ public class ElasticsearchIndexMappingProvider : IIndexMappingProvider, ITransie // Keyword 类型 case KeywordProperty keyword: fieldInfo.IsKeyword = true; - mappingInfo.KeywordFields.Add(fullPath); + mappingInfo.KeywordFields.Add(esFullPath); break; // Text 类型 - 包含多字段支持 case TextProperty text: fieldInfo.IsText = true; - mappingInfo.TextFields.Add(fullPath); + mappingInfo.TextFields.Add(esFullPath); // 处理 Text 的 Fields（多字段） if (text.Fields != null && text.Fields.Count() > 0) @@ -127,29 +181,34 @@ public class ElasticsearchIndexMappingProvider : IIndexMappingProvider, ITransie { var subFieldName = subFieldKvp.Key.ToString(); var subFieldProperty = subFieldKvp.Value; - var subFieldPath = $"{fullPath}.{subFieldName}"; + var subFieldEsPath = $"{esFullPath}.{subFieldName}"; + var subFieldClrPath = $"{clrFullPath}.{subFieldName}"; var subFieldInfo = new FieldMappingInfo { - Path = subFieldPath, + Path = subFieldEsPath, Name = subFieldName, - Type = GetPropertyType(subFieldProperty) + Type = GetPropertyType(subFieldProperty), + ClrType = clrType, + ClrPath = subFieldClrPath, + IsMultiField = true }; // 处理子字段的类型 if (subFieldProperty is KeywordProperty) { subFieldInfo.IsKeyword = true; - mappingInfo.KeywordFields.Add(subFieldPath); + mappingInfo.KeywordFields.Add(subFieldEsPath); } else if (subFieldProperty is TextProperty) { subFieldInfo.IsText = true; - mappingInfo.TextFields.Add(subFieldPath); + mappingInfo.TextFields.Add(subFieldEsPath); } fieldInfo.Properties[subFieldName] = subFieldInfo; - mappingInfo.Fields[subFieldPath] = subFieldInfo; + mappingInfo.Fields[subFieldEsPath] = subFieldInfo; + mappingInfo.ClrFields[subFieldClrPath] = subFieldInfo; } } break; @@ -158,14 +217,14 @@ public class ElasticsearchIndexMappingProvider : IIndexMappingProvider, ITransie case DateProperty date: fieldInfo.IsDate = true; fieldInfo.Format = date.Format; - mappingInfo.DateFields.Add(fullPath); + mappingInfo.DateFields.Add(esFullPath); break; // 日期纳秒类型 case DateNanosProperty dateNanos: fieldInfo.IsDate = true; fieldInfo.Format = dateNanos.Format; - mappingInfo.DateFields.Add(fullPath); + mappingInfo.DateFields.Add(esFullPath); break; // 数值类型 @@ -179,58 +238,71 @@ public class ElasticsearchIndexMappingProvider : IIndexMappingProvider, ITransie case ShortNumberProperty: case UnsignedLongNumberProperty: fieldInfo.IsNumeric = true; - mappingInfo.NumericFields.Add(fullPath); + mappingInfo.NumericFields.Add(esFullPath); break; // 布尔类型 case BooleanProperty: fieldInfo.IsBoolean = true; - mappingInfo.BooleanFields.Add(fullPath); + mappingInfo.BooleanFields.Add(esFullPath); break; // Nested 类型 case NestedProperty nested: fieldInfo.IsNested = true; fieldInfo.IsObject = true; - mappingInfo.NestedFieldPaths.Add(fullPath); + mappingInfo.NestedFieldPaths.Add(esFullPath); var nestedInfo = new NestedMappingInfo { - Path = fullPath, + Path = esFullPath, Name = propertyName, Properties = new Dictionary() }; + // 获取 nested 集合的元素类型 + var elementType = GetElementType(clrType); + if (nested.Properties != null) { // 先递归解析内部字段 - ParseProperties(nested.Properties, mappingInfo, fullPath); + ParseProperties( + nested.Properties, + mappingInfo, + esFullPath, + elementType ?? clrType, + clrFullPath); // 收集 nested 内部的字段信息 foreach (var innerKvp in nested.Properties) { var innerName = innerKvp.Key.ToString(); - var innerFullPath = $"{fullPath}.{innerName}"; + var innerEsFullPath = $"{esFullPath}.{innerName}"; + var innerClrFullPath = $"{clrFullPath}.{innerName}"; - if (mappingInfo.Fields.TryGetValue(innerFullPath, out var innerFieldInfo)) + if (mappingInfo.Fields.TryGetValue(innerEsFullPath, out var innerFieldInfo)) { nestedInfo.Properties[innerName] = innerFieldInfo; } else { + var innerClrType = ResolveClrType(elementType ?? clrType, innerName); innerFieldInfo = new FieldMappingInfo { - Path = innerFullPath, + Path = innerEsFullPath, Name = innerName, - Type = GetPropertyType(innerKvp.Value) + Type = GetPropertyType(innerKvp.Value), + ClrType = innerClrType, + ClrPath = innerClrFullPath }; nestedInfo.Properties[innerName] = innerFieldInfo; - mappingInfo.Fields[innerFullPath] = innerFieldInfo; + mappingInfo.Fields[innerEsFullPath] = innerFieldInfo; + mappingInfo.ClrFields[innerClrFullPath] = innerFieldInfo; } } } - mappingInfo.NestedFields[fullPath] = nestedInfo; + mappingInfo.NestedFields[esFullPath] = nestedInfo; break; // Object 类型 @@ -240,7 +312,12 @@ public class ElasticsearchIndexMappingProvider : IIndexMappingProvider, ITransie if (obj.Properties != null) { - ParseProperties(obj.Properties, mappingInfo, fullPath); + ParseProperties( + obj.Properties, + mappingInfo, + esFullPath, + clrType, + clrFullPath); } break; @@ -278,7 +355,7 @@ public class ElasticsearchIndexMappingProvider : IIndexMappingProvider, ITransie case MatchOnlyTextProperty matchOnlyText: fieldInfo.IsText = true; fieldInfo.Type = "match_only_text"; - mappingInfo.TextFields.Add(fullPath); + mappingInfo.TextFields.Add(esFullPath); // MatchOnlyText 也可能有 Fields if (matchOnlyText.Fields != null && matchOnlyText.Fields.Count() > 0) @@ -287,20 +364,25 @@ public class ElasticsearchIndexMappingProvider : IIndexMappingProvider, ITransie foreach (var subFieldKvp in matchOnlyText.Fields) { var subFieldName = subFieldKvp.Key.ToString(); - var subFieldPath = $"{fullPath}.{subFieldName}"; + var subFieldEsPath = $"{esFullPath}.{subFieldName}"; + var subFieldClrPath = $"{clrFullPath}.{subFieldName}"; var subFieldInfo = new FieldMappingInfo { - Path = subFieldPath, + Path = subFieldEsPath, Name = subFieldName, - Type = GetPropertyType(subFieldKvp.Value) + Type = GetPropertyType(subFieldKvp.Value), + ClrType = clrType, + ClrPath = subFieldClrPath, + IsMultiField = true }; if (subFieldKvp.Value is KeywordProperty) { subFieldInfo.IsKeyword = true; - mappingInfo.KeywordFields.Add(subFieldPath); + mappingInfo.KeywordFields.Add(subFieldEsPath); } fieldInfo.Properties[subFieldName] = subFieldInfo; - mappingInfo.Fields[subFieldPath] = subFieldInfo; + mappingInfo.Fields[subFieldEsPath] = subFieldInfo; + mappingInfo.ClrFields[subFieldClrPath] = subFieldInfo; } } break; @@ -308,7 +390,7 @@ public class ElasticsearchIndexMappingProvider : IIndexMappingProvider, ITransie case WildcardProperty: fieldInfo.IsWildcard = true; fieldInfo.Type = "wildcard"; - mappingInfo.WildcardFields.Add(fullPath); + mappingInfo.WildcardFields.Add(esFullPath); break; case CompletionProperty: @@ -344,11 +426,92 @@ public class ElasticsearchIndexMappingProvider : IIndexMappingProvider, ITransie break; } - mappingInfo.Fields[fullPath] = fieldInfo; + // 添加到映射集合 + mappingInfo.Fields[esFullPath] = fieldInfo; + + // 只有在有 CLR 类型信息时才添加到 CLR 字段集合 + if (clrType != null || parentClrType != null) + { + mappingInfo.ClrFields[clrFullPath] = fieldInfo; + } + } + } + + private static Type? ResolveClrType(Type? parentType, string propertyName) + { + if (parentType == null) + { + return null; } + + var propertyInfo = parentType.GetProperty( + propertyName, + BindingFlags.Public | BindingFlags.Instance | BindingFlags.IgnoreCase); + + return propertyInfo?.PropertyType; + } + + private static Type? GetElementType(Type? type) + { + if (type == null) + { + return null; + } + + // 处理数组 + if (type.IsArray) + { + return type.GetElementType(); + } + + // 处理 IEnumerable + if (type.IsGenericType) + { + var genericTypeDefinition = type.GetGenericTypeDefinition(); + if (genericTypeDefinition == typeof(IEnumerable<>) || + genericTypeDefinition == typeof(ICollection<>) || + genericTypeDefinition == typeof(IList<>) || + genericTypeDefinition == typeof(List<>)) + { + return type.GetGenericArguments()[0]; + } + } + + // 处理实现了 IEnumerable 的接口 + var enumerableInterface = type.GetInterfaces() + .FirstOrDefault(i => i.IsGenericType && + i.GetGenericTypeDefinition() == typeof(IEnumerable<>)); + + return enumerableInterface?.GetGenericArguments()[0]; + } + + private static string GetJsonPropertyName(Type? type, string propertyName) + { + if (type == null) + { + return propertyName; + } + + var propertyInfo = type.GetProperty( + propertyName, + BindingFlags.Public | BindingFlags.Instance | BindingFlags.IgnoreCase); + + if (propertyInfo == null) + { + return propertyName; + } + + // 检查 JsonPropertyName 特性 + var jsonPropertyNameAttribute = propertyInfo.GetCustomAttribute(); + if (jsonPropertyNameAttribute != null && !string.IsNullOrEmpty(jsonPropertyNameAttribute.Name)) + { + return jsonPropertyNameAttribute.Name; + } + + return propertyName; } - private string GetPropertyType(IProperty property) + private static string GetPropertyType(IProperty property) { return property switch { diff --git a/aspnet-core/framework/elasticsearch/LINGYUN.Abp.Elasticsearch/LINGYUN/Abp/Elasticsearch/EmptyDocument.cs b/aspnet-core/framework/elasticsearch/LINGYUN.Abp.Elasticsearch/LINGYUN/Abp/Elasticsearch/EmptyDocument.cs new file mode 100644 index 000000000..3a05fcb6b --- /dev/null +++ b/aspnet-core/framework/elasticsearch/LINGYUN.Abp.Elasticsearch/LINGYUN/Abp/Elasticsearch/EmptyDocument.cs @@ -0,0 +1,8 @@ +using System; + +namespace LINGYUN.Abp.Elasticsearch; + +[Serializable] +public class EmptyDocument +{ +} diff --git a/aspnet-core/framework/elasticsearch/LINGYUN.Abp.Elasticsearch/LINGYUN/Abp/Elasticsearch/ExpressionQueryService.cs b/aspnet-core/framework/elasticsearch/LINGYUN.Abp.Elasticsearch/LINGYUN/Abp/Elasticsearch/ExpressionQueryService.cs index f7107fcc5..f6a988ab7 100644 --- a/aspnet-core/framework/elasticsearch/LINGYUN.Abp.Elasticsearch/LINGYUN/Abp/Elasticsearch/ExpressionQueryService.cs +++ b/aspnet-core/framework/elasticsearch/LINGYUN.Abp.Elasticsearch/LINGYUN/Abp/Elasticsearch/ExpressionQueryService.cs @@ -1,5 +1,7 @@ using Elastic.Clients.Elasticsearch; using Elastic.Clients.Elasticsearch.QueryDsl; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Logging.Abstractions; using System; using System.Collections.Generic; using System.Linq; @@ -12,15 +14,21 @@ namespace LINGYUN.Abp.Elasticsearch; public class ExpressionQueryService : IExpressionQueryService, ITransientDependency { + public ILogger Logger { protected get; set; } protected IElasticsearchClientFactory ClientFactory { get; } + protected IIndexMappingProvider IndexMappingProvider { get; } protected IExpressionQueryTranslator ExpressionQueryTranslator { get; } public ExpressionQueryService( IElasticsearchClientFactory clientFactory, + IIndexMappingProvider indexMappingProvider, IExpressionQueryTranslator expressionQueryTranslator) { ClientFactory = clientFactory; + IndexMappingProvider = indexMappingProvider; ExpressionQueryTranslator = expressionQueryTranslator; + + Logger = NullLogger.Instance; } public async virtual Task GetCountAsync( @@ -55,19 +63,11 @@ public class ExpressionQueryService : IExpressionQueryService, ITransientDepende SortOptions[]? sorts = null; if (!sorting.IsNullOrWhiteSpace()) { - var sortOrder = !sorting.IsNullOrWhiteSpace() && sorting.EndsWith("asc", StringComparison.InvariantCultureIgnoreCase) - ? SortOrder.Asc : SortOrder.Desc; - - sorts = new SortOptions[1] + var indexMapping = await IndexMappingProvider.GetMappingAsync(indexName, cancellationToken); + if (indexMapping != null) { - new SortOptions - { - Field = new FieldSort(new Field(sorting)) - { - Order = sortOrder, - }, - } - }; + sorts = ResolveDefaultSorts(indexMapping, sorting); + } } // 数量超过10000且存在排序时才可以使用SearchAfter特性 @@ -126,8 +126,9 @@ public class ExpressionQueryService : IExpressionQueryService, ITransientDepende } }, cancellationToken); - if (!searchResponse.IsSuccess()) + if (searchResponse.TryGetErrorMessage(out var errorMessage)) { + Logger.LogWarning("Query document failed: {errorMessage}", errorMessage); return []; } @@ -153,7 +154,7 @@ public class ExpressionQueryService : IExpressionQueryService, ITransientDepende } else { - searchAfter = await GetSearchAfterValue( + searchAfter = await GetSearchAfterValue( client, indexName, query, @@ -184,15 +185,16 @@ public class ExpressionQueryService : IExpressionQueryService, ITransientDepende } }, cancellationToken); - if (!searchResponse.IsSuccess()) + if (searchResponse.TryGetErrorMessage(out var errorMessage)) { + Logger.LogWarning("Query document failed: {errorMessage}", errorMessage); return []; } return searchResponse.Documents.ToList(); } - private async Task?> GetSearchAfterValue( + private async Task?> GetSearchAfterValue( ElasticsearchClient client, string indexName, Query query, @@ -203,16 +205,19 @@ public class ExpressionQueryService : IExpressionQueryService, ITransientDepende // 10000以内直接取最后一条数据 if (skipCount < 10000) { - var response = await client.SearchAsync( + var response = await client.SearchAsync( dsl => dsl.Indices(indexName) .Query(query) .Sort(sorts) .From(skipCount) - .Size(1), + .Size(1) + .Source(false) + .TrackScores(false), cancellationToken); - if (!response.IsSuccess() || response.Hits == null || !response.Hits.Any()) + if (response.TryGetErrorMessage(out var oneError)) { + Logger.LogWarning("Failed to obtain the {skipCount}th sorting record. error: {error}", skipCount, oneError); return null; } @@ -221,48 +226,235 @@ public class ExpressionQueryService : IExpressionQueryService, ITransientDepende } // 获取第9999条数据Hits作为searchAfter - var firstResponse = await client.SearchAsync( + var firstResponse = await client.SearchAsync( dsl => dsl.Indices(indexName) .Query(query) .Sort(sorts) - .SourceIncludes([]) .From(9999) - .Size(1), + .Size(1) + .Source(false) + .TrackScores(false), cancellationToken); - if (!firstResponse.IsSuccess() || firstResponse.Hits == null || !firstResponse.Hits.Any()) + if (firstResponse.TryGetErrorMessage(out var firstError)) { + Logger.LogWarning("Failed to obtain the first sorted record after the {skipCount}th item. error: {error}", skipCount, firstError); return null; } - var firstHit = firstResponse.Hits.FirstOrDefault(); + var firstHit = firstResponse.Hits?.FirstOrDefault(); if (firstHit?.Sort == null || !firstHit.Sort.Any()) { + Logger.LogWarning("The first sorted record after the {skipCount}th item is empty!", skipCount); return null; } - // 获取skipCount最近一条数据作为searchAfter - var secondResponse = await client.SearchAsync( - dsl => dsl.Indices(indexName) - .Query(query) - // 反转排序取第一个数据作为起始索引 - .Sort(sorts.ReverseSort()!.ToArray()) - .SourceIncludes([]) - .SearchAfter(firstHit.Sort.ToList()) - .Size(1), + var remaining = skipCount - 10000; + + return await GetBatchSearchAfterValue( + client, + indexName, + query, + sorts, + [.. firstHit.Sort], + remaining, + remaining > 10000 ? 5000 : 1000, cancellationToken); + } + + private async Task?> GetBatchSearchAfterValue( + ElasticsearchClient client, + string indexName, + Query query, + SortOptions[] sorts, + FieldValue[] searchAfter, + int remaining, + int batchSize = 1000, + CancellationToken cancellationToken = default) + { + List? lastSort = null; + + while (remaining > 0) + { + var batch = Math.Min(remaining, batchSize); + + var response = await client.SearchAsync( + dsl => dsl.Indices(indexName) + .Query(query) + .Sort(sorts) + .SearchAfter(searchAfter) + .Size(batch) + .Source(false) + .TrackScores(false), + cancellationToken); + + if (response.TryGetErrorMessage(out var error)) + { + Logger.LogWarning("Failed to get batch records. remaining: {remaining}, error: {error}", remaining, error); + return null; + } + + if (response.Hits == null || !response.Hits.Any()) + { + Logger.LogWarning("No more records available. remaining: {remaining}", remaining); + return null; + } + + var hits = response.Hits.ToList(); + var hitCount = hits.Count; + + remaining -= hitCount; + + if (remaining <= 0) + { + var targetIndex = hitCount + remaining; + if (targetIndex == hitCount) + { + lastSort = hits.LastOrDefault()?.Sort?.ToList(); + } + else if (targetIndex >= 0 && targetIndex < hitCount) + { + lastSort = hits[targetIndex]?.Sort?.ToList(); + } + else + { + return null; + } + + return lastSort; + } + + var lastHit = hits.LastOrDefault(); + if (lastHit?.Sort == null || !lastHit.Sort.Any()) + { + return null; + } - if (!secondResponse.IsSuccess() || secondResponse.Hits == null || !secondResponse.Hits.Any()) + searchAfter = [.. lastHit.Sort]; + + if (hitCount < batch) + { + return null; + } + } + + return lastSort; + } + + private static SortOptions[]? ResolveDefaultSorts(IndexMappingInfo indexMappingInfo, string? sorting = null) + { + if (sorting.IsNullOrWhiteSpace()) { return null; } - var lastHit = secondResponse.Hits.LastOrDefault(); - if (lastHit?.Sort == null || !lastHit.Sort.Any()) + // eg: a desc, b.c asc; d+desc; e-asc; +f; -g + var sortFields = sorting.Split([';', ','], StringSplitOptions.RemoveEmptyEntries); + var sorts = new List(); + + foreach (var sortField in sortFields) + { + var trimmedSortField = sortField.Trim(); + if (trimmedSortField.IsNullOrWhiteSpace()) + { + continue; + } + + // [a, desc] + // [b.c, asc] + // [d, desc] + // [e, asc] + var parts = trimmedSortField.Split([' ', ':', '-', '+'], StringSplitOptions.RemoveEmptyEntries); + + string fieldName; + SortOrder sortOrder; + + if (parts.Length >= 2) + { + // b.c + fieldName = parts[0].Trim(); + // desc + var orderStr = parts[1].Trim(); + sortOrder = orderStr.Equals("desc", StringComparison.InvariantCultureIgnoreCase) || + orderStr.Equals("descending", StringComparison.InvariantCultureIgnoreCase) + ? SortOrder.Desc + : SortOrder.Asc; + } + else + { + fieldName = parts[0].Trim(); + // +f + if (fieldName.StartsWith("+")) + { + sortOrder = SortOrder.Asc; + fieldName = fieldName.Substring(1); + } + // -g + else if (fieldName.StartsWith("-")) + { + sortOrder = SortOrder.Desc; + fieldName = fieldName.Substring(1); + } + else + { + sortOrder = SortOrder.Asc; + } + } + + var resolvedField = ResolveSortField(indexMappingInfo, fieldName); + if (resolvedField != null) + { + var fieldPath = resolvedField.GetKeywordPath(); + if (!sorts.Any(x => x.Field?.Field?.Name == fieldPath)) + { + sorts.Add(new SortOptions + { + Field = new FieldSort(Field.FromString(fieldPath)) + { + Order = sortOrder, + } + }); + } + } + } + + return sorts?.ToArray(); + } + + private static FieldMappingInfo? ResolveSortField( + IndexMappingInfo indexMappingInfo, + string fieldPath) + { + if (fieldPath.IsNullOrWhiteSpace()) { return null; } - return lastHit.Sort.ToList(); + var directField = indexMappingInfo.GetField(fieldPath); + if (directField != null) + { + return directField; + } + + var clrField = indexMappingInfo.GetFieldByClrPath(fieldPath); + if (clrField != null) + { + return clrField; + } + + var caseInsensitiveEsField = indexMappingInfo.Fields + .FirstOrDefault(kvp => kvp.Key.Equals(fieldPath, StringComparison.InvariantCultureIgnoreCase)) + .Value; + + if (caseInsensitiveEsField != null) + { + return caseInsensitiveEsField; + } + + var caseInsensitiveClrField = indexMappingInfo.ClrFields + .FirstOrDefault(kvp => kvp.Key.Equals(fieldPath, StringComparison.InvariantCultureIgnoreCase)) + .Value; + + return caseInsensitiveClrField; } } diff --git a/aspnet-core/framework/elasticsearch/LINGYUN.Abp.Elasticsearch/LINGYUN/Abp/Elasticsearch/ExpressionQueryTranslator.Comparison.cs b/aspnet-core/framework/elasticsearch/LINGYUN.Abp.Elasticsearch/LINGYUN/Abp/Elasticsearch/ExpressionQueryTranslator.Comparison.cs index 7435ae1c4..cbc56a660 100644 --- a/aspnet-core/framework/elasticsearch/LINGYUN.Abp.Elasticsearch/LINGYUN/Abp/Elasticsearch/ExpressionQueryTranslator.Comparison.cs +++ b/aspnet-core/framework/elasticsearch/LINGYUN.Abp.Elasticsearch/LINGYUN/Abp/Elasticsearch/ExpressionQueryTranslator.Comparison.cs @@ -39,7 +39,7 @@ public partial class ExpressionQueryTranslator IndexMappingInfo? mappingInfo) { // 限定string方法调用 - if (methodCall.Method.DeclaringType != typeof(string)) + if (methodCall.Method.DeclaringType == typeof(string)) { return methodCall.Method.Name switch { diff --git a/aspnet-core/framework/elasticsearch/LINGYUN.Abp.Elasticsearch/LINGYUN/Abp/Elasticsearch/ExpressionQueryTranslator.String.cs b/aspnet-core/framework/elasticsearch/LINGYUN.Abp.Elasticsearch/LINGYUN/Abp/Elasticsearch/ExpressionQueryTranslator.String.cs index 29b9af748..f7df62498 100644 --- a/aspnet-core/framework/elasticsearch/LINGYUN.Abp.Elasticsearch/LINGYUN/Abp/Elasticsearch/ExpressionQueryTranslator.String.cs +++ b/aspnet-core/framework/elasticsearch/LINGYUN.Abp.Elasticsearch/LINGYUN/Abp/Elasticsearch/ExpressionQueryTranslator.String.cs @@ -298,45 +298,34 @@ public partial class ExpressionQueryTranslator string? prefix, IndexMappingInfo? mappingInfo) { - var compareValue = Evaluate(constantExpression)?.ToString() ?? string.Empty; - - // CompareTo 返回值： - // 0: 相等 - // > 0: 当前字符串在排序顺序中位于参数之后 - // < 0: 当前字符串在排序顺序中位于参数之前 + var (field, compareValue) = GetStringMethodOperands(methodCall, prefix, mappingInfo); + var compareResult = Convert.ToInt32(Evaluate(constantExpression)); switch (comparisonType) { - case ExpressionType.Equal: + case ExpressionType.Equal when compareResult == 0: // CompareTo == 0 表示相等 - var (field, value) = GetStringMethodOperands(methodCall, prefix, mappingInfo); - return BuildEquality(field, value); + return BuildEquality(field, compareValue); - case ExpressionType.NotEqual: + case ExpressionType.NotEqual when compareResult == 0: // CompareTo != 0 表示不相等 - var (notEqualField, notEqualValue) = GetStringMethodOperands(methodCall, prefix, mappingInfo); - var notEqualQuery = BuildEquality(notEqualField, notEqualValue); - return new BoolQuery - { - MustNot = new Query[] { notEqualQuery } - }; + return BuildNotEqualQuery(field, compareValue); - case ExpressionType.GreaterThan: + case ExpressionType.GreaterThan when compareResult == 0: // CompareTo > 0 表示当前字段值大于比较值 - // 这可以简化处理，但这里先返回 null 让默认处理逻辑处理 - return null; + return BuildRange(field, greaterThan: compareValue); - case ExpressionType.GreaterThanOrEqual: + case ExpressionType.GreaterThanOrEqual when compareResult == 0: // CompareTo >= 0 表示当前字段值大于或等于比较值 - return null; + return BuildRange(field, greaterThanOrEqualTo: compareValue); - case ExpressionType.LessThan: + case ExpressionType.LessThan when compareResult == 0: // CompareTo < 0 表示当前字段值小于比较值 - return null; + return BuildRange(field, lessThan: compareValue); - case ExpressionType.LessThanOrEqual: + case ExpressionType.LessThanOrEqual when compareResult == 0: // CompareTo <= 0 表示当前字段值小于或等于比较值 - return null; + return BuildRange(field, lessThanOrEqualTo: compareValue); default: return null; @@ -353,46 +342,28 @@ public partial class ExpressionQueryTranslator string? prefix, IndexMappingInfo? mappingInfo) { - var indexValue = Evaluate(constantExpression)?.ToString() ?? string.Empty; - - // IndexOf 返回值： - // >= 0: 找到了子字符串 - // < 0: 没有找到子字符串 + // 获取字段和搜索值 + var (field, searchValue) = GetStringMethodOperands(methodCall, prefix, mappingInfo); + var indexResult = Convert.ToInt32(Evaluate(constantExpression)); + var fieldMapping = mappingInfo?.GetField(field.Path); switch (comparisonType) { - case ExpressionType.GreaterThanOrEqual: - case ExpressionType.GreaterThan: - // IndexOf >= 0 表示包含 - var (field, value) = GetStringMethodOperands(methodCall, prefix, mappingInfo); - var fieldMapping = mappingInfo?.GetField(field.Path); - return TranslateStringContains(field, fieldMapping, value); - - case ExpressionType.LessThan: - case ExpressionType.LessThanOrEqual: - // IndexOf < 0 表示不包含 - var (notContainsField, notContainsValue) = GetStringMethodOperands(methodCall, prefix, mappingInfo); - var notContainsFieldMapping = mappingInfo?.GetField(notContainsField.Path); - var containsQuery = TranslateStringContains(notContainsField, notContainsFieldMapping, notContainsValue); + case ExpressionType.GreaterThanOrEqual when indexResult >= 0: + case ExpressionType.GreaterThan when indexResult > -1: + case ExpressionType.NotEqual when indexResult == -1: + // IndexOf >= 0, IndexOf > -1, IndexOf != -1 表示包含 + return TranslateStringContains(field, fieldMapping, searchValue); + + case ExpressionType.LessThan when indexResult <= 0: + case ExpressionType.Equal when indexResult == -1: + // IndexOf == -1, IndexOf < 0 表示不包含 + var containsQuery = TranslateStringContains(field, fieldMapping, searchValue); return new BoolQuery { MustNot = new Query[] { containsQuery } }; - case ExpressionType.Equal: - // IndexOf == 某个值，这里简化处理，只处理 == -1（不包含）的情况 - if (indexValue == "-1") - { - var (eqField, eqValue) = GetStringMethodOperands(methodCall, prefix, mappingInfo); - var eqFieldMapping = mappingInfo?.GetField(eqField.Path); - var eqContainsQuery = TranslateStringContains(eqField, eqFieldMapping, eqValue); - return new BoolQuery - { - MustNot = new Query[] { eqContainsQuery } - }; - } - return null; - default: return null; } diff --git a/aspnet-core/framework/elasticsearch/LINGYUN.Abp.Elasticsearch/LINGYUN/Abp/Elasticsearch/ExpressionQueryTranslator.cs b/aspnet-core/framework/elasticsearch/LINGYUN.Abp.Elasticsearch/LINGYUN/Abp/Elasticsearch/ExpressionQueryTranslator.cs index a8352be86..b1b8c87d4 100644 --- a/aspnet-core/framework/elasticsearch/LINGYUN.Abp.Elasticsearch/LINGYUN/Abp/Elasticsearch/ExpressionQueryTranslator.cs +++ b/aspnet-core/framework/elasticsearch/LINGYUN.Abp.Elasticsearch/LINGYUN/Abp/Elasticsearch/ExpressionQueryTranslator.cs @@ -6,7 +6,6 @@ using System.Collections.Generic; using System.Linq; using System.Linq.Expressions; using System.Reflection; -using System.Text.Json.Serialization; using System.Threading.Tasks; using Volo.Abp; using Volo.Abp.DependencyInjection; @@ -34,7 +33,7 @@ public partial class ExpressionQueryTranslator : IExpressionQueryTranslator, ISi Check.NotNullOrWhiteSpace(indexName, nameof(indexName)); Check.NotNull(expression, nameof(expression)); - var indexMapping = await _indexMappingProvider.GetMappingAsync(indexName); + var indexMapping = await _indexMappingProvider.GetMappingAsync(indexName); return TranslateNode(expression.Body, prefix: null, indexMapping); } diff --git a/aspnet-core/framework/elasticsearch/LINGYUN.Abp.Elasticsearch/LINGYUN/Abp/Elasticsearch/FieldMappingInfo.cs b/aspnet-core/framework/elasticsearch/LINGYUN.Abp.Elasticsearch/LINGYUN/Abp/Elasticsearch/FieldMappingInfo.cs index a43d393c1..df22ebc21 100644 --- a/aspnet-core/framework/elasticsearch/LINGYUN.Abp.Elasticsearch/LINGYUN/Abp/Elasticsearch/FieldMappingInfo.cs +++ b/aspnet-core/framework/elasticsearch/LINGYUN.Abp.Elasticsearch/LINGYUN/Abp/Elasticsearch/FieldMappingInfo.cs @@ -9,7 +9,9 @@ public class FieldMappingInfo public string Name { get; set; } = string.Empty; public string Type { get; set; } = string.Empty; public Type? ClrType { get; set; } + public string ClrPath { get; set; } = string.Empty; + public bool IsMultiField { get; set; } public bool IsKeyword { get; set; } public bool IsText { get; set; } public bool IsWildcard { get; set; } @@ -32,7 +34,10 @@ public class FieldMappingInfo public string GetKeywordPath() { - if (IsKeyword) return Path; + if (IsKeyword) + { + return Path; + } // 如果是 text 类型且有 keyword 子字段 if (IsText && Properties?.ContainsKey("keyword") == true) diff --git a/aspnet-core/framework/elasticsearch/LINGYUN.Abp.Elasticsearch/LINGYUN/Abp/Elasticsearch/IIndexMappingProvider.cs b/aspnet-core/framework/elasticsearch/LINGYUN.Abp.Elasticsearch/LINGYUN/Abp/Elasticsearch/IIndexMappingProvider.cs index 0a017cde2..3104e79b4 100644 --- a/aspnet-core/framework/elasticsearch/LINGYUN.Abp.Elasticsearch/LINGYUN/Abp/Elasticsearch/IIndexMappingProvider.cs +++ b/aspnet-core/framework/elasticsearch/LINGYUN.Abp.Elasticsearch/LINGYUN/Abp/Elasticsearch/IIndexMappingProvider.cs @@ -8,4 +8,8 @@ public interface IIndexMappingProvider Task GetMappingAsync( string indexPattern, CancellationToken cancellationToken = default); + + Task GetMappingAsync( + string indexPattern, + CancellationToken cancellationToken = default); } diff --git a/aspnet-core/framework/elasticsearch/LINGYUN.Abp.Elasticsearch/LINGYUN/Abp/Elasticsearch/IndexMappingInfo.cs b/aspnet-core/framework/elasticsearch/LINGYUN.Abp.Elasticsearch/LINGYUN/Abp/Elasticsearch/IndexMappingInfo.cs index f0887d72d..bf33d01dc 100644 --- a/aspnet-core/framework/elasticsearch/LINGYUN.Abp.Elasticsearch/LINGYUN/Abp/Elasticsearch/IndexMappingInfo.cs +++ b/aspnet-core/framework/elasticsearch/LINGYUN.Abp.Elasticsearch/LINGYUN/Abp/Elasticsearch/IndexMappingInfo.cs @@ -1,5 +1,7 @@ -using System.Collections.Generic; +using System; +using System.Collections.Generic; using System.Linq; +using System.Linq.Expressions; namespace LINGYUN.Abp.Elasticsearch; /// @@ -12,15 +14,25 @@ public class IndexMappingInfo /// public string IndexName { get; set; } = string.Empty; + /// + /// 文档类型 + /// + public Type? DocumentType { get; set; } + /// /// 所有字段映射（扁平化） /// - public Dictionary Fields { get; set; } = new(); + public Dictionary Fields { get; set; } = new Dictionary(StringComparer.CurrentCultureIgnoreCase); + + /// + /// 按 CLR 属性路径索引的字段映射 + /// + public Dictionary ClrFields { get; set; } = new Dictionary(StringComparer.CurrentCultureIgnoreCase); /// /// Nested 字段映射 /// - public Dictionary NestedFields { get; set; } = new(); + public Dictionary NestedFields { get; set; } = new Dictionary(StringComparer.CurrentCultureIgnoreCase); /// /// Keyword 字段列表 @@ -64,6 +76,33 @@ public class IndexMappingInfo return Fields.GetOrDefault(path); } + /// + /// 根据 CLR 属性路径获取字段映射信息 + /// + public FieldMappingInfo? GetFieldByClrPath(string clrPath) + { + return ClrFields.GetOrDefault(clrPath); + } + + /// + /// 根据 CLR 属性表达式获取字段映射信息 + /// + public FieldMappingInfo? GetFieldByExpression(Expression> expression) + { + var clrPath = GetPropertyPath(expression); + return GetFieldByClrPath(clrPath); + } + + /// + /// 根据 CLR 属性表达式获取 ES 字段路径 + /// + public string? GetElasticsearchFieldPath(Expression> expression) + { + var clrPath = GetPropertyPath(expression); + var field = GetFieldByClrPath(clrPath); + return field?.Path; + } + /// /// 判断是否为 Nested 字段 /// @@ -136,4 +175,43 @@ public class IndexMappingInfo return null; } + /// + /// 将 CLR 属性路径转换为 ES 字段路径 + /// + public string? ConvertClrPathToElasticsearchPath(string clrPath) + { + var field = GetFieldByClrPath(clrPath); + return field?.Path; + } + + /// + /// 将 ES 字段路径转换为 CLR 属性路径 + /// + public string? ConvertElasticsearchPathToClrPath(string esPath) + { + var field = GetField(esPath); + return field?.ClrPath; + } + + private static string GetPropertyPath(Expression> expression) + { + var parts = new List(); + var currentExpression = expression.Body; + + while (currentExpression is MemberExpression memberExpression) + { + parts.Insert(0, memberExpression.Member.Name); + currentExpression = memberExpression.Expression!; + } + + // 处理转换表达式 + if (currentExpression is UnaryExpression unaryExpression && + unaryExpression.NodeType == ExpressionType.Convert && + unaryExpression.Operand is MemberExpression convertMemberExpression) + { + parts.Insert(0, convertMemberExpression.Member.Name); + } + + return string.Join(".", parts); + } } \ No newline at end of file diff --git a/aspnet-core/framework/elasticsearch/LINGYUN.Abp.Elasticsearch/LINGYUN/Abp/Elasticsearch/IndexMappingInfoExtensions.cs b/aspnet-core/framework/elasticsearch/LINGYUN.Abp.Elasticsearch/LINGYUN/Abp/Elasticsearch/IndexMappingInfoExtensions.cs new file mode 100644 index 000000000..18a20bbd9 --- /dev/null +++ b/aspnet-core/framework/elasticsearch/LINGYUN.Abp.Elasticsearch/LINGYUN/Abp/Elasticsearch/IndexMappingInfoExtensions.cs @@ -0,0 +1,29 @@ +using System; +using System.Linq.Expressions; + +namespace LINGYUN.Abp.Elasticsearch; + +public static class IndexMappingInfoExtensions +{ + public static string? GetElasticsearchFieldPath( + this IndexMappingInfo mappingInfo, + Expression> expression) + { + return mappingInfo.GetElasticsearchFieldPath(expression); + } + + public static FieldMappingInfo? GetFieldMapping( + this IndexMappingInfo mappingInfo, + Expression> expression) + { + return mappingInfo.GetFieldByExpression(expression); + } + + public static string? GetKeywordPath( + this IndexMappingInfo mappingInfo, + Expression> expression) + { + var field = mappingInfo.GetFieldByExpression(expression); + return field?.GetKeywordPath(); + } +} diff --git a/aspnet-core/framework/logging/LINGYUN.Abp.Logging.Serilog.Elasticsearch/LINGYUN/Abp/AuditLogging/Serilog/Elasticsearch/SerilogElasticsearchLoggingManager.cs b/aspnet-core/framework/logging/LINGYUN.Abp.Logging.Serilog.Elasticsearch/LINGYUN/Abp/AuditLogging/Serilog/Elasticsearch/SerilogElasticsearchLoggingManager.cs index a2732ba45..d5cd2eb65 100644 --- a/aspnet-core/framework/logging/LINGYUN.Abp.Logging.Serilog.Elasticsearch/LINGYUN/Abp/AuditLogging/Serilog/Elasticsearch/SerilogElasticsearchLoggingManager.cs +++ b/aspnet-core/framework/logging/LINGYUN.Abp.Logging.Serilog.Elasticsearch/LINGYUN/Abp/AuditLogging/Serilog/Elasticsearch/SerilogElasticsearchLoggingManager.cs @@ -1,5 +1,5 @@ using Elastic.Clients.Elasticsearch; -using Elastic.Clients.Elasticsearch.QueryDsl; +using Elastic.Transport.Diagnostics.Auditing; using LINGYUN.Abp.Elasticsearch; using LINGYUN.Linq.Dynamic.Queryable; using Microsoft.Extensions.Logging; @@ -10,6 +10,7 @@ using Serilog.Formatting.Elasticsearch; using System; using System.Collections.Generic; using System.Linq; +using System.Linq.Expressions; using System.Text.RegularExpressions; using System.Threading; using System.Threading.Tasks; @@ -37,7 +38,6 @@ public class SerilogElasticsearchLoggingManager : ILoggingManager, ISingletonDep private readonly ICurrentTenant _currentTenant; private readonly AbpLoggingSerilogElasticsearchOptions _options; private readonly IElasticsearchClientFactory _clientFactory; - private readonly IIndexMappingProvider _indexMappingProvider; private readonly IExpressionQueryService _expressionQueryService; private readonly IObjectMapper _objectMapper; @@ -48,7 +48,6 @@ public class SerilogElasticsearchLoggingManager : ILoggingManager, ISingletonDep ICurrentTenant currentTenant, IOptions options, IElasticsearchClientFactory clientFactory, - IIndexMappingProvider indexMappingProvider, IExpressionQueryService expressionQueryService, IObjectMapper objectMapper) { @@ -56,7 +55,6 @@ public class SerilogElasticsearchLoggingManager : ILoggingManager, ISingletonDep _objectMapper = objectMapper; _currentTenant = currentTenant; _clientFactory = clientFactory; - _indexMappingProvider = indexMappingProvider; _expressionQueryService = expressionQueryService; _options = options.Value; @@ -67,12 +65,11 @@ public class SerilogElasticsearchLoggingManager : ILoggingManager, ISingletonDep ISpecification specification, CancellationToken cancellationToken = default) { - var indexName = CreateIndex(); var converter = new ExpressionQueryConverter(_defaultTypeMap); var expression = converter.Convert(specification.ToExpression()); return await _expressionQueryService.GetCountAsync( - indexName, + CreateIndex(), expression, cancellationToken); } @@ -82,35 +79,28 @@ public class SerilogElasticsearchLoggingManager : ILoggingManager, ISingletonDep string? sorting = null, int maxResultCount = 50, int skipCount = 0, + bool includeDetails = false, CancellationToken cancellationToken = default) { - var indexName = CreateIndex(); var converter = new ExpressionQueryConverter(_defaultTypeMap); var expression = converter.Convert(specification.ToExpression()); - - var sortingField = sorting; - if (sortingField.IsNullOrWhiteSpace()) + if (sorting.IsNullOrWhiteSpace()) { - var indexMapping = await _indexMappingProvider.GetMappingAsync(indexName, cancellationToken); - if (indexMapping != null) - { - var sortingFieldMap = indexMapping.Fields - .Where(x => x.Key.Equals(sortingField, StringComparison.CurrentCultureIgnoreCase)) - .Select(x => x.Value) - .FirstOrDefault(); - if (sortingFieldMap != null) - { - sortingField = sortingFieldMap.Path; - } - } + sorting = ElasticsearchJsonFormatter.TimestampPropertyName; } var serilogLogs = await _expressionQueryService.GetListAsync( - indexName, + CreateIndex(), expression, - sortingField, + sorting, maxResultCount, skipCount, + sourceExcludes: includeDetails == false + ? Fields.FromFields( + [ + new Field("exceptions"), + ]) + : null, cancellationToken: cancellationToken); return _objectMapper.Map, List>(serilogLogs); @@ -126,83 +116,18 @@ public class SerilogElasticsearchLoggingManager : ILoggingManager, ISingletonDep string id, CancellationToken cancellationToken = default) { - var indexName = CreateIndex(); var client = _clientFactory.Create(); - var indexMapping = await _indexMappingProvider.GetMappingAsync(indexName, cancellationToken); - - SearchResponse response; - - if (_currentTenant.IsAvailable) - { - /* - "query": { - "bool": { - "must": [ - { - "term": { - "fields.TenantId.keyword": { - "value": _currentTenant.GetId() - } - } - }, - { - "term": { - "fields.UniqueId": { - "value": "1474021081433481216" - } - } - } - ] - } - } - */ - response = await client.SearchAsync( - dsl => - dsl.Indices(CreateIndex()) - .Query( - (q) => q.Bool( - (b) => b.Must( - (s) => s.Term( - (t) => t.Field(GetField(indexMapping, "fields.UniqueId")).Value(id)), - (s) => s.Term( - (t) => t.Field(GetField(indexMapping, "fields.TenantId")).Value(_currentTenant.GetId().ToString()))))) - .Size(1), - cancellationToken); - } - else - { - /* - "query": { - "bool": { - "must": [ - { - "term": { - "fields.UniqueId": { - "value": "1474021081433481216" - } - } - } - ] - } - } - */ - response = await client.SearchAsync( - dsl => - dsl.Indices(CreateIndex()) - .Query( - (q) => q.Bool( - (b) => b.Must( - (s) => s.Term( - (t) => t.Field(GetField(indexMapping, "fields.UniqueId")).Value(id))))) - .Size(1), - cancellationToken); - if (response.TryGetErrorMessage(out var errorMessage)) - { - Logger.LogWarning("Query logs failed: {errorMessage}", errorMessage); - } - } + Expression> expression = x => x.Fields.UniqueId == long.Parse(id); + expression = expression.AndIf(_currentTenant.IsAvailable, x => x.Fields.TenantId == _currentTenant.Id); + + var serilogs = await _expressionQueryService.GetListAsync( + CreateIndex(), + x => x.Fields.UniqueId == long.Parse(id), + sorting: $"{ElasticsearchJsonFormatter.TimestampPropertyName} DESC", + maxResultCount: 1, + cancellationToken: cancellationToken); - return _objectMapper.Map(response.Documents.FirstOrDefault()); + return _objectMapper.Map(serilogs.FirstOrDefault()); } public async virtual Task GetCountAsync( @@ -221,36 +146,30 @@ public class SerilogElasticsearchLoggingManager : ILoggingManager, ISingletonDep bool? hasException = null, CancellationToken cancellationToken = default) { - var indexName = CreateIndex(); var client = _clientFactory.Create(); - var indexMapping = await _indexMappingProvider.GetMappingAsync(indexName, cancellationToken); - var querys = BuildQueryDescriptor( - indexMapping, - startTime, - endTime, - level, - machineName, - environment, - application, - context, - requestId, - requestPath, - correlationId, - processId, - threadId, - hasException); + Expression> expression = _ => true; + + expression = expression + .AndIf(startTime.HasValue, x => x.TimeStamp >= _clock.Normalize(startTime!.Value)) + .AndIf(endTime.HasValue, x => x.TimeStamp <= _clock.Normalize(endTime!.Value)) + .AndIf(level.HasValue, x => x.Level == GetLogEventLevel(level!.Value)) + .AndIf(!machineName.IsNullOrWhiteSpace(), x => x.Fields.MachineName!.Contains(machineName!)) + .AndIf(!environment.IsNullOrWhiteSpace(), x => x.Fields.Environment!.Contains(environment!)) + .AndIf(!application.IsNullOrWhiteSpace(), x => x.Fields.Application!.Contains(application!)) + .AndIf(!context.IsNullOrWhiteSpace(), x => x.Fields.Context == context) + .AndIf(!requestId.IsNullOrWhiteSpace(), x => x.Fields.RequestId == requestId) + .AndIf(!requestPath.IsNullOrWhiteSpace(), x => x.Fields.RequestPath!.StartsWith(requestPath!)) + .AndIf(!correlationId.IsNullOrWhiteSpace(), x => x.Fields.CorrelationId!.Contains(correlationId!)) + .AndIf(processId.HasValue, x => x.Fields.ProcessId == processId) + .AndIf(threadId.HasValue, x => x.Fields.ThreadId == threadId) + .AndIf(hasException == true, x => x.Exceptions != null) + .AndIf(hasException == false, x => x.Exceptions == null); - var response = await client.CountAsync((dsl) => - dsl.Indices(indexName) - .Query(log => log.Bool(b => b.Must(querys.ToArray()))), + return await _expressionQueryService.GetCountAsync( + CreateIndex(), + expression, cancellationToken); - if (response.TryGetErrorMessage(out var errorMessage)) - { - Logger.LogWarning("Query log count failed: {errorMessage}", errorMessage); - } - - return response.Count; } /// @@ -295,308 +214,52 @@ public class SerilogElasticsearchLoggingManager : ILoggingManager, ISingletonDep bool includeDetails = false, CancellationToken cancellationToken = default) { - var indexName = CreateIndex(); var client = _clientFactory.Create(); - var indexMapping = await _indexMappingProvider.GetMappingAsync(indexName, cancellationToken); - - var sorts = GetOrDefaultSort(indexMapping, sorting); - - var querys = BuildQueryDescriptor( - indexMapping, - startTime, - endTime, - level, - machineName, - environment, - application, - context, - requestId, - requestPath, - correlationId, - processId, - threadId, - hasException); - - var query = new BoolQuery { Must = querys }; - - var serilogLogs = skipCount >= 10000 && sorts != null - ? await SearchAfterSerilogLogs(client, query, sorts, maxResultCount, skipCount, cancellationToken) - : await SearchFromSizeSerilogLogs(client, query, sorts, maxResultCount, skipCount, cancellationToken); - - return _objectMapper.Map, List>(serilogLogs); - } - - protected virtual List BuildQueryDescriptor( - IndexMappingInfo indexMappingInfo, - DateTime? startTime = null, - DateTime? endTime = null, - LogLevel? level = null, - string? machineName = null, - string? environment = null, - string? application = null, - string? context = null, - string? requestId = null, - string? requestPath = null, - string? correlationId = null, - int? processId = null, - int? threadId = null, - bool? hasException = null) - { - var queries = new List(); - - if (_currentTenant.IsAvailable) - { - queries.Add(new TermQuery(GetField(indexMappingInfo, "fields.TenantId"), _currentTenant.GetId().ToString())); - } - if (startTime.HasValue) - { - queries.Add(new DateRangeQuery(GetField(indexMappingInfo, "@timestamp")) - { - Gte = _clock.Normalize(startTime.Value), - }); - } - if (endTime.HasValue) - { - queries.Add(new DateRangeQuery(GetField(indexMappingInfo, "@timestamp")) - { - Lte = _clock.Normalize(endTime.Value), - }); - } - if (level.HasValue) - { - queries.Add(new TermQuery(GetField(indexMappingInfo, "level"), GetLogEventLevel(level.Value).ToString())); - } - if (!machineName.IsNullOrWhiteSpace()) - { - // 模糊匹配 - queries.Add(new WildcardQuery(GetField(indexMappingInfo, "fields.MachineName")) - { - Value = $"*{machineName}*" - }); - } - if (!environment.IsNullOrWhiteSpace()) - { - // 模糊匹配 - queries.Add(new WildcardQuery(GetField(indexMappingInfo, "fields.EnvironmentName")) - { - Value = $"*{environment}*" - }); - } - if (!application.IsNullOrWhiteSpace()) - { - // 模糊匹配 - queries.Add(new WildcardQuery(GetField(indexMappingInfo, "fields.ApplicationName")) - { - Value = $"*{application}*" - }); - } - if (!context.IsNullOrWhiteSpace()) - { - queries.Add(new TermQuery(GetField(indexMappingInfo, "fields.SourceContext"), context)); - } - if (!requestId.IsNullOrWhiteSpace()) - { - queries.Add(new TermQuery(GetField(indexMappingInfo, "fields.RequestId"), requestId)); - } - if (!requestPath.IsNullOrWhiteSpace()) - { - // 前缀匹配 - queries.Add(new MatchPhrasePrefixQuery(GetField(indexMappingInfo, "fields.RequestPath"), requestPath)); - } - if (!correlationId.IsNullOrWhiteSpace()) - { - // 模糊匹配 - queries.Add(new WildcardQuery(GetField(indexMappingInfo, "fields.CorrelationId")) - { - Value = $"*{correlationId}*" - }); - } - if (processId.HasValue) - { - queries.Add(new TermQuery(GetField(indexMappingInfo, "fields.ProcessId"), FieldValue.FromValue(processId.Value))); - } - if (threadId.HasValue) - { - queries.Add(new TermQuery(GetField(indexMappingInfo, "fields.ThreadId"), FieldValue.FromValue(threadId.Value))); - } - - if (hasException.HasValue) - { - if (hasException.Value) - { - /* 存在exceptions字段则就是有异常信息 - * "exists": { - "field": "exceptions" - } - */ - queries.Add(new ExistsQuery(GetField(indexMappingInfo, "fields.Exceptions"))); - } - else - { - // 不存在 exceptions字段就是没有异常信息的消息 - /* - * "bool": { - "must_not": [ - { - "exists": { - "field": "exceptions" - } - } - ] - } - */ - queries.Add(new BoolQuery - { - MustNot = new List - { - new ExistsQuery(GetField(indexMappingInfo, "fields.Exceptions")) - } - }); - } - } - - return queries; - } - - private async Task> SearchFromSizeSerilogLogs( - ElasticsearchClient client, - Query query, - SortOptions[]? sorts = null, - int maxResultCount = 50, - int skipCount = 0, - CancellationToken cancellationToken = default) - { - var searchResponse = await client.SearchAsync(dsl => - { - dsl.Indices(CreateIndex()) - .Query(query) - .From(skipCount) - .Size(maxResultCount); - if (sorts != null) - { - dsl.Sort(sorts); - } - }, cancellationToken); - - if (searchResponse.TryGetErrorMessage(out var errorMessage)) - { - Logger.LogWarning("Query log failed: {errorMessage}", errorMessage); - return []; - } - - return searchResponse.Documents.ToList(); - } - - private async Task> SearchAfterSerilogLogs( - ElasticsearchClient client, - Query query, - SortOptions[] sorts, - int maxResultCount = 50, - int skipCount = 0, - CancellationToken cancellationToken = default) - { - var searchAfter = await GetSearchAfterValue( - client, - query, - sorts, - skipCount, - cancellationToken); - - if (searchAfter == null || !searchAfter.Any()) - { - return []; - } - - var searchResponse = await client.SearchAsync(dsl => - { - dsl.Indices(CreateIndex()) - .Query(query) - .Sort(sorts) - .Size(maxResultCount) - .SearchAfter(searchAfter); - }, cancellationToken); - - if (searchResponse.TryGetErrorMessage(out var errorMessage)) - { - Logger.LogWarning("Query log failed: {errorMessage}", errorMessage); - return []; - } - - return searchResponse.Documents.ToList(); - } - - private async Task?> GetSearchAfterValue( - ElasticsearchClient client, - Query query, - SortOptions[] sorts, - int skipCount, - CancellationToken cancellationToken = default) - { - // 10000以内直接取最后一条数据 - if (skipCount < 10000) - { - var response = await client.SearchAsync( - dsl => dsl.Indices(CreateIndex()) - .Query(query) - .Sort(sorts) - .SourceIncludes(x => x.Level) - .From(skipCount) - .Size(1), - cancellationToken); - - if (!response.IsSuccess() || response.Hits == null || !response.Hits.Any()) - { - return null; - } - - var hit = response.Hits.FirstOrDefault(); - return hit?.Sort?.ToList(); - } - - // 获取第9999条数据Hits作为searchAfter - var firstResponse = await client.SearchAsync( - dsl => dsl.Indices(CreateIndex()) - .Query(query) - .Sort(sorts) - .SourceIncludes(x => x.Level) - .From(9999) - .Size(1), - cancellationToken); - - if (!firstResponse.IsSuccess() || firstResponse.Hits == null || !firstResponse.Hits.Any()) + if (sorting.IsNullOrWhiteSpace()) { - return null; + sorting = $"{ElasticsearchJsonFormatter.TimestampPropertyName} DESC"; } - - var firstHit = firstResponse.Hits.FirstOrDefault(); - if (firstHit?.Sort == null || !firstHit.Sort.Any()) + // 额外处理一下timestamp字段 + else if (sorting.Contains("timestamp", StringComparison.CurrentCultureIgnoreCase)) { - return null; + sorting = sorting + .Replace("timestamp", ElasticsearchJsonFormatter.TimestampPropertyName, StringComparison.CurrentCultureIgnoreCase) + .Replace("@@", "@"); } - // 获取skipCount最近一条数据作为searchAfter - var secondResponse = await client.SearchAsync( - dsl => dsl.Indices(CreateIndex()) - .Query(query) - // 反转排序取第一个数据作为起始索引 - .Sort(sorts.ReverseSort()!.ToArray()) - .SourceIncludes(x => x.Level) - .SearchAfter(firstHit.Sort.ToList()) - .Size(1), - cancellationToken); - - if (!secondResponse.IsSuccess() || secondResponse.Hits == null || !secondResponse.Hits.Any()) - { - return null; - } + Expression> expression = _ => true; + + expression = expression + .AndIf(startTime.HasValue, x => x.TimeStamp >= _clock.Normalize(startTime!.Value)) + .AndIf(endTime.HasValue, x => x.TimeStamp <= _clock.Normalize(endTime!.Value)) + .AndIf(level.HasValue, x => x.Level == GetLogEventLevel(level!.Value)) + .AndIf(!machineName.IsNullOrWhiteSpace(), x => x.Fields.MachineName!.Contains(machineName!)) + .AndIf(!environment.IsNullOrWhiteSpace(), x => x.Fields.Environment!.Contains(environment!)) + .AndIf(!application.IsNullOrWhiteSpace(), x => x.Fields.Application!.Contains(application!)) + .AndIf(!context.IsNullOrWhiteSpace(), x => x.Fields.Context == context) + .AndIf(!requestId.IsNullOrWhiteSpace(), x => x.Fields.RequestId == requestId) + .AndIf(!requestPath.IsNullOrWhiteSpace(), x => x.Fields.RequestPath!.StartsWith(requestPath!)) + .AndIf(!correlationId.IsNullOrWhiteSpace(), x => x.Fields.CorrelationId!.Contains(correlationId!)) + .AndIf(processId.HasValue, x => x.Fields.ProcessId == processId) + .AndIf(threadId.HasValue, x => x.Fields.ThreadId == threadId) + .AndIf(hasException == true, x => x.Exceptions != null) + .AndIf(hasException == false, x => x.Exceptions == null); - var lastHit = secondResponse.Hits.LastOrDefault(); - if (lastHit?.Sort == null || !lastHit.Sort.Any()) - { - return null; - } + var serilogLogs = await _expressionQueryService.GetListAsync( + CreateIndex(), + expression, + sorting: sorting, + maxResultCount: maxResultCount, + skipCount: skipCount, + sourceExcludes: includeDetails == false + ? Fields.FromFields( + [ + new Field("exceptions"), + ]) + : null, + cancellationToken: cancellationToken); - return lastHit.Sort.ToList(); + return _objectMapper.Map, List>(serilogLogs); } protected virtual string CreateIndex(DateTimeOffset? offset = null) @@ -620,43 +283,4 @@ public class SerilogElasticsearchLoggingManager : ILoggingManager, ISingletonDep _ => LogEventLevel.Verbose, }; } - - private static SortOptions[]? GetOrDefaultSort(IndexMappingInfo indexMappingInfo, string? sorting = null) - { - var sortOrder = !sorting.IsNullOrWhiteSpace() && sorting.EndsWith("asc", StringComparison.InvariantCultureIgnoreCase) - ? SortOrder.Asc : SortOrder.Desc; - sorting = !sorting.IsNullOrWhiteSpace() - ? sorting.Split()[0] - : ElasticsearchJsonFormatter.TimestampPropertyName; - - SortOptions[]? sorts = null; - var sortingFieldMap = indexMappingInfo.Fields - .Where(x => x.Key.Equals(sorting, StringComparison.CurrentCultureIgnoreCase)) - .Select(x => x.Value) - .FirstOrDefault(); - if (sortingFieldMap != null) - { - sorting = sortingFieldMap.Path; - } - if (!sorting.IsNullOrWhiteSpace()) - { - sorts = new SortOptions[1] - { - new SortOptions - { - Field = new FieldSort(new Field(sorting)) - { - Order = sortOrder, - }, - } - }; - } - - return sorts; - } - - private static string GetField(IndexMappingInfo indexMappingInfo, string fieldFullPath) - { - return indexMappingInfo.GetExactFieldPath(fieldFullPath); - } } diff --git a/aspnet-core/framework/logging/LINGYUN.Abp.Logging.Serilog.Elasticsearch/System/Linq/Expressions/ExpressionFuncExtensions.cs b/aspnet-core/framework/logging/LINGYUN.Abp.Logging.Serilog.Elasticsearch/System/Linq/Expressions/ExpressionFuncExtensions.cs new file mode 100644 index 000000000..fd7a017ea --- /dev/null +++ b/aspnet-core/framework/logging/LINGYUN.Abp.Logging.Serilog.Elasticsearch/System/Linq/Expressions/ExpressionFuncExtensions.cs @@ -0,0 +1,32 @@ +using Volo.Abp.Specifications; + +namespace System.Linq.Expressions; + +internal static class ExpressionFuncExtensions +{ + public static Expression> AndIf( + this Expression> first, + bool condition, + Expression> second) + { + if (condition) + { + return ExpressionFuncExtender.And(first, second); + } + + return first; + } + + public static Expression> OrIf( + this Expression> first, + bool condition, + Expression> second) + { + if (condition) + { + return ExpressionFuncExtender.Or(first, second); + } + + return first; + } +} diff --git a/aspnet-core/framework/logging/LINGYUN.Abp.Logging/LINGYUN/Abp/AuditLogging/DefaultLoggingManager.cs b/aspnet-core/framework/logging/LINGYUN.Abp.Logging/LINGYUN/Abp/AuditLogging/DefaultLoggingManager.cs index 01e50f8fc..0daccc77a 100644 --- a/aspnet-core/framework/logging/LINGYUN.Abp.Logging/LINGYUN/Abp/AuditLogging/DefaultLoggingManager.cs +++ b/aspnet-core/framework/logging/LINGYUN.Abp.Logging/LINGYUN/Abp/AuditLogging/DefaultLoggingManager.cs @@ -83,6 +83,7 @@ public class DefaultLoggingManager : ILoggingManager, ISingletonDependency string? sorting = null, int maxResultCount = 50, int skipCount = 0, + bool includeDetails = false, CancellationToken cancellationToken = default) { Logger.LogDebug("No logging manager is available!"); diff --git a/aspnet-core/framework/logging/LINGYUN.Abp.Logging/LINGYUN/Abp/AuditLogging/ILoggingManager.cs b/aspnet-core/framework/logging/LINGYUN.Abp.Logging/LINGYUN/Abp/AuditLogging/ILoggingManager.cs index c81f5e1c9..0b2859391 100644 --- a/aspnet-core/framework/logging/LINGYUN.Abp.Logging/LINGYUN/Abp/AuditLogging/ILoggingManager.cs +++ b/aspnet-core/framework/logging/LINGYUN.Abp.Logging/LINGYUN/Abp/AuditLogging/ILoggingManager.cs @@ -58,5 +58,6 @@ public interface ILoggingManager string? sorting = null, int maxResultCount = 50, int skipCount = 0, + bool includeDetails = false, CancellationToken cancellationToken = default); }