From 078dbb11c0886f9fd8e9df1d22c74c7ec5c19476 Mon Sep 17 00:00:00 2001 From: colin Date: Thu, 8 Jan 2026 18:00:05 +0800 Subject: [PATCH] fix(auditing): Fix the ES query for auditlog MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - 审计日志ES集成客户端库查询语法变更 --- .../AbpAuditLoggingElasticsearchOptions.cs | 4 +- .../ElasticsearchAuditLogManager.cs | 146 +++++----- .../ElasticsearchEntityChangeStore.cs | 260 +++++++++--------- .../ElasticsearchSecurityLogManager.cs | 59 ++-- .../Elasticsearch/IndexInitializer.cs | 168 ++++++++--- 5 files changed, 364 insertions(+), 273 deletions(-) diff --git a/aspnet-core/framework/auditing/LINGYUN.Abp.AuditLogging.Elasticsearch/LINGYUN/Abp/AuditLogging/Elasticsearch/AbpAuditLoggingElasticsearchOptions.cs b/aspnet-core/framework/auditing/LINGYUN.Abp.AuditLogging.Elasticsearch/LINGYUN/Abp/AuditLogging/Elasticsearch/AbpAuditLoggingElasticsearchOptions.cs index 7773ae201..6d54ccd6b 100644 --- a/aspnet-core/framework/auditing/LINGYUN.Abp.AuditLogging.Elasticsearch/LINGYUN/Abp/AuditLogging/Elasticsearch/AbpAuditLoggingElasticsearchOptions.cs +++ b/aspnet-core/framework/auditing/LINGYUN.Abp.AuditLogging.Elasticsearch/LINGYUN/Abp/AuditLogging/Elasticsearch/AbpAuditLoggingElasticsearchOptions.cs @@ -1,4 +1,4 @@ -using Nest; +using Elastic.Clients.Elasticsearch.IndexManagement; namespace LINGYUN.Abp.AuditLogging.Elasticsearch; @@ -6,7 +6,7 @@ public class AbpAuditLoggingElasticsearchOptions { public const string DefaultIndexPrefix = "auditlogging"; public string IndexPrefix { get; set; } - public IIndexSettings IndexSettings { get; set; } + public IndexSettings IndexSettings { get; set; } public AbpAuditLoggingElasticsearchOptions() { diff --git a/aspnet-core/framework/auditing/LINGYUN.Abp.AuditLogging.Elasticsearch/LINGYUN/Abp/AuditLogging/Elasticsearch/ElasticsearchAuditLogManager.cs b/aspnet-core/framework/auditing/LINGYUN.Abp.AuditLogging.Elasticsearch/LINGYUN/Abp/AuditLogging/Elasticsearch/ElasticsearchAuditLogManager.cs index 6dca20c17..2c7a8cd25 100644 --- a/aspnet-core/framework/auditing/LINGYUN.Abp.AuditLogging.Elasticsearch/LINGYUN/Abp/AuditLogging/Elasticsearch/ElasticsearchAuditLogManager.cs +++ b/aspnet-core/framework/auditing/LINGYUN.Abp.AuditLogging.Elasticsearch/LINGYUN/Abp/AuditLogging/Elasticsearch/ElasticsearchAuditLogManager.cs @@ -1,8 +1,9 @@ -using LINGYUN.Abp.Elasticsearch; +using Elastic.Clients.Elasticsearch; +using Elastic.Clients.Elasticsearch.QueryDsl; +using LINGYUN.Abp.Elasticsearch; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Logging.Abstractions; using Microsoft.Extensions.Options; -using Nest; using System; using System.Collections.Generic; using System.Linq; @@ -82,8 +83,11 @@ public class ElasticsearchAuditLogManager : IAuditLogManager, ITransientDependen httpStatusCode); var response = await client.CountAsync(dsl => - dsl.Index(CreateIndex()) - .Query(log => log.Bool(b => b.Must(querys.ToArray()))), + dsl.Indices(CreateIndex()) + .Query(new BoolQuery + { + Must = querys + }), cancellationToken); return response.Count; @@ -113,7 +117,7 @@ public class ElasticsearchAuditLogManager : IAuditLogManager, ITransientDependen var client = _clientFactory.Create(); var sortOrder = !sorting.IsNullOrWhiteSpace() && sorting.EndsWith("asc", StringComparison.InvariantCultureIgnoreCase) - ? SortOrder.Ascending : SortOrder.Descending; + ? SortOrder.Asc : SortOrder.Desc; sorting = !sorting.IsNullOrWhiteSpace() ? sorting.Split()[0] : nameof(AuditLog.ExecutionTime); @@ -134,31 +138,32 @@ public class ElasticsearchAuditLogManager : IAuditLogManager, ITransientDependen hasException, httpStatusCode); - SourceFilterDescriptor SourceFilter(SourceFilterDescriptor selector) + var searchResponse = await client.SearchAsync(dsl => { - selector.IncludeAll(); + dsl.Indices(CreateIndex()) + .Query(new BoolQuery + { + Must = querys + }) + .Sort(s => s.Field(new FieldSort(GetField(sorting)) + { + Order = sortOrder + })) + .From(skipCount) + .Size(maxResultCount); + + // 字段过滤 if (!includeDetails) { - selector.Excludes(field => - field.Field(f => f.Actions) - .Field(f => f.Comments) - .Field(f => f.Exceptions) - .Field(f => f.EntityChanges)); + dsl.SourceExcludes( + ex => ex.Actions, + ex => ex.Comments, + ex => ex.Exceptions, + ex => ex.EntityChanges); } + }, cancellationToken); - return selector; - } - - var response = await client.SearchAsync(dsl => - dsl.Index(CreateIndex()) - .Query(log => log.Bool(b => b.Must(querys.ToArray()))) - .Source(SourceFilter) - .Sort(log => log.Field(GetField(sorting), sortOrder)) - .From(skipCount) - .Size(maxResultCount), - cancellationToken); - - return response.Documents.ToList(); + return searchResponse.Documents.ToList(); } public async virtual Task GetAsync( @@ -169,9 +174,8 @@ public class ElasticsearchAuditLogManager : IAuditLogManager, ITransientDependen var client = _clientFactory.Create(); var response = await client.GetAsync( - id, - dsl => - dsl.Index(CreateIndex()), + id.ToString(), + dsl => dsl.Index(CreateIndex()), cancellationToken); return response.Source; @@ -182,9 +186,8 @@ public class ElasticsearchAuditLogManager : IAuditLogManager, ITransientDependen var client = _clientFactory.Create(); await client.DeleteAsync( - id, - dsl => - dsl.Index(CreateIndex()), + id.ToString(), + dsl => dsl.Index(CreateIndex()), cancellationToken); } @@ -192,12 +195,13 @@ public class ElasticsearchAuditLogManager : IAuditLogManager, ITransientDependen { var client = _clientFactory.Create(); + var idValues = ids.Select(id => FieldValue.String(id.ToString())).ToList(); await client.DeleteByQueryAsync( - x => x.Index(CreateIndex()) + x => x.Indices(CreateIndex()) .Query(query => query.Terms(terms => terms.Field(field => field.Id) - .Terms(ids))), + .Terms(new TermsQueryField(idValues)))), cancellationToken); } @@ -217,7 +221,7 @@ public class ElasticsearchAuditLogManager : IAuditLogManager, ITransientDependen catch (Exception ex) { Logger.LogWarning("Could not save the audit log object: " + Environment.NewLine + auditInfo.ToString()); - Logger.LogException(ex, Microsoft.Extensions.Logging.LogLevel.Error); + Logger.LogException(ex, LogLevel.Error); } return ""; } @@ -239,14 +243,12 @@ public class ElasticsearchAuditLogManager : IAuditLogManager, ITransientDependen // 使用 Bulk 命令传输可能存在参数庞大的日志结构 var response = await client.BulkAsync( dsl => dsl.Index(CreateIndex()) - .Create(ct => - ct.Id(auditLog.Id) - .Document(auditLog))); + .Create(auditLog, ct => ct.Id(auditLog.Id))); return response.Items?.FirstOrDefault()?.Id; } - protected virtual List, QueryContainer>> BuildQueryDescriptor( + protected virtual List BuildQueryDescriptor( DateTime? startTime = null, DateTime? endTime = null, string httpMethod = null, @@ -262,55 +264,70 @@ public class ElasticsearchAuditLogManager : IAuditLogManager, ITransientDependen bool? hasException = null, HttpStatusCode? httpStatusCode = null) { - var querys = new List, QueryContainer>>(); + var queries = new List(); if (startTime.HasValue) { - querys.Add((log) => log.DateRange((q) => q.Field(GetField(nameof(AuditLog.ExecutionTime))).GreaterThanOrEquals(_clock.Normalize(startTime.Value)))); + queries.Add(new DateRangeQuery(GetField(nameof(AuditLog.ExecutionTime))) + { + Gte = _clock.Normalize(startTime.Value) + }); } if (endTime.HasValue) { - querys.Add((log) => log.DateRange((q) => q.Field(GetField(nameof(AuditLog.ExecutionTime))).LessThanOrEquals(_clock.Normalize(endTime.Value)))); + queries.Add(new DateRangeQuery(GetField(nameof(AuditLog.ExecutionTime))) + { + Lte = _clock.Normalize(endTime.Value) + }); } if (!httpMethod.IsNullOrWhiteSpace()) { - querys.Add((log) => log.Term((q) => q.Field(GetField(nameof(AuditLog.HttpMethod))).Value(httpMethod))); + queries.Add(new TermQuery(GetField(nameof(AuditLog.HttpMethod)), httpMethod)); } if (!url.IsNullOrWhiteSpace()) { - querys.Add((log) => log.Wildcard((q) => q.Field(GetField(nameof(AuditLog.Url))).Value($"*{url}*"))); + queries.Add(new WildcardQuery(GetField(nameof(AuditLog.Url))) + { + Value = $"*{url}*" + }); } if (userId.HasValue) { - querys.Add((log) => log.Term((q) => q.Field(GetField(nameof(AuditLog.UserId))).Value(userId))); + queries.Add(new TermQuery(GetField(nameof(AuditLog.UserId)), userId.Value.ToString())); } if (!userName.IsNullOrWhiteSpace()) { - querys.Add((log) => log.Term((q) => q.Field(GetField(nameof(AuditLog.UserName))).Value(userName))); + queries.Add(new TermQuery(GetField(nameof(AuditLog.UserName)), userName)); } if (!applicationName.IsNullOrWhiteSpace()) { - querys.Add((log) => log.Term((q) => q.Field(GetField(nameof(AuditLog.ApplicationName))).Value(applicationName))); + queries.Add(new TermQuery(GetField(nameof(AuditLog.ApplicationName)), applicationName)); } if (!correlationId.IsNullOrWhiteSpace()) { - querys.Add((log) => log.Term((q) => q.Field(GetField(nameof(AuditLog.CorrelationId))).Value(correlationId))); + queries.Add(new TermQuery(GetField(nameof(AuditLog.CorrelationId)), correlationId)); } if (!clientId.IsNullOrWhiteSpace()) { - querys.Add((log) => log.Term((q) => q.Field(GetField(nameof(AuditLog.ClientId))).Value(clientId))); + queries.Add(new TermQuery(GetField(nameof(AuditLog.ClientId)), clientId)); } if (!clientIpAddress.IsNullOrWhiteSpace()) { - querys.Add((log) => log.Term((q) => q.Field(GetField(nameof(AuditLog.ClientIpAddress))).Value(clientIpAddress))); + queries.Add(new TermQuery(GetField(nameof(AuditLog.ClientIpAddress)), clientIpAddress)); } if (maxExecutionDuration.HasValue) { - querys.Add((log) => log.Range((q) => q.Field(GetField(nameof(AuditLog.ExecutionDuration))).LessThanOrEquals(maxExecutionDuration))); + queries.Add(new NumberRangeQuery(GetField(nameof(AuditLog.ExecutionDuration))) + { + Lte = maxExecutionDuration.Value + }); } if (minExecutionDuration.HasValue) { - querys.Add((log) => log.Range((q) => q.Field(GetField(nameof(AuditLog.ExecutionDuration))).GreaterThanOrEquals(minExecutionDuration))); + queries.Add(new NumberRangeQuery(GetField(nameof(AuditLog.ExecutionDuration))) + { + Gte = minExecutionDuration.Value + }); } @@ -318,33 +335,26 @@ public class ElasticsearchAuditLogManager : IAuditLogManager, ITransientDependen { if (hasException.Value) { - querys.Add( - (q) => q.Bool( - (b) => b.Must( - (m) => m.Exists( - (e) => e.Field((f) => f.Exceptions))) - ) - ); + queries.Add(new ExistsQuery(GetField("Exceptions"))); } else { - querys.Add( - (q) => q.Bool( - (b) => b.MustNot( - (mn) => mn.Exists( - (e) => e.Field( - (f) => f.Exceptions))) - ) - ); + queries.Add(new BoolQuery + { + MustNot = new List + { + new ExistsQuery(GetField("Exceptions")) + } + }); } } if (httpStatusCode.HasValue) { - querys.Add((log) => log.Term((q) => q.Field(GetField(nameof(AuditLog.HttpStatusCode))).Value(httpStatusCode))); + queries.Add(new TermQuery(GetField(nameof(AuditLog.HttpStatusCode)), ((int)httpStatusCode.Value).ToString())); } - return querys; + return queries; } protected virtual string CreateIndex() diff --git a/aspnet-core/framework/auditing/LINGYUN.Abp.AuditLogging.Elasticsearch/LINGYUN/Abp/AuditLogging/Elasticsearch/ElasticsearchEntityChangeStore.cs b/aspnet-core/framework/auditing/LINGYUN.Abp.AuditLogging.Elasticsearch/LINGYUN/Abp/AuditLogging/Elasticsearch/ElasticsearchEntityChangeStore.cs index 78be1b4a6..6d17c8dc4 100644 --- a/aspnet-core/framework/auditing/LINGYUN.Abp.AuditLogging.Elasticsearch/LINGYUN/Abp/AuditLogging/Elasticsearch/ElasticsearchEntityChangeStore.cs +++ b/aspnet-core/framework/auditing/LINGYUN.Abp.AuditLogging.Elasticsearch/LINGYUN/Abp/AuditLogging/Elasticsearch/ElasticsearchEntityChangeStore.cs @@ -1,8 +1,9 @@ -using LINGYUN.Abp.Elasticsearch; +using Elastic.Clients.Elasticsearch; +using Elastic.Clients.Elasticsearch.QueryDsl; +using LINGYUN.Abp.Elasticsearch; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Logging.Abstractions; using Microsoft.Extensions.Options; -using Nest; using System; using System.Collections.Generic; using System.Linq; @@ -44,31 +45,34 @@ public class ElasticsearchEntityChangeStore : IEntityChangeStore, ITransientDepe { var client = _clientFactory.Create(); - var sortOrder = SortOrder.Descending; + var sortOrder = SortOrder.Desc; var querys = BuildQueryDescriptor(entityChangeId: entityChangeId); - static SourceFilterDescriptor SourceFilter(SourceFilterDescriptor selector) + var searchResponse = await client.SearchAsync(dsl => { - selector.IncludeAll() - .Excludes(field => - field.Field(f => f.Actions) - .Field(f => f.Comments) - .Field(f => f.Exceptions)); + dsl.Indices(CreateIndex()) + .From(0) + .Size(1) + .Query(new BoolQuery + { + Must = querys + }); - return selector; - } + dsl.Sort(s => s.Field(new FieldSort(GetField(nameof(EntityChange.ChangeTime))) + { + Order = sortOrder + })); - var response = await client.SearchAsync(dsl => - dsl.Index(CreateIndex()) - .Query(log => log.Bool(b => b.Must(querys.ToArray()))) - .Source(SourceFilter) - .Sort(log => log.Field(GetField(nameof(EntityChange.ChangeTime)), sortOrder)) - .From(0) - .Size(1), - cancellationToken); + dsl.SourceIncludes(ix => ix.EntityChanges); - var auditLog = response.Documents.FirstOrDefault(); + dsl.SourceExcludes( + ex => ex.Actions, + ex => ex.Comments, + ex => ex.Exceptions); + }, cancellationToken); + + var auditLog = searchResponse.Documents.FirstOrDefault(); if (auditLog != null) { return auditLog @@ -99,27 +103,25 @@ public class ElasticsearchEntityChangeStore : IEntityChangeStore, ITransientDepe entityId, entityTypeFullName); - SourceFilterDescriptor SourceFilter(SourceFilterDescriptor selector) + var searchResponse = await client.SearchAsync(dsl => { - return selector - .Includes(field => - field.Field(f => f.UserName) - .Field(f => f.EntityChanges)) - .Excludes(field => - field.Field(f => f.Actions) - .Field(f => f.Comments) - .Field(f => f.Exceptions)); - } - - var response = await client.SearchAsync(dsl => - dsl.Index(CreateIndex()) - .Query(log => log.Bool(b => b.Must(querys.ToArray()))) - .Source(SourceFilter) + dsl.Indices(CreateIndex()) + .Query(new BoolQuery + { + Must = querys + }) + .SourceIncludes( + ix => ix.UserName, + ix => ix.EntityChanges) + .SourceExcludes( + ex => ex.Actions, + ex => ex.Comments, + ex => ex.Exceptions) .From(0) - .Size(1000), - cancellationToken); + .Size(1000); + }, cancellationToken); - var auditLogs = response.Documents.ToList(); + var auditLogs = searchResponse.Documents.ToList(); return auditLogs.Sum(log => log.EntityChanges.Count); } @@ -143,7 +145,7 @@ public class ElasticsearchEntityChangeStore : IEntityChangeStore, ITransientDepe var client = _clientFactory.Create(); var sortOrder = !sorting.IsNullOrWhiteSpace() && sorting.EndsWith("asc", StringComparison.InvariantCultureIgnoreCase) - ? SortOrder.Ascending : SortOrder.Descending; + ? SortOrder.Asc : SortOrder.Desc; sorting = !sorting.IsNullOrWhiteSpace() ? sorting.Split()[0] : nameof(EntityChange.ChangeTime); @@ -156,33 +158,28 @@ public class ElasticsearchEntityChangeStore : IEntityChangeStore, ITransientDepe entityId, entityTypeFullName); - SourceFilterDescriptor SourceFilter(SourceFilterDescriptor selector) + var searchResponse = await client.SearchAsync(dsl => { - selector - .Includes(field => - field.Field(f => f.UserName) - .Field(f => f.EntityChanges)); + dsl.Indices(CreateIndex()) + .Query(new BoolQuery + { + Must = querys + }) + .SourceIncludes( + ix => ix.UserName, + ix => ix.EntityChanges) + .From(0) + .Size(1000); if (includeDetails) { - selector.Includes(field => - field.Field(f => f.Actions) - .Field(f => f.Comments) - .Field(f => f.Exceptions)); + dsl.SourceExcludes( + ex => ex.Actions, + ex => ex.Comments, + ex => ex.Exceptions); } + }, cancellationToken); - return selector; - } - - var response = await client.SearchAsync(dsl => - dsl.Index(CreateIndex()) - .Query(log => log.Bool(b => b.Must(querys.ToArray()))) - .Source(SourceFilter) - .Sort(log => log.Field(GetField(nameof(EntityChange.ChangeTime)), sortOrder)) - .From(0) - .Size(1000), - cancellationToken); - - var auditLogs = response.Documents.ToList(); + var auditLogs = searchResponse.Documents.ToList(); if (auditLogs.Any()) { var groupAuditLogs = auditLogs.GroupBy(log => log.UserName); @@ -213,31 +210,29 @@ public class ElasticsearchEntityChangeStore : IEntityChangeStore, ITransientDepe { var client = _clientFactory.Create(); - var sortOrder = SortOrder.Descending; + var sortOrder = SortOrder.Desc; var querys = BuildQueryDescriptor(entityChangeId: entityChangeId); - static SourceFilterDescriptor SourceFilter(SourceFilterDescriptor selector) + var searchResponse = await client.SearchAsync(dsl => { - selector.IncludeAll() - .Excludes(field => - field.Field(f => f.Actions) - .Field(f => f.Comments) - .Field(f => f.Exceptions)); - - return selector; - } - - var response = await client.SearchAsync(dsl => - dsl.Index(CreateIndex()) - .Query(log => log.Bool(b => b.Must(querys.ToArray()))) - .Source(SourceFilter) - .Sort(log => log.Field(GetField(nameof(EntityChange.ChangeTime)), sortOrder)) + dsl.Indices(CreateIndex()) + .Query(new BoolQuery + { + Must = querys + }) + .SourceExcludes( + ix => ix.UserName, + ix => ix.EntityChanges) + .Sort(s => s.Field(new FieldSort(GetField(nameof(EntityChange.ChangeTime))) + { + Order = sortOrder + })) .From(0) - .Size(1), - cancellationToken); + .Size(1); + }, cancellationToken); - var auditLog = response.Documents.FirstOrDefault(); + var auditLog = searchResponse.Documents.FirstOrDefault(); if (auditLog != null) { return auditLog.EntityChanges.Select(e => @@ -260,31 +255,30 @@ public class ElasticsearchEntityChangeStore : IEntityChangeStore, ITransientDepe var result = new List(); var client = _clientFactory.Create(); - var sortOrder = SortOrder.Descending; + var sortOrder = SortOrder.Desc; var querys = BuildQueryDescriptor(entityId: entityId, entityTypeFullName: entityTypeFullName); - static SourceFilterDescriptor SourceFilter(SourceFilterDescriptor selector) + var searchResponse = await client.SearchAsync(dsl => { - selector.IncludeAll() - .Excludes(field => - field.Field(f => f.Actions) - .Field(f => f.Comments) - .Field(f => f.Exceptions)); - - return selector; - } - - var response = await client.SearchAsync(dsl => - dsl.Index(CreateIndex()) - .Query(log => log.Bool(b => b.Must(querys.ToArray()))) - .Source(SourceFilter) - .Sort(log => log.Field(GetField(nameof(EntityChange.ChangeTime)), sortOrder)) + dsl.Indices(CreateIndex()) + .Query(new BoolQuery + { + Must = querys + }) + .SourceExcludes( + ix => ix.Actions, + ix => ix.Comments, + ix => ix.Exceptions) + .Sort(s => s.Field(new FieldSort(GetField(nameof(EntityChange.ChangeTime))) + { + Order = sortOrder + })) .From(0) - .Size(100), - cancellationToken); + .Size(1); + }, cancellationToken); - var auditLogs = response.Documents.ToList(); + var auditLogs = searchResponse.Documents.ToList(); if (auditLogs.Any()) { var groupAuditLogs = auditLogs.GroupBy(log => log.UserName); @@ -310,7 +304,7 @@ public class ElasticsearchEntityChangeStore : IEntityChangeStore, ITransientDepe return result; } - protected virtual List, QueryContainer>> BuildQueryDescriptor( + protected virtual List BuildQueryDescriptor( Guid? auditLogId = null, DateTime? startTime = null, DateTime? endTime = null, @@ -319,38 +313,47 @@ public class ElasticsearchEntityChangeStore : IEntityChangeStore, ITransientDepe string entityTypeFullName = null, Guid? entityChangeId = null) { - var querys = new List, QueryContainer>>(); + var queries = new List(); if (auditLogId.HasValue) { - querys.Add(entity => entity.Term(q => q.Field(GetField(nameof(EntityChange.AuditLogId))).Value(auditLogId))); + queries.Add(new TermQuery(GetField(nameof(EntityChange.AuditLogId)), auditLogId.Value.ToString())); } if (startTime.HasValue) { - querys.Add(entity => entity.DateRange(q => q.Field(GetField(nameof(EntityChange.ChangeTime))).GreaterThanOrEquals(_clock.Normalize(startTime.Value)))); + queries.Add(new DateRangeQuery(GetField(nameof(EntityChange.ChangeTime))) + { + Gte = _clock.Normalize(startTime.Value) + }); } if (endTime.HasValue) { - querys.Add(entity => entity.DateRange(q => q.Field(GetField(nameof(EntityChange.ChangeTime))).LessThanOrEquals(_clock.Normalize(endTime.Value)))); + queries.Add(new DateRangeQuery(GetField(nameof(EntityChange.ChangeTime))) + { + Lte = _clock.Normalize(endTime.Value) + }); } if (changeType.HasValue) { - querys.Add(entity => entity.Term(q => q.Field(GetField(nameof(EntityChange.ChangeType))).Value(changeType))); + queries.Add(new TermQuery(GetField(nameof(EntityChange.ChangeType)), ((int)changeType.Value).ToString())); } if (!entityId.IsNullOrWhiteSpace()) { - querys.Add(entity => entity.Term(q => q.Field(GetField(nameof(EntityChange.EntityId))).Value(entityId))); + queries.Add(new TermQuery(GetField(nameof(EntityChange.EntityId)), entityId)); } if (!entityTypeFullName.IsNullOrWhiteSpace()) { - querys.Add(entity => entity.Wildcard(q => q.Field(GetField(nameof(EntityChange.EntityTypeFullName))).Value($"*{entityTypeFullName}*"))); + queries.Add(new WildcardQuery(GetField(nameof(EntityChange.EntityTypeFullName))) + { + Value = $"*{entityTypeFullName}*" + }); } if (entityChangeId.HasValue) { - querys.Add(entity => entity.Term(q => q.Field(GetField(nameof(EntityChange.Id))).Value(entityChangeId))); + queries.Add(new TermQuery(GetField(nameof(EntityChange.Id)), entityChangeId.Value.ToString())); } - return querys; + return queries; } protected virtual string CreateIndex() @@ -358,34 +361,19 @@ public class ElasticsearchEntityChangeStore : IEntityChangeStore, ITransientDepe return _indexNameNormalizer.NormalizeIndex("audit-log"); } - protected Func, IPromise> GetEntityChangeSources() - { - return field => field - .Field("EntityChanges.Id") - .Field("EntityChanges.AuditLogId") - .Field("EntityChanges.TenantId") - .Field("EntityChanges.ChangeTime") - .Field("EntityChanges.ChangeType") - .Field("EntityChanges.EntityTenantId") - .Field("EntityChanges.EntityId") - .Field("EntityChanges.EntityTypeFullName") - .Field("EntityChanges.PropertyChanges") - .Field("EntityChanges.ExtraProperties"); - } - private readonly static IDictionary _fieldMaps = new Dictionary(StringComparer.InvariantCultureIgnoreCase) - { - { "Id", "EntityChanges.Id.keyword" }, - { "AuditLogId", "EntityChanges.AuditLogId.keyword" }, - { "TenantId", "EntityChanges.TenantId.keyword" }, - { "EntityTenantId", "EntityChanges.EntityTenantId.keyword" }, - { "EntityId", "EntityChanges.EntityId.keyword" }, - { "EntityTypeFullName", "EntityChanges.EntityTypeFullName.keyword" }, - { "PropertyChanges", "EntityChanges.PropertyChanges" }, - { "ExtraProperties", "EntityChanges.ExtraProperties" }, - { "ChangeType", "EntityChanges.ChangeType" }, - { "ChangeTime", "EntityChanges.ChangeTime" }, - }; + { + { "Id", "EntityChanges.Id.keyword" }, + { "AuditLogId", "EntityChanges.AuditLogId.keyword" }, + { "TenantId", "EntityChanges.TenantId.keyword" }, + { "EntityTenantId", "EntityChanges.EntityTenantId.keyword" }, + { "EntityId", "EntityChanges.EntityId.keyword" }, + { "EntityTypeFullName", "EntityChanges.EntityTypeFullName.keyword" }, + { "PropertyChanges", "EntityChanges.PropertyChanges" }, + { "ExtraProperties", "EntityChanges.ExtraProperties" }, + { "ChangeType", "EntityChanges.ChangeType" }, + { "ChangeTime", "EntityChanges.ChangeTime" }, + }; protected virtual string GetField(string field) { if (_fieldMaps.TryGetValue(field, out var mapField)) diff --git a/aspnet-core/framework/auditing/LINGYUN.Abp.AuditLogging.Elasticsearch/LINGYUN/Abp/AuditLogging/Elasticsearch/ElasticsearchSecurityLogManager.cs b/aspnet-core/framework/auditing/LINGYUN.Abp.AuditLogging.Elasticsearch/LINGYUN/Abp/AuditLogging/Elasticsearch/ElasticsearchSecurityLogManager.cs index 47264f922..dbe73e0a1 100644 --- a/aspnet-core/framework/auditing/LINGYUN.Abp.AuditLogging.Elasticsearch/LINGYUN/Abp/AuditLogging/Elasticsearch/ElasticsearchSecurityLogManager.cs +++ b/aspnet-core/framework/auditing/LINGYUN.Abp.AuditLogging.Elasticsearch/LINGYUN/Abp/AuditLogging/Elasticsearch/ElasticsearchSecurityLogManager.cs @@ -1,8 +1,9 @@ -using LINGYUN.Abp.Elasticsearch; +using Elastic.Clients.Elasticsearch; +using Elastic.Clients.Elasticsearch.QueryDsl; +using LINGYUN.Abp.Elasticsearch; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Logging.Abstractions; using Microsoft.Extensions.Options; -using Nest; using System; using System.Collections.Generic; using System.Linq; @@ -99,12 +100,13 @@ public class ElasticsearchSecurityLogManager : ISecurityLogManager, ITransientDe { var client = _clientFactory.Create(); + var idValues = ids.Select(x => FieldValue.String(x.ToString())).ToList(); await client.DeleteByQueryAsync( - x => x.Index(CreateIndex()) + x => x.Indices(CreateIndex()) .Query(query => query.Terms(terms => terms.Field(field => field.Id) - .Terms(ids))), + .Terms(new TermsQueryField(idValues)))), cancellationToken); } @@ -128,7 +130,7 @@ public class ElasticsearchSecurityLogManager : ISecurityLogManager, ITransientDe var client = _clientFactory.Create(); var sortOrder = !sorting.IsNullOrWhiteSpace() && sorting.EndsWith("asc", StringComparison.InvariantCultureIgnoreCase) - ? SortOrder.Ascending : SortOrder.Descending; + ? SortOrder.Asc : SortOrder.Desc; sorting = !sorting.IsNullOrWhiteSpace() ? sorting.Split()[0] : nameof(SecurityLog.CreationTime); @@ -146,9 +148,11 @@ public class ElasticsearchSecurityLogManager : ISecurityLogManager, ITransientDe correlationId); var response = await client.SearchAsync(dsl => - dsl.Index(CreateIndex()) - .Query(log => log.Bool(b => b.Must(querys.ToArray()))) - .Source(log => log.IncludeAll()) + dsl.Indices(CreateIndex()) + .Query(new BoolQuery + { + Must = querys + }) .Sort(log => log.Field(GetField(sorting), sortOrder)) .From(skipCount) .Size(maxResultCount), @@ -186,14 +190,17 @@ public class ElasticsearchSecurityLogManager : ISecurityLogManager, ITransientDe correlationId); var response = await client.CountAsync(dsl => - dsl.Index(CreateIndex()) - .Query(log => log.Bool(b => b.Must(querys.ToArray()))), + dsl.Indices(CreateIndex()) + .Query(new BoolQuery + { + Must = querys + }), cancellationToken); return response.Count; } - protected virtual List, QueryContainer>> BuildQueryDescriptor( + protected virtual List BuildQueryDescriptor( DateTime? startTime = null, DateTime? endTime = null, string applicationName = null, @@ -205,50 +212,56 @@ public class ElasticsearchSecurityLogManager : ISecurityLogManager, ITransientDe string clientIpAddress = null, string correlationId = null) { - var querys = new List, QueryContainer>>(); + var queries = new List(); if (startTime.HasValue) { - querys.Add((log) => log.DateRange((q) => q.Field(GetField(nameof(SecurityLog.CreationTime))).GreaterThanOrEquals(_clock.Normalize(startTime.Value)))); + queries.Add(new DateRangeQuery(GetField(nameof(SecurityLog.CreationTime))) + { + Gte = _clock.Normalize(startTime.Value) + }); } if (endTime.HasValue) { - querys.Add((log) => log.DateRange((q) => q.Field(GetField(nameof(SecurityLog.CreationTime))).LessThanOrEquals(_clock.Normalize(endTime.Value)))); + queries.Add(new DateRangeQuery(GetField(nameof(SecurityLog.CreationTime))) + { + Lte = _clock.Normalize(endTime.Value) + }); } if (!applicationName.IsNullOrWhiteSpace()) { - querys.Add((log) => log.Term((q) => q.Field(GetField(nameof(SecurityLog.ApplicationName))).Value(applicationName))); + queries.Add(new TermQuery(GetField(nameof(SecurityLog.ApplicationName)), applicationName)); } if (!identity.IsNullOrWhiteSpace()) { - querys.Add((log) => log.Term((q) => q.Field(GetField(nameof(SecurityLog.Identity))).Value(identity))); + queries.Add(new TermQuery(GetField(nameof(SecurityLog.Identity)), identity)); } if (!action.IsNullOrWhiteSpace()) { - querys.Add((log) => log.Term((q) => q.Field(GetField(nameof(SecurityLog.Action))).Value(action))); + queries.Add(new TermQuery(GetField(nameof(SecurityLog.Action)), action)); } if (userId.HasValue) { - querys.Add((log) => log.Term((q) => q.Field(GetField(nameof(SecurityLog.UserId))).Value(userId))); + queries.Add(new TermQuery(GetField(nameof(SecurityLog.UserId)), userId.Value.ToString())); } if (!userName.IsNullOrWhiteSpace()) { - querys.Add((log) => log.Term((q) => q.Field(GetField(nameof(SecurityLog.UserName))).Value(userName))); + queries.Add(new TermQuery(GetField(nameof(SecurityLog.UserName)), userName)); } if (!clientId.IsNullOrWhiteSpace()) { - querys.Add((log) => log.Term((q) => q.Field(GetField(nameof(SecurityLog.ClientId))).Value(clientId))); + queries.Add(new TermQuery(GetField(nameof(SecurityLog.ClientId)), clientId)); } if (!clientIpAddress.IsNullOrWhiteSpace()) { - querys.Add((log) => log.Term((q) => q.Field(GetField(nameof(SecurityLog.ClientIpAddress))).Value(clientIpAddress))); + queries.Add(new TermQuery(GetField(nameof(SecurityLog.ClientIpAddress)), clientIpAddress)); } if (!correlationId.IsNullOrWhiteSpace()) { - querys.Add((log) => log.Term((q) => q.Field(GetField(nameof(SecurityLog.CorrelationId))).Value(correlationId))); + queries.Add(new TermQuery(GetField(nameof(SecurityLog.CorrelationId)), correlationId)); } - return querys; + return queries; } protected virtual string CreateIndex() diff --git a/aspnet-core/framework/auditing/LINGYUN.Abp.AuditLogging.Elasticsearch/LINGYUN/Abp/AuditLogging/Elasticsearch/IndexInitializer.cs b/aspnet-core/framework/auditing/LINGYUN.Abp.AuditLogging.Elasticsearch/LINGYUN/Abp/AuditLogging/Elasticsearch/IndexInitializer.cs index d394c08fd..6a3a9253e 100644 --- a/aspnet-core/framework/auditing/LINGYUN.Abp.AuditLogging.Elasticsearch/LINGYUN/Abp/AuditLogging/Elasticsearch/IndexInitializer.cs +++ b/aspnet-core/framework/auditing/LINGYUN.Abp.AuditLogging.Elasticsearch/LINGYUN/Abp/AuditLogging/Elasticsearch/IndexInitializer.cs @@ -1,11 +1,11 @@ -using LINGYUN.Abp.Elasticsearch; +using Elastic.Clients.Elasticsearch; +using Elastic.Clients.Elasticsearch.Mapping; +using LINGYUN.Abp.Elasticsearch; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Logging.Abstractions; using Microsoft.Extensions.Options; -using Nest; using System; using System.Threading.Tasks; -using Volo.Abp.Data; using Volo.Abp.DependencyInjection; using Volo.Abp.Json; @@ -40,66 +40,146 @@ public class IndexInitializer : IIndexInitializer, ISingletonDependency var dateTimeFormat = !_jsonOptions.OutputDateTimeFormat.IsNullOrWhiteSpace() ? $"{_jsonOptions.OutputDateTimeFormat}||strict_date_optional_time||epoch_millis" : "strict_date_optional_time||epoch_millis"; - var indexState = new IndexState - { - Settings = _elasticsearchOptions.IndexSettings, - }; - await InitlizeAuditLogIndex(client, indexState, dateTimeFormat); - await InitlizeSecurityLogIndex(client, indexState, dateTimeFormat); + await InitlizeAuditLogIndex(client, dateTimeFormat); + await InitlizeSecurityLogIndex(client, dateTimeFormat); } - protected async virtual Task InitlizeAuditLogIndex(IElasticClient client, IIndexState indexState, string dateTimeFormat) + protected async virtual Task InitlizeAuditLogIndex(ElasticsearchClient client, string dateTimeFormat) { var indexName = _nameNormalizer.NormalizeIndex("audit-log"); var indexExists = await client.Indices.ExistsAsync(indexName); if (!indexExists.Exists) { - var indexCreateResponse = await client.Indices.CreateAsync( - indexName, - dsl => dsl.InitializeUsing(indexState) - .Map(map => - map.AutoMap() - .Properties(mp => - mp.Date(p => p.Name(n => n.ExecutionTime).Format(dateTimeFormat)) - .Object(p => p.Name(n => n.ExtraProperties)) - .Nested(n => - n.AutoMap() - .Name(nameof(AuditLog.EntityChanges)) - .Properties(np => - np.Object(p => p.Name(n => n.ExtraProperties)) - .Date(p => p.Name(n => n.ChangeTime).Format(dateTimeFormat)) - .Nested(npn => npn.Name(nameof(EntityChange.PropertyChanges))))) - .Nested(n => n.Name(nameof(AuditLog.Actions)) - .AutoMap() - .Properties((np => - np.Object(p => p.Name(n => n.ExtraProperties)) - .Date(p => p.Name(n => n.ExecutionTime).Format(dateTimeFormat)))))))); - if (!indexCreateResponse.IsValid) + var indexCreateResponse = await client.Indices.CreateAsync(indexName, c => + { + c.Settings(_elasticsearchOptions.IndexSettings); + c.Mappings(mp => mp + .Properties(pd => + { + pd.Keyword(k => k.Id); + pd.Keyword(k => k.ApplicationName); + pd.Keyword(k => k.UserId); + pd.Keyword(k => k.UserName); + pd.Keyword(k => k.TenantId); + pd.Keyword(k => k.TenantName); + pd.Keyword(k => k.ImpersonatorUserId); + pd.Keyword(k => k.ImpersonatorUserName); + pd.Keyword(k => k.ImpersonatorTenantId); + pd.Keyword(k => k.ImpersonatorTenantName); + pd.Date(d => d.ExecutionTime, d => d.Format(dateTimeFormat)); + pd.IntegerNumber(n => n.ExecutionDuration); + pd.Keyword(k => k.ClientIpAddress); + pd.Keyword(k => k.ClientName); + pd.Keyword(k => k.ClientId); + pd.Keyword(k => k.CorrelationId); + pd.Keyword(k => k.BrowserInfo); + pd.Keyword(k => k.HttpMethod); + pd.Keyword(k => k.Url); + pd.Keyword(k => k.Exceptions); + pd.Keyword(k => k.Comments); + pd.IntegerNumber(n => n.HttpStatusCode); + pd.Nested(n => n.EntityChanges, np => + { + np.Dynamic(DynamicMapping.False); + np.Properties(npd => + { + npd.Keyword(nameof(EntityChange.Id)); + npd.Keyword(nameof(EntityChange.AuditLogId)); + npd.Keyword(nameof(EntityChange.TenantId)); + npd.Date(nameof(EntityChange.ChangeTime), d => d.Format(dateTimeFormat)); + npd.IntegerNumber(nameof(EntityChange.ChangeType)); + npd.Keyword(nameof(EntityChange.EntityTenantId)); + npd.Keyword(nameof(EntityChange.EntityId)); + npd.Keyword(nameof(EntityChange.EntityTypeFullName)); + npd.Nested(nameof(EntityChange.PropertyChanges), pc => + { + pc.Dynamic(DynamicMapping.False); + pc.Properties(pcn => + { + pcn.Keyword(nameof(EntityPropertyChange.Id)); + pcn.Keyword(nameof(EntityPropertyChange.TenantId)); + pcn.Keyword(nameof(EntityPropertyChange.EntityChangeId)); + pcn.Keyword(nameof(EntityPropertyChange.NewValue)); + pcn.Keyword(nameof(EntityPropertyChange.OriginalValue)); + pcn.Keyword(nameof(EntityPropertyChange.PropertyName)); + pcn.Keyword(nameof(EntityPropertyChange.PropertyTypeFullName)); + }); + }); + npd.Object(nameof(EntityChange.ExtraProperties)); + }); + }); + pd.Nested(n => n.Actions, np => + { + np.Dynamic(DynamicMapping.False); + np.Properties(npd => + { + npd.Keyword(nameof(AuditLogAction.Id)); + npd.Keyword(nameof(AuditLogAction.TenantId)); + npd.Keyword(nameof(AuditLogAction.AuditLogId)); + npd.Keyword(nameof(AuditLogAction.ServiceName)); + npd.Keyword(nameof(AuditLogAction.MethodName)); + npd.Keyword(nameof(AuditLogAction.Parameters)); + npd.Date(nameof(AuditLogAction.ExecutionTime), d => d.Format(dateTimeFormat)); + npd.IntegerNumber(nameof(AuditLogAction.ExecutionDuration)); + npd.Object(nameof(AuditLogAction.ExtraProperties)); + }); + }); + pd.Object(f => f.ExtraProperties); + })); + }); + + if (!indexCreateResponse.IsValidResponse) { + if (indexCreateResponse.TryGetOriginalException(out var ex)) + { + Logger.LogWarning(ex, "Failed to initialize index and audit log may not be retrieved."); + return; + } Logger.LogWarning("Failed to initialize index and audit log may not be retrieved."); - Logger.LogWarning(indexCreateResponse.OriginalException.ToString()); + Logger.LogWarning(indexCreateResponse.DebugInformation); } } } - protected async virtual Task InitlizeSecurityLogIndex(IElasticClient client, IIndexState indexState, string dateTimeFormat) + protected async virtual Task InitlizeSecurityLogIndex(ElasticsearchClient client, string dateTimeFormat) { var indexName = _nameNormalizer.NormalizeIndex("security-log"); var indexExists = await client.Indices.ExistsAsync(indexName); if (!indexExists.Exists) { - var indexCreateResponse = await client.Indices.CreateAsync( - indexName, - dsl => dsl.InitializeUsing(indexState) - .Map(map => - map.AutoMap() - .Properties(mp => - mp.Object(p => p.Name(n => n.ExtraProperties)) - .Date(p => p.Name(n => n.CreationTime).Format(dateTimeFormat))))); - if (!indexCreateResponse.IsValid) + var indexCreateResponse = await client.Indices.CreateAsync(indexName, c => { - Logger.LogWarning("Failed to initialize index and security log may not be retrieved."); - Logger.LogWarning(indexCreateResponse.OriginalException.ToString()); + c.Settings(_elasticsearchOptions.IndexSettings); + c.Mappings(mp => + { + mp.Properties(pd => + { + pd.Keyword(k => k.Id); + pd.Keyword(k => k.TenantId); + pd.Keyword(k => k.ApplicationName); + pd.Keyword(k => k.Identity); + pd.Keyword(k => k.Action); + pd.Keyword(k => k.UserId); + pd.Keyword(k => k.UserName); + pd.Keyword(k => k.TenantName); + pd.Keyword(k => k.ClientId); + pd.Keyword(k => k.CorrelationId); + pd.Keyword(k => k.ClientIpAddress); + pd.Keyword(k => k.BrowserInfo); + pd.Date(k => k.CreationTime, d => d.Format(dateTimeFormat)); + pd.Object(f => f.ExtraProperties); + }); + }); + }); + if (!indexCreateResponse.IsValidResponse) + { + if (indexCreateResponse.TryGetOriginalException(out var ex)) + { + Logger.LogWarning(ex, "Failed to initialize index and audit log may not be retrieved."); + return; + } + Logger.LogWarning("Failed to initialize index and audit log may not be retrieved."); + Logger.LogWarning(indexCreateResponse.DebugInformation); } } }