tsai
/
abp-next-admin
mirror of https://github.com/colinin/abp-next-admin.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity
Browse Source

fix(auditing): Fix the ES query for auditlog 

- 审计日志ES集成客户端库查询语法变更
pull/1416/head
colin 4 weeks ago
parent
ea40a8640c
commit
078dbb11c0
5 changed files with 364 additions and 273 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 4
      aspnet-core/framework/auditing/LINGYUN.Abp.AuditLogging.Elasticsearch/LINGYUN/Abp/AuditLogging/Elasticsearch/AbpAuditLoggingElasticsearchOptions.cs
  2. 146
      aspnet-core/framework/auditing/LINGYUN.Abp.AuditLogging.Elasticsearch/LINGYUN/Abp/AuditLogging/Elasticsearch/ElasticsearchAuditLogManager.cs
  3. 260
      aspnet-core/framework/auditing/LINGYUN.Abp.AuditLogging.Elasticsearch/LINGYUN/Abp/AuditLogging/Elasticsearch/ElasticsearchEntityChangeStore.cs
  4. 59
      aspnet-core/framework/auditing/LINGYUN.Abp.AuditLogging.Elasticsearch/LINGYUN/Abp/AuditLogging/Elasticsearch/ElasticsearchSecurityLogManager.cs
  5. 168
      aspnet-core/framework/auditing/LINGYUN.Abp.AuditLogging.Elasticsearch/LINGYUN/Abp/AuditLogging/Elasticsearch/IndexInitializer.cs

4
aspnet-core/framework/auditing/LINGYUN.Abp.AuditLogging.Elasticsearch/LINGYUN/Abp/AuditLogging/Elasticsearch/AbpAuditLoggingElasticsearchOptions.cs
View File

@ -1,4 +1,4 @@
using Nest;
using Elastic.Clients.Elasticsearch.IndexManagement;
namespace LINGYUN.Abp.AuditLogging.Elasticsearch;
@ -6,7 +6,7 @@ public class AbpAuditLoggingElasticsearchOptions
{
public const string DefaultIndexPrefix = "auditlogging";
public string IndexPrefix { get; set; }
public IIndexSettings IndexSettings { get; set; }
public IndexSettings IndexSettings { get; set; }
public AbpAuditLoggingElasticsearchOptions()
{

146
aspnet-core/framework/auditing/LINGYUN.Abp.AuditLogging.Elasticsearch/LINGYUN/Abp/AuditLogging/Elasticsearch/ElasticsearchAuditLogManager.cs
View File

@ -1,8 +1,9 @@
using LINGYUN.Abp.Elasticsearch;
using Elastic.Clients.Elasticsearch;
using Elastic.Clients.Elasticsearch.QueryDsl;
using LINGYUN.Abp.Elasticsearch;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Logging.Abstractions;
using Microsoft.Extensions.Options;
using Nest;
using System;
using System.Collections.Generic;
using System.Linq;
@ -82,8 +83,11 @@ public class ElasticsearchAuditLogManager : IAuditLogManager, ITransientDependen
httpStatusCode);
var response = await client.CountAsync<AuditLog>(dsl =>
dsl.Index(CreateIndex())
.Query(log => log.Bool(b => b.Must(querys.ToArray()))),
dsl.Indices(CreateIndex())
.Query(new BoolQuery
{
Must = querys
}),
cancellationToken);
return response.Count;
@ -113,7 +117,7 @@ public class ElasticsearchAuditLogManager : IAuditLogManager, ITransientDependen
var client = _clientFactory.Create();
var sortOrder = !sorting.IsNullOrWhiteSpace() && sorting.EndsWith("asc", StringComparison.InvariantCultureIgnoreCase)
? SortOrder.Ascending : SortOrder.Descending;
? SortOrder.Asc : SortOrder.Desc;
sorting = !sorting.IsNullOrWhiteSpace()
? sorting.Split()[0]
: nameof(AuditLog.ExecutionTime);
@ -134,31 +138,32 @@ public class ElasticsearchAuditLogManager : IAuditLogManager, ITransientDependen
hasException,
httpStatusCode);
SourceFilterDescriptor<AuditLog> SourceFilter(SourceFilterDescriptor<AuditLog> selector)
var searchResponse = await client.SearchAsync<AuditLog>(dsl =>
{
selector.IncludeAll();
dsl.Indices(CreateIndex())
.Query(new BoolQuery
{
Must = querys
})
.Sort(s => s.Field(new FieldSort(GetField(sorting))
{
Order = sortOrder
}))
.From(skipCount)
.Size(maxResultCount);
// 字段过滤
if (!includeDetails)
{
selector.Excludes(field =>
field.Field(f => f.Actions)
.Field(f => f.Comments)
.Field(f => f.Exceptions)
.Field(f => f.EntityChanges));
dsl.SourceExcludes(
ex => ex.Actions,
ex => ex.Comments,
ex => ex.Exceptions,
ex => ex.EntityChanges);
}
}, cancellationToken);
return selector;
}
var response = await client.SearchAsync<AuditLog>(dsl =>
dsl.Index(CreateIndex())
.Query(log => log.Bool(b => b.Must(querys.ToArray())))
.Source(SourceFilter)
.Sort(log => log.Field(GetField(sorting), sortOrder))
.From(skipCount)
.Size(maxResultCount),
cancellationToken);
return response.Documents.ToList();
return searchResponse.Documents.ToList();
}
public async virtual Task<AuditLog> GetAsync(
@ -169,9 +174,8 @@ public class ElasticsearchAuditLogManager : IAuditLogManager, ITransientDependen
var client = _clientFactory.Create();
var response = await client.GetAsync<AuditLog>(
id,
dsl =>
dsl.Index(CreateIndex()),
id.ToString(),
dsl => dsl.Index(CreateIndex()),
cancellationToken);
return response.Source;
@ -182,9 +186,8 @@ public class ElasticsearchAuditLogManager : IAuditLogManager, ITransientDependen
var client = _clientFactory.Create();
await client.DeleteAsync<AuditLog>(
id,
dsl =>
dsl.Index(CreateIndex()),
id.ToString(),
dsl => dsl.Index(CreateIndex()),
cancellationToken);
}
@ -192,12 +195,13 @@ public class ElasticsearchAuditLogManager : IAuditLogManager, ITransientDependen
{
var client = _clientFactory.Create();
var idValues = ids.Select(id => FieldValue.String(id.ToString())).ToList();
await client.DeleteByQueryAsync<AuditLog>(
x => x.Index(CreateIndex())
x => x.Indices(CreateIndex())
.Query(query =>
query.Terms(terms =>
terms.Field(field => field.Id)
.Terms(ids))),
.Terms(new TermsQueryField(idValues)))),
cancellationToken);
}
@ -217,7 +221,7 @@ public class ElasticsearchAuditLogManager : IAuditLogManager, ITransientDependen
catch (Exception ex)
{
Logger.LogWarning("Could not save the audit log object: " + Environment.NewLine + auditInfo.ToString());
Logger.LogException(ex, Microsoft.Extensions.Logging.LogLevel.Error);
Logger.LogException(ex, LogLevel.Error);
}
return "";
}
@ -239,14 +243,12 @@ public class ElasticsearchAuditLogManager : IAuditLogManager, ITransientDependen
// 使用 Bulk 命令传输可能存在参数庞大的日志结构
var response = await client.BulkAsync(
dsl => dsl.Index(CreateIndex())
.Create<AuditLog>(ct =>
ct.Id(auditLog.Id)
.Document(auditLog)));
.Create(auditLog, ct => ct.Id(auditLog.Id)));
return response.Items?.FirstOrDefault()?.Id;
}
protected virtual List<Func<QueryContainerDescriptor<AuditLog>, QueryContainer>> BuildQueryDescriptor(
protected virtual List<Query> BuildQueryDescriptor(
DateTime? startTime = null,
DateTime? endTime = null,
string httpMethod = null,
@ -262,55 +264,70 @@ public class ElasticsearchAuditLogManager : IAuditLogManager, ITransientDependen
bool? hasException = null,
HttpStatusCode? httpStatusCode = null)
{
var querys = new List<Func<QueryContainerDescriptor<AuditLog>, QueryContainer>>();
var queries = new List<Query>();
if (startTime.HasValue)
{
querys.Add((log) => log.DateRange((q) => q.Field(GetField(nameof(AuditLog.ExecutionTime))).GreaterThanOrEquals(_clock.Normalize(startTime.Value))));
queries.Add(new DateRangeQuery(GetField(nameof(AuditLog.ExecutionTime)))
{
Gte = _clock.Normalize(startTime.Value)
});
}
if (endTime.HasValue)
{
querys.Add((log) => log.DateRange((q) => q.Field(GetField(nameof(AuditLog.ExecutionTime))).LessThanOrEquals(_clock.Normalize(endTime.Value))));
queries.Add(new DateRangeQuery(GetField(nameof(AuditLog.ExecutionTime)))
{
Lte = _clock.Normalize(endTime.Value)
});
}
if (!httpMethod.IsNullOrWhiteSpace())
{
querys.Add((log) => log.Term((q) => q.Field(GetField(nameof(AuditLog.HttpMethod))).Value(httpMethod)));
queries.Add(new TermQuery(GetField(nameof(AuditLog.HttpMethod)), httpMethod));
}
if (!url.IsNullOrWhiteSpace())
{
querys.Add((log) => log.Wildcard((q) => q.Field(GetField(nameof(AuditLog.Url))).Value($"*{url}*")));
queries.Add(new WildcardQuery(GetField(nameof(AuditLog.Url)))
{
Value = $"*{url}*"
});
}
if (userId.HasValue)
{
querys.Add((log) => log.Term((q) => q.Field(GetField(nameof(AuditLog.UserId))).Value(userId)));
queries.Add(new TermQuery(GetField(nameof(AuditLog.UserId)), userId.Value.ToString()));
}
if (!userName.IsNullOrWhiteSpace())
{
querys.Add((log) => log.Term((q) => q.Field(GetField(nameof(AuditLog.UserName))).Value(userName)));
queries.Add(new TermQuery(GetField(nameof(AuditLog.UserName)), userName));
}
if (!applicationName.IsNullOrWhiteSpace())
{
querys.Add((log) => log.Term((q) => q.Field(GetField(nameof(AuditLog.ApplicationName))).Value(applicationName)));
queries.Add(new TermQuery(GetField(nameof(AuditLog.ApplicationName)), applicationName));
}
if (!correlationId.IsNullOrWhiteSpace())
{
querys.Add((log) => log.Term((q) => q.Field(GetField(nameof(AuditLog.CorrelationId))).Value(correlationId)));
queries.Add(new TermQuery(GetField(nameof(AuditLog.CorrelationId)), correlationId));
}
if (!clientId.IsNullOrWhiteSpace())
{
querys.Add((log) => log.Term((q) => q.Field(GetField(nameof(AuditLog.ClientId))).Value(clientId)));
queries.Add(new TermQuery(GetField(nameof(AuditLog.ClientId)), clientId));
}
if (!clientIpAddress.IsNullOrWhiteSpace())
{
querys.Add((log) => log.Term((q) => q.Field(GetField(nameof(AuditLog.ClientIpAddress))).Value(clientIpAddress)));
queries.Add(new TermQuery(GetField(nameof(AuditLog.ClientIpAddress)), clientIpAddress));
}
if (maxExecutionDuration.HasValue)
{
querys.Add((log) => log.Range((q) => q.Field(GetField(nameof(AuditLog.ExecutionDuration))).LessThanOrEquals(maxExecutionDuration)));
queries.Add(new NumberRangeQuery(GetField(nameof(AuditLog.ExecutionDuration)))
{
Lte = maxExecutionDuration.Value
});
}
if (minExecutionDuration.HasValue)
{
querys.Add((log) => log.Range((q) => q.Field(GetField(nameof(AuditLog.ExecutionDuration))).GreaterThanOrEquals(minExecutionDuration)));
queries.Add(new NumberRangeQuery(GetField(nameof(AuditLog.ExecutionDuration)))
{
Gte = minExecutionDuration.Value
});
}
@ -318,33 +335,26 @@ public class ElasticsearchAuditLogManager : IAuditLogManager, ITransientDependen
{
if (hasException.Value)
{
querys.Add(
(q) => q.Bool(
(b) => b.Must(
(m) => m.Exists(
(e) => e.Field((f) => f.Exceptions)))
)
);
queries.Add(new ExistsQuery(GetField("Exceptions")));
}
else
{
querys.Add(
(q) => q.Bool(
(b) => b.MustNot(
(mn) => mn.Exists(
(e) => e.Field(
(f) => f.Exceptions)))
)
);
queries.Add(new BoolQuery
{
MustNot = new List<Query>
{
new ExistsQuery(GetField("Exceptions"))
}
});
}
}
if (httpStatusCode.HasValue)
{
querys.Add((log) => log.Term((q) => q.Field(GetField(nameof(AuditLog.HttpStatusCode))).Value(httpStatusCode)));
queries.Add(new TermQuery(GetField(nameof(AuditLog.HttpStatusCode)), ((int)httpStatusCode.Value).ToString()));
}
return querys;
return queries;
}
protected virtual string CreateIndex()

260
aspnet-core/framework/auditing/LINGYUN.Abp.AuditLogging.Elasticsearch/LINGYUN/Abp/AuditLogging/Elasticsearch/ElasticsearchEntityChangeStore.cs
View File

@ -1,8 +1,9 @@
using LINGYUN.Abp.Elasticsearch;
using Elastic.Clients.Elasticsearch;
using Elastic.Clients.Elasticsearch.QueryDsl;
using LINGYUN.Abp.Elasticsearch;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Logging.Abstractions;
using Microsoft.Extensions.Options;
using Nest;
using System;
using System.Collections.Generic;
using System.Linq;
@ -44,31 +45,34 @@ public class ElasticsearchEntityChangeStore : IEntityChangeStore, ITransientDepe
{
var client = _clientFactory.Create();
var sortOrder = SortOrder.Descending;
var sortOrder = SortOrder.Desc;
var querys = BuildQueryDescriptor(entityChangeId: entityChangeId);
static SourceFilterDescriptor<AuditLog> SourceFilter(SourceFilterDescriptor<AuditLog> selector)
var searchResponse = await client.SearchAsync<AuditLog>(dsl =>
{
selector.IncludeAll()
.Excludes(field =>
field.Field(f => f.Actions)
.Field(f => f.Comments)
.Field(f => f.Exceptions));
dsl.Indices(CreateIndex())
.From(0)
.Size(1)
.Query(new BoolQuery
{
Must = querys
});
return selector;
}
dsl.Sort(s => s.Field(new FieldSort(GetField(nameof(EntityChange.ChangeTime)))
{
Order = sortOrder
}));
var response = await client.SearchAsync<AuditLog>(dsl =>
dsl.Index(CreateIndex())
.Query(log => log.Bool(b => b.Must(querys.ToArray())))
.Source(SourceFilter)
.Sort(log => log.Field(GetField(nameof(EntityChange.ChangeTime)), sortOrder))
.From(0)
.Size(1),
cancellationToken);
dsl.SourceIncludes(ix => ix.EntityChanges);
var auditLog = response.Documents.FirstOrDefault();
dsl.SourceExcludes(
ex => ex.Actions,
ex => ex.Comments,
ex => ex.Exceptions);
}, cancellationToken);
var auditLog = searchResponse.Documents.FirstOrDefault();
if (auditLog != null)
{
return auditLog
@ -99,27 +103,25 @@ public class ElasticsearchEntityChangeStore : IEntityChangeStore, ITransientDepe
entityId,
entityTypeFullName);
SourceFilterDescriptor<AuditLog> SourceFilter(SourceFilterDescriptor<AuditLog> selector)
var searchResponse = await client.SearchAsync<AuditLog>(dsl =>
{
return selector
.Includes(field =>
field.Field(f => f.UserName)
.Field(f => f.EntityChanges))
.Excludes(field =>
field.Field(f => f.Actions)
.Field(f => f.Comments)
.Field(f => f.Exceptions));
}
var response = await client.SearchAsync<AuditLog>(dsl =>
dsl.Index(CreateIndex())
.Query(log => log.Bool(b => b.Must(querys.ToArray())))
.Source(SourceFilter)
dsl.Indices(CreateIndex())
.Query(new BoolQuery
{
Must = querys
})
.SourceIncludes(
ix => ix.UserName,
ix => ix.EntityChanges)
.SourceExcludes(
ex => ex.Actions,
ex => ex.Comments,
ex => ex.Exceptions)
.From(0)
.Size(1000),
cancellationToken);
.Size(1000);
}, cancellationToken);
var auditLogs = response.Documents.ToList();
var auditLogs = searchResponse.Documents.ToList();
return auditLogs.Sum(log => log.EntityChanges.Count);
}
@ -143,7 +145,7 @@ public class ElasticsearchEntityChangeStore : IEntityChangeStore, ITransientDepe
var client = _clientFactory.Create();
var sortOrder = !sorting.IsNullOrWhiteSpace() && sorting.EndsWith("asc", StringComparison.InvariantCultureIgnoreCase)
? SortOrder.Ascending : SortOrder.Descending;
? SortOrder.Asc : SortOrder.Desc;
sorting = !sorting.IsNullOrWhiteSpace()
? sorting.Split()[0]
: nameof(EntityChange.ChangeTime);
@ -156,33 +158,28 @@ public class ElasticsearchEntityChangeStore : IEntityChangeStore, ITransientDepe
entityId,
entityTypeFullName);
SourceFilterDescriptor<AuditLog> SourceFilter(SourceFilterDescriptor<AuditLog> selector)
var searchResponse = await client.SearchAsync<AuditLog>(dsl =>
{
selector
.Includes(field =>
field.Field(f => f.UserName)
.Field(f => f.EntityChanges));
dsl.Indices(CreateIndex())
.Query(new BoolQuery
{
Must = querys
})
.SourceIncludes(
ix => ix.UserName,
ix => ix.EntityChanges)
.From(0)
.Size(1000);
if (includeDetails)
{
selector.Includes(field =>
field.Field(f => f.Actions)
.Field(f => f.Comments)
.Field(f => f.Exceptions));
dsl.SourceExcludes(
ex => ex.Actions,
ex => ex.Comments,
ex => ex.Exceptions);
}
}, cancellationToken);
return selector;
}
var response = await client.SearchAsync<AuditLog>(dsl =>
dsl.Index(CreateIndex())
.Query(log => log.Bool(b => b.Must(querys.ToArray())))
.Source(SourceFilter)
.Sort(log => log.Field(GetField(nameof(EntityChange.ChangeTime)), sortOrder))
.From(0)
.Size(1000),
cancellationToken);
var auditLogs = response.Documents.ToList();
var auditLogs = searchResponse.Documents.ToList();
if (auditLogs.Any())
{
var groupAuditLogs = auditLogs.GroupBy(log => log.UserName);
@ -213,31 +210,29 @@ public class ElasticsearchEntityChangeStore : IEntityChangeStore, ITransientDepe
{
var client = _clientFactory.Create();
var sortOrder = SortOrder.Descending;
var sortOrder = SortOrder.Desc;
var querys = BuildQueryDescriptor(entityChangeId: entityChangeId);
static SourceFilterDescriptor<AuditLog> SourceFilter(SourceFilterDescriptor<AuditLog> selector)
var searchResponse = await client.SearchAsync<AuditLog>(dsl =>
{
selector.IncludeAll()
.Excludes(field =>
field.Field(f => f.Actions)
.Field(f => f.Comments)
.Field(f => f.Exceptions));
return selector;
}
var response = await client.SearchAsync<AuditLog>(dsl =>
dsl.Index(CreateIndex())
.Query(log => log.Bool(b => b.Must(querys.ToArray())))
.Source(SourceFilter)
.Sort(log => log.Field(GetField(nameof(EntityChange.ChangeTime)), sortOrder))
dsl.Indices(CreateIndex())
.Query(new BoolQuery
{
Must = querys
})
.SourceExcludes(
ix => ix.UserName,
ix => ix.EntityChanges)
.Sort(s => s.Field(new FieldSort(GetField(nameof(EntityChange.ChangeTime)))
{
Order = sortOrder
}))
.From(0)
.Size(1),
cancellationToken);
.Size(1);
}, cancellationToken);
var auditLog = response.Documents.FirstOrDefault();
var auditLog = searchResponse.Documents.FirstOrDefault();
if (auditLog != null)
{
return auditLog.EntityChanges.Select(e =>
@ -260,31 +255,30 @@ public class ElasticsearchEntityChangeStore : IEntityChangeStore, ITransientDepe
var result = new List<EntityChangeWithUsername>();
var client = _clientFactory.Create();
var sortOrder = SortOrder.Descending;
var sortOrder = SortOrder.Desc;
var querys = BuildQueryDescriptor(entityId: entityId, entityTypeFullName: entityTypeFullName);
static SourceFilterDescriptor<AuditLog> SourceFilter(SourceFilterDescriptor<AuditLog> selector)
var searchResponse = await client.SearchAsync<AuditLog>(dsl =>
{
selector.IncludeAll()
.Excludes(field =>
field.Field(f => f.Actions)
.Field(f => f.Comments)
.Field(f => f.Exceptions));
return selector;
}
var response = await client.SearchAsync<AuditLog>(dsl =>
dsl.Index(CreateIndex())
.Query(log => log.Bool(b => b.Must(querys.ToArray())))
.Source(SourceFilter)
.Sort(log => log.Field(GetField(nameof(EntityChange.ChangeTime)), sortOrder))
dsl.Indices(CreateIndex())
.Query(new BoolQuery
{
Must = querys
})
.SourceExcludes(
ix => ix.Actions,
ix => ix.Comments,
ix => ix.Exceptions)
.Sort(s => s.Field(new FieldSort(GetField(nameof(EntityChange.ChangeTime)))
{
Order = sortOrder
}))
.From(0)
.Size(100),
cancellationToken);
.Size(1);
}, cancellationToken);
var auditLogs = response.Documents.ToList();
var auditLogs = searchResponse.Documents.ToList();
if (auditLogs.Any())
{
var groupAuditLogs = auditLogs.GroupBy(log => log.UserName);
@ -310,7 +304,7 @@ public class ElasticsearchEntityChangeStore : IEntityChangeStore, ITransientDepe
return result;
}
protected virtual List<Func<QueryContainerDescriptor<AuditLog>, QueryContainer>> BuildQueryDescriptor(
protected virtual List<Query> BuildQueryDescriptor(
Guid? auditLogId = null,
DateTime? startTime = null,
DateTime? endTime = null,
@ -319,38 +313,47 @@ public class ElasticsearchEntityChangeStore : IEntityChangeStore, ITransientDepe
string entityTypeFullName = null,
Guid? entityChangeId = null)
{
var querys = new List<Func<QueryContainerDescriptor<AuditLog>, QueryContainer>>();
var queries = new List<Query>();
if (auditLogId.HasValue)
{
querys.Add(entity => entity.Term(q => q.Field(GetField(nameof(EntityChange.AuditLogId))).Value(auditLogId)));
queries.Add(new TermQuery(GetField(nameof(EntityChange.AuditLogId)), auditLogId.Value.ToString()));
}
if (startTime.HasValue)
{
querys.Add(entity => entity.DateRange(q => q.Field(GetField(nameof(EntityChange.ChangeTime))).GreaterThanOrEquals(_clock.Normalize(startTime.Value))));
queries.Add(new DateRangeQuery(GetField(nameof(EntityChange.ChangeTime)))
{
Gte = _clock.Normalize(startTime.Value)
});
}
if (endTime.HasValue)
{
querys.Add(entity => entity.DateRange(q => q.Field(GetField(nameof(EntityChange.ChangeTime))).LessThanOrEquals(_clock.Normalize(endTime.Value))));
queries.Add(new DateRangeQuery(GetField(nameof(EntityChange.ChangeTime)))
{
Lte = _clock.Normalize(endTime.Value)
});
}
if (changeType.HasValue)
{
querys.Add(entity => entity.Term(q => q.Field(GetField(nameof(EntityChange.ChangeType))).Value(changeType)));
queries.Add(new TermQuery(GetField(nameof(EntityChange.ChangeType)), ((int)changeType.Value).ToString()));
}
if (!entityId.IsNullOrWhiteSpace())
{
querys.Add(entity => entity.Term(q => q.Field(GetField(nameof(EntityChange.EntityId))).Value(entityId)));
queries.Add(new TermQuery(GetField(nameof(EntityChange.EntityId)), entityId));
}
if (!entityTypeFullName.IsNullOrWhiteSpace())
{
querys.Add(entity => entity.Wildcard(q => q.Field(GetField(nameof(EntityChange.EntityTypeFullName))).Value($"*{entityTypeFullName}*")));
queries.Add(new WildcardQuery(GetField(nameof(EntityChange.EntityTypeFullName)))
{
Value = $"*{entityTypeFullName}*"
});
}
if (entityChangeId.HasValue)
{
querys.Add(entity => entity.Term(q => q.Field(GetField(nameof(EntityChange.Id))).Value(entityChangeId)));
queries.Add(new TermQuery(GetField(nameof(EntityChange.Id)), entityChangeId.Value.ToString()));
}
return querys;
return queries;
}
protected virtual string CreateIndex()
@ -358,34 +361,19 @@ public class ElasticsearchEntityChangeStore : IEntityChangeStore, ITransientDepe
return _indexNameNormalizer.NormalizeIndex("audit-log");
}
protected Func<FieldsDescriptor<EntityChange>, IPromise<Fields>> GetEntityChangeSources()
{
return field => field
.Field("EntityChanges.Id")
.Field("EntityChanges.AuditLogId")
.Field("EntityChanges.TenantId")
.Field("EntityChanges.ChangeTime")
.Field("EntityChanges.ChangeType")
.Field("EntityChanges.EntityTenantId")
.Field("EntityChanges.EntityId")
.Field("EntityChanges.EntityTypeFullName")
.Field("EntityChanges.PropertyChanges")
.Field("EntityChanges.ExtraProperties");
}
private readonly static IDictionary<string, string> _fieldMaps = new Dictionary<string, string>(StringComparer.InvariantCultureIgnoreCase)
{
{ "Id", "EntityChanges.Id.keyword" },
{ "AuditLogId", "EntityChanges.AuditLogId.keyword" },
{ "TenantId", "EntityChanges.TenantId.keyword" },
{ "EntityTenantId", "EntityChanges.EntityTenantId.keyword" },
{ "EntityId", "EntityChanges.EntityId.keyword" },
{ "EntityTypeFullName", "EntityChanges.EntityTypeFullName.keyword" },
{ "PropertyChanges", "EntityChanges.PropertyChanges" },
{ "ExtraProperties", "EntityChanges.ExtraProperties" },
{ "ChangeType", "EntityChanges.ChangeType" },
{ "ChangeTime", "EntityChanges.ChangeTime" },
};
{
{ "Id", "EntityChanges.Id.keyword" },
{ "AuditLogId", "EntityChanges.AuditLogId.keyword" },
{ "TenantId", "EntityChanges.TenantId.keyword" },
{ "EntityTenantId", "EntityChanges.EntityTenantId.keyword" },
{ "EntityId", "EntityChanges.EntityId.keyword" },
{ "EntityTypeFullName", "EntityChanges.EntityTypeFullName.keyword" },
{ "PropertyChanges", "EntityChanges.PropertyChanges" },
{ "ExtraProperties", "EntityChanges.ExtraProperties" },
{ "ChangeType", "EntityChanges.ChangeType" },
{ "ChangeTime", "EntityChanges.ChangeTime" },
};
protected virtual string GetField(string field)
{
if (_fieldMaps.TryGetValue(field, out var mapField))

59
aspnet-core/framework/auditing/LINGYUN.Abp.AuditLogging.Elasticsearch/LINGYUN/Abp/AuditLogging/Elasticsearch/ElasticsearchSecurityLogManager.cs
View File

@ -1,8 +1,9 @@
using LINGYUN.Abp.Elasticsearch;
using Elastic.Clients.Elasticsearch;
using Elastic.Clients.Elasticsearch.QueryDsl;
using LINGYUN.Abp.Elasticsearch;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Logging.Abstractions;
using Microsoft.Extensions.Options;
using Nest;
using System;
using System.Collections.Generic;
using System.Linq;
@ -99,12 +100,13 @@ public class ElasticsearchSecurityLogManager : ISecurityLogManager, ITransientDe
{
var client = _clientFactory.Create();
var idValues = ids.Select(x => FieldValue.String(x.ToString())).ToList();
await client.DeleteByQueryAsync<SecurityLog>(
x => x.Index(CreateIndex())
x => x.Indices(CreateIndex())
.Query(query =>
query.Terms(terms =>
terms.Field(field => field.Id)
.Terms(ids))),
.Terms(new TermsQueryField(idValues)))),
cancellationToken);
}
@ -128,7 +130,7 @@ public class ElasticsearchSecurityLogManager : ISecurityLogManager, ITransientDe
var client = _clientFactory.Create();
var sortOrder = !sorting.IsNullOrWhiteSpace() && sorting.EndsWith("asc", StringComparison.InvariantCultureIgnoreCase)
? SortOrder.Ascending : SortOrder.Descending;
? SortOrder.Asc : SortOrder.Desc;
sorting = !sorting.IsNullOrWhiteSpace()
? sorting.Split()[0]
: nameof(SecurityLog.CreationTime);
@ -146,9 +148,11 @@ public class ElasticsearchSecurityLogManager : ISecurityLogManager, ITransientDe
correlationId);
var response = await client.SearchAsync<SecurityLog>(dsl =>
dsl.Index(CreateIndex())
.Query(log => log.Bool(b => b.Must(querys.ToArray())))
.Source(log => log.IncludeAll())
dsl.Indices(CreateIndex())
.Query(new BoolQuery
{
Must = querys
})
.Sort(log => log.Field(GetField(sorting), sortOrder))
.From(skipCount)
.Size(maxResultCount),
@ -186,14 +190,17 @@ public class ElasticsearchSecurityLogManager : ISecurityLogManager, ITransientDe
correlationId);
var response = await client.CountAsync<SecurityLog>(dsl =>
dsl.Index(CreateIndex())
.Query(log => log.Bool(b => b.Must(querys.ToArray()))),
dsl.Indices(CreateIndex())
.Query(new BoolQuery
{
Must = querys
}),
cancellationToken);
return response.Count;
}
protected virtual List<Func<QueryContainerDescriptor<SecurityLog>, QueryContainer>> BuildQueryDescriptor(
protected virtual List<Query> BuildQueryDescriptor(
DateTime? startTime = null,
DateTime? endTime = null,
string applicationName = null,
@ -205,50 +212,56 @@ public class ElasticsearchSecurityLogManager : ISecurityLogManager, ITransientDe
string clientIpAddress = null,
string correlationId = null)
{
var querys = new List<Func<QueryContainerDescriptor<SecurityLog>, QueryContainer>>();
var queries = new List<Query>();
if (startTime.HasValue)
{
querys.Add((log) => log.DateRange((q) => q.Field(GetField(nameof(SecurityLog.CreationTime))).GreaterThanOrEquals(_clock.Normalize(startTime.Value))));
queries.Add(new DateRangeQuery(GetField(nameof(SecurityLog.CreationTime)))
{
Gte = _clock.Normalize(startTime.Value)
});
}
if (endTime.HasValue)
{
querys.Add((log) => log.DateRange((q) => q.Field(GetField(nameof(SecurityLog.CreationTime))).LessThanOrEquals(_clock.Normalize(endTime.Value))));
queries.Add(new DateRangeQuery(GetField(nameof(SecurityLog.CreationTime)))
{
Lte = _clock.Normalize(endTime.Value)
});
}
if (!applicationName.IsNullOrWhiteSpace())
{
querys.Add((log) => log.Term((q) => q.Field(GetField(nameof(SecurityLog.ApplicationName))).Value(applicationName)));
queries.Add(new TermQuery(GetField(nameof(SecurityLog.ApplicationName)), applicationName));
}
if (!identity.IsNullOrWhiteSpace())
{
querys.Add((log) => log.Term((q) => q.Field(GetField(nameof(SecurityLog.Identity))).Value(identity)));
queries.Add(new TermQuery(GetField(nameof(SecurityLog.Identity)), identity));
}
if (!action.IsNullOrWhiteSpace())
{
querys.Add((log) => log.Term((q) => q.Field(GetField(nameof(SecurityLog.Action))).Value(action)));
queries.Add(new TermQuery(GetField(nameof(SecurityLog.Action)), action));
}
if (userId.HasValue)
{
querys.Add((log) => log.Term((q) => q.Field(GetField(nameof(SecurityLog.UserId))).Value(userId)));
queries.Add(new TermQuery(GetField(nameof(SecurityLog.UserId)), userId.Value.ToString()));
}
if (!userName.IsNullOrWhiteSpace())
{
querys.Add((log) => log.Term((q) => q.Field(GetField(nameof(SecurityLog.UserName))).Value(userName)));
queries.Add(new TermQuery(GetField(nameof(SecurityLog.UserName)), userName));
}
if (!clientId.IsNullOrWhiteSpace())
{
querys.Add((log) => log.Term((q) => q.Field(GetField(nameof(SecurityLog.ClientId))).Value(clientId)));
queries.Add(new TermQuery(GetField(nameof(SecurityLog.ClientId)), clientId));
}
if (!clientIpAddress.IsNullOrWhiteSpace())
{
querys.Add((log) => log.Term((q) => q.Field(GetField(nameof(SecurityLog.ClientIpAddress))).Value(clientIpAddress)));
queries.Add(new TermQuery(GetField(nameof(SecurityLog.ClientIpAddress)), clientIpAddress));
}
if (!correlationId.IsNullOrWhiteSpace())
{
querys.Add((log) => log.Term((q) => q.Field(GetField(nameof(SecurityLog.CorrelationId))).Value(correlationId)));
queries.Add(new TermQuery(GetField(nameof(SecurityLog.CorrelationId)), correlationId));
}
return querys;
return queries;
}
protected virtual string CreateIndex()

168
aspnet-core/framework/auditing/LINGYUN.Abp.AuditLogging.Elasticsearch/LINGYUN/Abp/AuditLogging/Elasticsearch/IndexInitializer.cs
View File

@ -1,11 +1,11 @@
using LINGYUN.Abp.Elasticsearch;
using Elastic.Clients.Elasticsearch;
using Elastic.Clients.Elasticsearch.Mapping;
using LINGYUN.Abp.Elasticsearch;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Logging.Abstractions;
using Microsoft.Extensions.Options;
using Nest;
using System;
using System.Threading.Tasks;
using Volo.Abp.Data;
using Volo.Abp.DependencyInjection;
using Volo.Abp.Json;
@ -40,66 +40,146 @@ public class IndexInitializer : IIndexInitializer, ISingletonDependency
var dateTimeFormat = !_jsonOptions.OutputDateTimeFormat.IsNullOrWhiteSpace()
? $"{_jsonOptions.OutputDateTimeFormat}||strict_date_optional_time||epoch_millis"
: "strict_date_optional_time||epoch_millis";
var indexState = new IndexState
{
Settings = _elasticsearchOptions.IndexSettings,
};
await InitlizeAuditLogIndex(client, indexState, dateTimeFormat);
await InitlizeSecurityLogIndex(client, indexState, dateTimeFormat);
await InitlizeAuditLogIndex(client, dateTimeFormat);
await InitlizeSecurityLogIndex(client, dateTimeFormat);
}
protected async virtual Task InitlizeAuditLogIndex(IElasticClient client, IIndexState indexState, string dateTimeFormat)
protected async virtual Task InitlizeAuditLogIndex(ElasticsearchClient client, string dateTimeFormat)
{
var indexName = _nameNormalizer.NormalizeIndex("audit-log");
var indexExists = await client.Indices.ExistsAsync(indexName);
if (!indexExists.Exists)
{
var indexCreateResponse = await client.Indices.CreateAsync(
indexName,
dsl => dsl.InitializeUsing(indexState)
.Map<AuditLog>(map =>
map.AutoMap()
.Properties(mp =>
mp.Date(p => p.Name(n => n.ExecutionTime).Format(dateTimeFormat))
.Object<ExtraPropertyDictionary>(p => p.Name(n => n.ExtraProperties))
.Nested<EntityChange>(n =>
n.AutoMap()
.Name(nameof(AuditLog.EntityChanges))
.Properties(np =>
np.Object<ExtraPropertyDictionary>(p => p.Name(n => n.ExtraProperties))
.Date(p => p.Name(n => n.ChangeTime).Format(dateTimeFormat))
.Nested<EntityPropertyChange>(npn => npn.Name(nameof(EntityChange.PropertyChanges)))))
.Nested<AuditLogAction>(n => n.Name(nameof(AuditLog.Actions))
.AutoMap()
.Properties((np =>
np.Object<ExtraPropertyDictionary>(p => p.Name(n => n.ExtraProperties))
.Date(p => p.Name(n => n.ExecutionTime).Format(dateTimeFormat))))))));
if (!indexCreateResponse.IsValid)
var indexCreateResponse = await client.Indices.CreateAsync(indexName, c =>
{
c.Settings(_elasticsearchOptions.IndexSettings);
c.Mappings(mp => mp
.Properties<AuditLog>(pd =>
{
pd.Keyword(k => k.Id);
pd.Keyword(k => k.ApplicationName);
pd.Keyword(k => k.UserId);
pd.Keyword(k => k.UserName);
pd.Keyword(k => k.TenantId);
pd.Keyword(k => k.TenantName);
pd.Keyword(k => k.ImpersonatorUserId);
pd.Keyword(k => k.ImpersonatorUserName);
pd.Keyword(k => k.ImpersonatorTenantId);
pd.Keyword(k => k.ImpersonatorTenantName);
pd.Date(d => d.ExecutionTime, d => d.Format(dateTimeFormat));
pd.IntegerNumber(n => n.ExecutionDuration);
pd.Keyword(k => k.ClientIpAddress);
pd.Keyword(k => k.ClientName);
pd.Keyword(k => k.ClientId);
pd.Keyword(k => k.CorrelationId);
pd.Keyword(k => k.BrowserInfo);
pd.Keyword(k => k.HttpMethod);
pd.Keyword(k => k.Url);
pd.Keyword(k => k.Exceptions);
pd.Keyword(k => k.Comments);
pd.IntegerNumber(n => n.HttpStatusCode);
pd.Nested(n => n.EntityChanges, np =>
{
np.Dynamic(DynamicMapping.False);
np.Properties(npd =>
{
npd.Keyword(nameof(EntityChange.Id));
npd.Keyword(nameof(EntityChange.AuditLogId));
npd.Keyword(nameof(EntityChange.TenantId));
npd.Date(nameof(EntityChange.ChangeTime), d => d.Format(dateTimeFormat));
npd.IntegerNumber(nameof(EntityChange.ChangeType));
npd.Keyword(nameof(EntityChange.EntityTenantId));
npd.Keyword(nameof(EntityChange.EntityId));
npd.Keyword(nameof(EntityChange.EntityTypeFullName));
npd.Nested(nameof(EntityChange.PropertyChanges), pc =>
{
pc.Dynamic(DynamicMapping.False);
pc.Properties(pcn =>
{
pcn.Keyword(nameof(EntityPropertyChange.Id));
pcn.Keyword(nameof(EntityPropertyChange.TenantId));
pcn.Keyword(nameof(EntityPropertyChange.EntityChangeId));
pcn.Keyword(nameof(EntityPropertyChange.NewValue));
pcn.Keyword(nameof(EntityPropertyChange.OriginalValue));
pcn.Keyword(nameof(EntityPropertyChange.PropertyName));
pcn.Keyword(nameof(EntityPropertyChange.PropertyTypeFullName));
});
});
npd.Object(nameof(EntityChange.ExtraProperties));
});
});
pd.Nested(n => n.Actions, np =>
{
np.Dynamic(DynamicMapping.False);
np.Properties(npd =>
{
npd.Keyword(nameof(AuditLogAction.Id));
npd.Keyword(nameof(AuditLogAction.TenantId));
npd.Keyword(nameof(AuditLogAction.AuditLogId));
npd.Keyword(nameof(AuditLogAction.ServiceName));
npd.Keyword(nameof(AuditLogAction.MethodName));
npd.Keyword(nameof(AuditLogAction.Parameters));
npd.Date(nameof(AuditLogAction.ExecutionTime), d => d.Format(dateTimeFormat));
npd.IntegerNumber(nameof(AuditLogAction.ExecutionDuration));
npd.Object(nameof(AuditLogAction.ExtraProperties));
});
});
pd.Object(f => f.ExtraProperties);
}));
});
if (!indexCreateResponse.IsValidResponse)
{
if (indexCreateResponse.TryGetOriginalException(out var ex))
{
Logger.LogWarning(ex, "Failed to initialize index and audit log may not be retrieved.");
return;
}
Logger.LogWarning("Failed to initialize index and audit log may not be retrieved.");
Logger.LogWarning(indexCreateResponse.OriginalException.ToString());
Logger.LogWarning(indexCreateResponse.DebugInformation);
}
}
}
protected async virtual Task InitlizeSecurityLogIndex(IElasticClient client, IIndexState indexState, string dateTimeFormat)
protected async virtual Task InitlizeSecurityLogIndex(ElasticsearchClient client, string dateTimeFormat)
{
var indexName = _nameNormalizer.NormalizeIndex("security-log");
var indexExists = await client.Indices.ExistsAsync(indexName);
if (!indexExists.Exists)
{
var indexCreateResponse = await client.Indices.CreateAsync(
indexName,
dsl => dsl.InitializeUsing(indexState)
.Map<SecurityLog>(map =>
map.AutoMap()
.Properties(mp =>
mp.Object<ExtraPropertyDictionary>(p => p.Name(n => n.ExtraProperties))
.Date(p => p.Name(n => n.CreationTime).Format(dateTimeFormat)))));
if (!indexCreateResponse.IsValid)
var indexCreateResponse = await client.Indices.CreateAsync(indexName, c =>
{
Logger.LogWarning("Failed to initialize index and security log may not be retrieved.");
Logger.LogWarning(indexCreateResponse.OriginalException.ToString());
c.Settings(_elasticsearchOptions.IndexSettings);
c.Mappings(mp =>
{
mp.Properties<SecurityLog>(pd =>
{
pd.Keyword(k => k.Id);
pd.Keyword(k => k.TenantId);
pd.Keyword(k => k.ApplicationName);
pd.Keyword(k => k.Identity);
pd.Keyword(k => k.Action);
pd.Keyword(k => k.UserId);
pd.Keyword(k => k.UserName);
pd.Keyword(k => k.TenantName);
pd.Keyword(k => k.ClientId);
pd.Keyword(k => k.CorrelationId);
pd.Keyword(k => k.ClientIpAddress);
pd.Keyword(k => k.BrowserInfo);
pd.Date(k => k.CreationTime, d => d.Format(dateTimeFormat));
pd.Object(f => f.ExtraProperties);
});
});
});
if (!indexCreateResponse.IsValidResponse)
{
if (indexCreateResponse.TryGetOriginalException(out var ex))
{
Logger.LogWarning(ex, "Failed to initialize index and audit log may not be retrieved.");
return;
}
Logger.LogWarning("Failed to initialize index and audit log may not be retrieved.");
Logger.LogWarning(indexCreateResponse.DebugInformation);
}
}
}

Write Preview
Loading…
Cancel
Save