|
|
@ -1,5 +1,5 @@ |
|
|
using Elastic.Clients.Elasticsearch; |
|
|
using Elastic.Clients.Elasticsearch; |
|
|
using Elastic.Clients.Elasticsearch.QueryDsl; |
|
|
using Elastic.Transport.Diagnostics.Auditing; |
|
|
using LINGYUN.Abp.Elasticsearch; |
|
|
using LINGYUN.Abp.Elasticsearch; |
|
|
using LINGYUN.Linq.Dynamic.Queryable; |
|
|
using LINGYUN.Linq.Dynamic.Queryable; |
|
|
using Microsoft.Extensions.Logging; |
|
|
using Microsoft.Extensions.Logging; |
|
|
@ -10,6 +10,7 @@ using Serilog.Formatting.Elasticsearch; |
|
|
using System; |
|
|
using System; |
|
|
using System.Collections.Generic; |
|
|
using System.Collections.Generic; |
|
|
using System.Linq; |
|
|
using System.Linq; |
|
|
|
|
|
using System.Linq.Expressions; |
|
|
using System.Text.RegularExpressions; |
|
|
using System.Text.RegularExpressions; |
|
|
using System.Threading; |
|
|
using System.Threading; |
|
|
using System.Threading.Tasks; |
|
|
using System.Threading.Tasks; |
|
|
@ -37,7 +38,6 @@ public class SerilogElasticsearchLoggingManager : ILoggingManager, ISingletonDep |
|
|
private readonly ICurrentTenant _currentTenant; |
|
|
private readonly ICurrentTenant _currentTenant; |
|
|
private readonly AbpLoggingSerilogElasticsearchOptions _options; |
|
|
private readonly AbpLoggingSerilogElasticsearchOptions _options; |
|
|
private readonly IElasticsearchClientFactory _clientFactory; |
|
|
private readonly IElasticsearchClientFactory _clientFactory; |
|
|
private readonly IIndexMappingProvider _indexMappingProvider; |
|
|
|
|
|
private readonly IExpressionQueryService _expressionQueryService; |
|
|
private readonly IExpressionQueryService _expressionQueryService; |
|
|
private readonly IObjectMapper<AbpLoggingSerilogElasticsearchModule> _objectMapper; |
|
|
private readonly IObjectMapper<AbpLoggingSerilogElasticsearchModule> _objectMapper; |
|
|
|
|
|
|
|
|
@ -48,7 +48,6 @@ public class SerilogElasticsearchLoggingManager : ILoggingManager, ISingletonDep |
|
|
ICurrentTenant currentTenant, |
|
|
ICurrentTenant currentTenant, |
|
|
IOptions<AbpLoggingSerilogElasticsearchOptions> options, |
|
|
IOptions<AbpLoggingSerilogElasticsearchOptions> options, |
|
|
IElasticsearchClientFactory clientFactory, |
|
|
IElasticsearchClientFactory clientFactory, |
|
|
IIndexMappingProvider indexMappingProvider, |
|
|
|
|
|
IExpressionQueryService expressionQueryService, |
|
|
IExpressionQueryService expressionQueryService, |
|
|
IObjectMapper<AbpLoggingSerilogElasticsearchModule> objectMapper) |
|
|
IObjectMapper<AbpLoggingSerilogElasticsearchModule> objectMapper) |
|
|
{ |
|
|
{ |
|
|
@ -56,7 +55,6 @@ public class SerilogElasticsearchLoggingManager : ILoggingManager, ISingletonDep |
|
|
_objectMapper = objectMapper; |
|
|
_objectMapper = objectMapper; |
|
|
_currentTenant = currentTenant; |
|
|
_currentTenant = currentTenant; |
|
|
_clientFactory = clientFactory; |
|
|
_clientFactory = clientFactory; |
|
|
_indexMappingProvider = indexMappingProvider; |
|
|
|
|
|
_expressionQueryService = expressionQueryService; |
|
|
_expressionQueryService = expressionQueryService; |
|
|
_options = options.Value; |
|
|
_options = options.Value; |
|
|
|
|
|
|
|
|
@ -67,12 +65,11 @@ public class SerilogElasticsearchLoggingManager : ILoggingManager, ISingletonDep |
|
|
ISpecification<LogInfo> specification, |
|
|
ISpecification<LogInfo> specification, |
|
|
CancellationToken cancellationToken = default) |
|
|
CancellationToken cancellationToken = default) |
|
|
{ |
|
|
{ |
|
|
var indexName = CreateIndex(); |
|
|
|
|
|
var converter = new ExpressionQueryConverter<LogInfo, SerilogInfo>(_defaultTypeMap); |
|
|
var converter = new ExpressionQueryConverter<LogInfo, SerilogInfo>(_defaultTypeMap); |
|
|
var expression = converter.Convert(specification.ToExpression()); |
|
|
var expression = converter.Convert(specification.ToExpression()); |
|
|
|
|
|
|
|
|
return await _expressionQueryService.GetCountAsync( |
|
|
return await _expressionQueryService.GetCountAsync( |
|
|
indexName, |
|
|
CreateIndex(), |
|
|
expression, |
|
|
expression, |
|
|
cancellationToken); |
|
|
cancellationToken); |
|
|
} |
|
|
} |
|
|
@ -82,35 +79,28 @@ public class SerilogElasticsearchLoggingManager : ILoggingManager, ISingletonDep |
|
|
string? sorting = null, |
|
|
string? sorting = null, |
|
|
int maxResultCount = 50, |
|
|
int maxResultCount = 50, |
|
|
int skipCount = 0, |
|
|
int skipCount = 0, |
|
|
|
|
|
bool includeDetails = false, |
|
|
CancellationToken cancellationToken = default) |
|
|
CancellationToken cancellationToken = default) |
|
|
{ |
|
|
{ |
|
|
var indexName = CreateIndex(); |
|
|
|
|
|
var converter = new ExpressionQueryConverter<LogInfo, SerilogInfo>(_defaultTypeMap); |
|
|
var converter = new ExpressionQueryConverter<LogInfo, SerilogInfo>(_defaultTypeMap); |
|
|
var expression = converter.Convert(specification.ToExpression()); |
|
|
var expression = converter.Convert(specification.ToExpression()); |
|
|
|
|
|
if (sorting.IsNullOrWhiteSpace()) |
|
|
var sortingField = sorting; |
|
|
|
|
|
if (sortingField.IsNullOrWhiteSpace()) |
|
|
|
|
|
{ |
|
|
{ |
|
|
var indexMapping = await _indexMappingProvider.GetMappingAsync(indexName, cancellationToken); |
|
|
sorting = ElasticsearchJsonFormatter.TimestampPropertyName; |
|
|
if (indexMapping != null) |
|
|
|
|
|
{ |
|
|
|
|
|
var sortingFieldMap = indexMapping.Fields |
|
|
|
|
|
.Where(x => x.Key.Equals(sortingField, StringComparison.CurrentCultureIgnoreCase)) |
|
|
|
|
|
.Select(x => x.Value) |
|
|
|
|
|
.FirstOrDefault(); |
|
|
|
|
|
if (sortingFieldMap != null) |
|
|
|
|
|
{ |
|
|
|
|
|
sortingField = sortingFieldMap.Path; |
|
|
|
|
|
} |
|
|
|
|
|
} |
|
|
|
|
|
} |
|
|
} |
|
|
|
|
|
|
|
|
var serilogLogs = await _expressionQueryService.GetListAsync( |
|
|
var serilogLogs = await _expressionQueryService.GetListAsync( |
|
|
indexName, |
|
|
CreateIndex(), |
|
|
expression, |
|
|
expression, |
|
|
sortingField, |
|
|
sorting, |
|
|
maxResultCount, |
|
|
maxResultCount, |
|
|
skipCount, |
|
|
skipCount, |
|
|
|
|
|
sourceExcludes: includeDetails == false |
|
|
|
|
|
? Fields.FromFields( |
|
|
|
|
|
[ |
|
|
|
|
|
new Field("exceptions"), |
|
|
|
|
|
]) |
|
|
|
|
|
: null, |
|
|
cancellationToken: cancellationToken); |
|
|
cancellationToken: cancellationToken); |
|
|
|
|
|
|
|
|
return _objectMapper.Map<List<SerilogInfo>, List<LogInfo>>(serilogLogs); |
|
|
return _objectMapper.Map<List<SerilogInfo>, List<LogInfo>>(serilogLogs); |
|
|
@ -126,83 +116,18 @@ public class SerilogElasticsearchLoggingManager : ILoggingManager, ISingletonDep |
|
|
string id, |
|
|
string id, |
|
|
CancellationToken cancellationToken = default) |
|
|
CancellationToken cancellationToken = default) |
|
|
{ |
|
|
{ |
|
|
var indexName = CreateIndex(); |
|
|
|
|
|
var client = _clientFactory.Create(); |
|
|
var client = _clientFactory.Create(); |
|
|
var indexMapping = await _indexMappingProvider.GetMappingAsync(indexName, cancellationToken); |
|
|
Expression<Func<SerilogInfo, bool>> expression = x => x.Fields.UniqueId == long.Parse(id); |
|
|
|
|
|
expression = expression.AndIf(_currentTenant.IsAvailable, x => x.Fields.TenantId == _currentTenant.Id); |
|
|
SearchResponse<SerilogInfo> response; |
|
|
|
|
|
|
|
|
var serilogs = await _expressionQueryService.GetListAsync<SerilogInfo>( |
|
|
if (_currentTenant.IsAvailable) |
|
|
CreateIndex(), |
|
|
{ |
|
|
x => x.Fields.UniqueId == long.Parse(id), |
|
|
/* |
|
|
sorting: $"{ElasticsearchJsonFormatter.TimestampPropertyName} DESC", |
|
|
"query": { |
|
|
maxResultCount: 1, |
|
|
"bool": { |
|
|
cancellationToken: cancellationToken); |
|
|
"must": [ |
|
|
|
|
|
{ |
|
|
|
|
|
"term": { |
|
|
|
|
|
"fields.TenantId.keyword": { |
|
|
|
|
|
"value": _currentTenant.GetId() |
|
|
|
|
|
} |
|
|
|
|
|
} |
|
|
|
|
|
}, |
|
|
|
|
|
{ |
|
|
|
|
|
"term": { |
|
|
|
|
|
"fields.UniqueId": { |
|
|
|
|
|
"value": "1474021081433481216" |
|
|
|
|
|
} |
|
|
|
|
|
} |
|
|
|
|
|
} |
|
|
|
|
|
] |
|
|
|
|
|
} |
|
|
|
|
|
} |
|
|
|
|
|
*/ |
|
|
|
|
|
response = await client.SearchAsync<SerilogInfo>( |
|
|
|
|
|
dsl => |
|
|
|
|
|
dsl.Indices(CreateIndex()) |
|
|
|
|
|
.Query( |
|
|
|
|
|
(q) => q.Bool( |
|
|
|
|
|
(b) => b.Must( |
|
|
|
|
|
(s) => s.Term( |
|
|
|
|
|
(t) => t.Field(GetField(indexMapping, "fields.UniqueId")).Value(id)), |
|
|
|
|
|
(s) => s.Term( |
|
|
|
|
|
(t) => t.Field(GetField(indexMapping, "fields.TenantId")).Value(_currentTenant.GetId().ToString()))))) |
|
|
|
|
|
.Size(1), |
|
|
|
|
|
cancellationToken); |
|
|
|
|
|
} |
|
|
|
|
|
else |
|
|
|
|
|
{ |
|
|
|
|
|
/* |
|
|
|
|
|
"query": { |
|
|
|
|
|
"bool": { |
|
|
|
|
|
"must": [ |
|
|
|
|
|
{ |
|
|
|
|
|
"term": { |
|
|
|
|
|
"fields.UniqueId": { |
|
|
|
|
|
"value": "1474021081433481216" |
|
|
|
|
|
} |
|
|
|
|
|
} |
|
|
|
|
|
} |
|
|
|
|
|
] |
|
|
|
|
|
} |
|
|
|
|
|
} |
|
|
|
|
|
*/ |
|
|
|
|
|
response = await client.SearchAsync<SerilogInfo>( |
|
|
|
|
|
dsl => |
|
|
|
|
|
dsl.Indices(CreateIndex()) |
|
|
|
|
|
.Query( |
|
|
|
|
|
(q) => q.Bool( |
|
|
|
|
|
(b) => b.Must( |
|
|
|
|
|
(s) => s.Term( |
|
|
|
|
|
(t) => t.Field(GetField(indexMapping, "fields.UniqueId")).Value(id))))) |
|
|
|
|
|
.Size(1), |
|
|
|
|
|
cancellationToken); |
|
|
|
|
|
if (response.TryGetErrorMessage(out var errorMessage)) |
|
|
|
|
|
{ |
|
|
|
|
|
Logger.LogWarning("Query logs failed: {errorMessage}", errorMessage); |
|
|
|
|
|
} |
|
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
|
|
return _objectMapper.Map<SerilogInfo?, LogInfo?>(response.Documents.FirstOrDefault()); |
|
|
return _objectMapper.Map<SerilogInfo?, LogInfo?>(serilogs.FirstOrDefault()); |
|
|
} |
|
|
} |
|
|
|
|
|
|
|
|
public async virtual Task<long> GetCountAsync( |
|
|
public async virtual Task<long> GetCountAsync( |
|
|
@ -221,36 +146,30 @@ public class SerilogElasticsearchLoggingManager : ILoggingManager, ISingletonDep |
|
|
bool? hasException = null, |
|
|
bool? hasException = null, |
|
|
CancellationToken cancellationToken = default) |
|
|
CancellationToken cancellationToken = default) |
|
|
{ |
|
|
{ |
|
|
var indexName = CreateIndex(); |
|
|
|
|
|
var client = _clientFactory.Create(); |
|
|
var client = _clientFactory.Create(); |
|
|
var indexMapping = await _indexMappingProvider.GetMappingAsync(indexName, cancellationToken); |
|
|
|
|
|
|
|
|
|
|
|
var querys = BuildQueryDescriptor( |
|
|
Expression<Func<SerilogInfo, bool>> expression = _ => true; |
|
|
indexMapping, |
|
|
|
|
|
startTime, |
|
|
expression = expression |
|
|
endTime, |
|
|
.AndIf(startTime.HasValue, x => x.TimeStamp >= _clock.Normalize(startTime!.Value)) |
|
|
level, |
|
|
.AndIf(endTime.HasValue, x => x.TimeStamp <= _clock.Normalize(endTime!.Value)) |
|
|
machineName, |
|
|
.AndIf(level.HasValue, x => x.Level == GetLogEventLevel(level!.Value)) |
|
|
environment, |
|
|
.AndIf(!machineName.IsNullOrWhiteSpace(), x => x.Fields.MachineName!.Contains(machineName!)) |
|
|
application, |
|
|
.AndIf(!environment.IsNullOrWhiteSpace(), x => x.Fields.Environment!.Contains(environment!)) |
|
|
context, |
|
|
.AndIf(!application.IsNullOrWhiteSpace(), x => x.Fields.Application!.Contains(application!)) |
|
|
requestId, |
|
|
.AndIf(!context.IsNullOrWhiteSpace(), x => x.Fields.Context == context) |
|
|
requestPath, |
|
|
.AndIf(!requestId.IsNullOrWhiteSpace(), x => x.Fields.RequestId == requestId) |
|
|
correlationId, |
|
|
.AndIf(!requestPath.IsNullOrWhiteSpace(), x => x.Fields.RequestPath!.StartsWith(requestPath!)) |
|
|
processId, |
|
|
.AndIf(!correlationId.IsNullOrWhiteSpace(), x => x.Fields.CorrelationId!.Contains(correlationId!)) |
|
|
threadId, |
|
|
.AndIf(processId.HasValue, x => x.Fields.ProcessId == processId) |
|
|
hasException); |
|
|
.AndIf(threadId.HasValue, x => x.Fields.ThreadId == threadId) |
|
|
|
|
|
.AndIf(hasException == true, x => x.Exceptions != null) |
|
|
|
|
|
.AndIf(hasException == false, x => x.Exceptions == null); |
|
|
|
|
|
|
|
|
var response = await client.CountAsync<SerilogInfo>((dsl) => |
|
|
return await _expressionQueryService.GetCountAsync( |
|
|
dsl.Indices(indexName) |
|
|
CreateIndex(), |
|
|
.Query(log => log.Bool(b => b.Must(querys.ToArray()))), |
|
|
expression, |
|
|
cancellationToken); |
|
|
cancellationToken); |
|
|
if (response.TryGetErrorMessage(out var errorMessage)) |
|
|
|
|
|
{ |
|
|
|
|
|
Logger.LogWarning("Query log count failed: {errorMessage}", errorMessage); |
|
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
|
|
return response.Count; |
|
|
|
|
|
} |
|
|
} |
|
|
|
|
|
|
|
|
/// <summary>
|
|
|
/// <summary>
|
|
|
@ -295,308 +214,52 @@ public class SerilogElasticsearchLoggingManager : ILoggingManager, ISingletonDep |
|
|
bool includeDetails = false, |
|
|
bool includeDetails = false, |
|
|
CancellationToken cancellationToken = default) |
|
|
CancellationToken cancellationToken = default) |
|
|
{ |
|
|
{ |
|
|
var indexName = CreateIndex(); |
|
|
|
|
|
var client = _clientFactory.Create(); |
|
|
var client = _clientFactory.Create(); |
|
|
var indexMapping = await _indexMappingProvider.GetMappingAsync(indexName, cancellationToken); |
|
|
if (sorting.IsNullOrWhiteSpace()) |
|
|
|
|
|
|
|
|
var sorts = GetOrDefaultSort(indexMapping, sorting); |
|
|
|
|
|
|
|
|
|
|
|
var querys = BuildQueryDescriptor( |
|
|
|
|
|
indexMapping, |
|
|
|
|
|
startTime, |
|
|
|
|
|
endTime, |
|
|
|
|
|
level, |
|
|
|
|
|
machineName, |
|
|
|
|
|
environment, |
|
|
|
|
|
application, |
|
|
|
|
|
context, |
|
|
|
|
|
requestId, |
|
|
|
|
|
requestPath, |
|
|
|
|
|
correlationId, |
|
|
|
|
|
processId, |
|
|
|
|
|
threadId, |
|
|
|
|
|
hasException); |
|
|
|
|
|
|
|
|
|
|
|
var query = new BoolQuery { Must = querys }; |
|
|
|
|
|
|
|
|
|
|
|
var serilogLogs = skipCount >= 10000 && sorts != null |
|
|
|
|
|
? await SearchAfterSerilogLogs(client, query, sorts, maxResultCount, skipCount, cancellationToken) |
|
|
|
|
|
: await SearchFromSizeSerilogLogs(client, query, sorts, maxResultCount, skipCount, cancellationToken); |
|
|
|
|
|
|
|
|
|
|
|
return _objectMapper.Map<List<SerilogInfo>, List<LogInfo>>(serilogLogs); |
|
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
|
|
protected virtual List<Query> BuildQueryDescriptor( |
|
|
|
|
|
IndexMappingInfo indexMappingInfo, |
|
|
|
|
|
DateTime? startTime = null, |
|
|
|
|
|
DateTime? endTime = null, |
|
|
|
|
|
LogLevel? level = null, |
|
|
|
|
|
string? machineName = null, |
|
|
|
|
|
string? environment = null, |
|
|
|
|
|
string? application = null, |
|
|
|
|
|
string? context = null, |
|
|
|
|
|
string? requestId = null, |
|
|
|
|
|
string? requestPath = null, |
|
|
|
|
|
string? correlationId = null, |
|
|
|
|
|
int? processId = null, |
|
|
|
|
|
int? threadId = null, |
|
|
|
|
|
bool? hasException = null) |
|
|
|
|
|
{ |
|
|
|
|
|
var queries = new List<Query>(); |
|
|
|
|
|
|
|
|
|
|
|
if (_currentTenant.IsAvailable) |
|
|
|
|
|
{ |
|
|
|
|
|
queries.Add(new TermQuery(GetField(indexMappingInfo, "fields.TenantId"), _currentTenant.GetId().ToString())); |
|
|
|
|
|
} |
|
|
|
|
|
if (startTime.HasValue) |
|
|
|
|
|
{ |
|
|
|
|
|
queries.Add(new DateRangeQuery(GetField(indexMappingInfo, "@timestamp")) |
|
|
|
|
|
{ |
|
|
|
|
|
Gte = _clock.Normalize(startTime.Value), |
|
|
|
|
|
}); |
|
|
|
|
|
} |
|
|
|
|
|
if (endTime.HasValue) |
|
|
|
|
|
{ |
|
|
|
|
|
queries.Add(new DateRangeQuery(GetField(indexMappingInfo, "@timestamp")) |
|
|
|
|
|
{ |
|
|
|
|
|
Lte = _clock.Normalize(endTime.Value), |
|
|
|
|
|
}); |
|
|
|
|
|
} |
|
|
|
|
|
if (level.HasValue) |
|
|
|
|
|
{ |
|
|
|
|
|
queries.Add(new TermQuery(GetField(indexMappingInfo, "level"), GetLogEventLevel(level.Value).ToString())); |
|
|
|
|
|
} |
|
|
|
|
|
if (!machineName.IsNullOrWhiteSpace()) |
|
|
|
|
|
{ |
|
|
|
|
|
// 模糊匹配
|
|
|
|
|
|
queries.Add(new WildcardQuery(GetField(indexMappingInfo, "fields.MachineName")) |
|
|
|
|
|
{ |
|
|
|
|
|
Value = $"*{machineName}*" |
|
|
|
|
|
}); |
|
|
|
|
|
} |
|
|
|
|
|
if (!environment.IsNullOrWhiteSpace()) |
|
|
|
|
|
{ |
|
|
|
|
|
// 模糊匹配
|
|
|
|
|
|
queries.Add(new WildcardQuery(GetField(indexMappingInfo, "fields.EnvironmentName")) |
|
|
|
|
|
{ |
|
|
|
|
|
Value = $"*{environment}*" |
|
|
|
|
|
}); |
|
|
|
|
|
} |
|
|
|
|
|
if (!application.IsNullOrWhiteSpace()) |
|
|
|
|
|
{ |
|
|
|
|
|
// 模糊匹配
|
|
|
|
|
|
queries.Add(new WildcardQuery(GetField(indexMappingInfo, "fields.ApplicationName")) |
|
|
|
|
|
{ |
|
|
|
|
|
Value = $"*{application}*" |
|
|
|
|
|
}); |
|
|
|
|
|
} |
|
|
|
|
|
if (!context.IsNullOrWhiteSpace()) |
|
|
|
|
|
{ |
|
|
|
|
|
queries.Add(new TermQuery(GetField(indexMappingInfo, "fields.SourceContext"), context)); |
|
|
|
|
|
} |
|
|
|
|
|
if (!requestId.IsNullOrWhiteSpace()) |
|
|
|
|
|
{ |
|
|
|
|
|
queries.Add(new TermQuery(GetField(indexMappingInfo, "fields.RequestId"), requestId)); |
|
|
|
|
|
} |
|
|
|
|
|
if (!requestPath.IsNullOrWhiteSpace()) |
|
|
|
|
|
{ |
|
|
|
|
|
// 前缀匹配
|
|
|
|
|
|
queries.Add(new MatchPhrasePrefixQuery(GetField(indexMappingInfo, "fields.RequestPath"), requestPath)); |
|
|
|
|
|
} |
|
|
|
|
|
if (!correlationId.IsNullOrWhiteSpace()) |
|
|
|
|
|
{ |
|
|
|
|
|
// 模糊匹配
|
|
|
|
|
|
queries.Add(new WildcardQuery(GetField(indexMappingInfo, "fields.CorrelationId")) |
|
|
|
|
|
{ |
|
|
|
|
|
Value = $"*{correlationId}*" |
|
|
|
|
|
}); |
|
|
|
|
|
} |
|
|
|
|
|
if (processId.HasValue) |
|
|
|
|
|
{ |
|
|
|
|
|
queries.Add(new TermQuery(GetField(indexMappingInfo, "fields.ProcessId"), FieldValue.FromValue(processId.Value))); |
|
|
|
|
|
} |
|
|
|
|
|
if (threadId.HasValue) |
|
|
|
|
|
{ |
|
|
|
|
|
queries.Add(new TermQuery(GetField(indexMappingInfo, "fields.ThreadId"), FieldValue.FromValue(threadId.Value))); |
|
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
|
|
if (hasException.HasValue) |
|
|
|
|
|
{ |
|
|
|
|
|
if (hasException.Value) |
|
|
|
|
|
{ |
|
|
|
|
|
/* 存在exceptions字段则就是有异常信息 |
|
|
|
|
|
* "exists": { |
|
|
|
|
|
"field": "exceptions" |
|
|
|
|
|
} |
|
|
|
|
|
*/ |
|
|
|
|
|
queries.Add(new ExistsQuery(GetField(indexMappingInfo, "fields.Exceptions"))); |
|
|
|
|
|
} |
|
|
|
|
|
else |
|
|
|
|
|
{ |
|
|
|
|
|
// 不存在 exceptions字段就是没有异常信息的消息
|
|
|
|
|
|
/* |
|
|
|
|
|
* "bool": { |
|
|
|
|
|
"must_not": [ |
|
|
|
|
|
{ |
|
|
|
|
|
"exists": { |
|
|
|
|
|
"field": "exceptions" |
|
|
|
|
|
} |
|
|
|
|
|
} |
|
|
|
|
|
] |
|
|
|
|
|
} |
|
|
|
|
|
*/ |
|
|
|
|
|
queries.Add(new BoolQuery |
|
|
|
|
|
{ |
|
|
|
|
|
MustNot = new List<Query> |
|
|
|
|
|
{ |
|
|
|
|
|
new ExistsQuery(GetField(indexMappingInfo, "fields.Exceptions")) |
|
|
|
|
|
} |
|
|
|
|
|
}); |
|
|
|
|
|
} |
|
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
|
|
return queries; |
|
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
|
|
private async Task<List<SerilogInfo>> SearchFromSizeSerilogLogs( |
|
|
|
|
|
ElasticsearchClient client, |
|
|
|
|
|
Query query, |
|
|
|
|
|
SortOptions[]? sorts = null, |
|
|
|
|
|
int maxResultCount = 50, |
|
|
|
|
|
int skipCount = 0, |
|
|
|
|
|
CancellationToken cancellationToken = default) |
|
|
|
|
|
{ |
|
|
|
|
|
var searchResponse = await client.SearchAsync<SerilogInfo>(dsl => |
|
|
|
|
|
{ |
|
|
|
|
|
dsl.Indices(CreateIndex()) |
|
|
|
|
|
.Query(query) |
|
|
|
|
|
.From(skipCount) |
|
|
|
|
|
.Size(maxResultCount); |
|
|
|
|
|
if (sorts != null) |
|
|
|
|
|
{ |
|
|
|
|
|
dsl.Sort(sorts); |
|
|
|
|
|
} |
|
|
|
|
|
}, cancellationToken); |
|
|
|
|
|
|
|
|
|
|
|
if (searchResponse.TryGetErrorMessage(out var errorMessage)) |
|
|
|
|
|
{ |
|
|
|
|
|
Logger.LogWarning("Query log failed: {errorMessage}", errorMessage); |
|
|
|
|
|
return []; |
|
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
|
|
return searchResponse.Documents.ToList(); |
|
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
|
|
private async Task<List<SerilogInfo>> SearchAfterSerilogLogs( |
|
|
|
|
|
ElasticsearchClient client, |
|
|
|
|
|
Query query, |
|
|
|
|
|
SortOptions[] sorts, |
|
|
|
|
|
int maxResultCount = 50, |
|
|
|
|
|
int skipCount = 0, |
|
|
|
|
|
CancellationToken cancellationToken = default) |
|
|
|
|
|
{ |
|
|
|
|
|
var searchAfter = await GetSearchAfterValue( |
|
|
|
|
|
client, |
|
|
|
|
|
query, |
|
|
|
|
|
sorts, |
|
|
|
|
|
skipCount, |
|
|
|
|
|
cancellationToken); |
|
|
|
|
|
|
|
|
|
|
|
if (searchAfter == null || !searchAfter.Any()) |
|
|
|
|
|
{ |
|
|
|
|
|
return []; |
|
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
|
|
var searchResponse = await client.SearchAsync<SerilogInfo>(dsl => |
|
|
|
|
|
{ |
|
|
|
|
|
dsl.Indices(CreateIndex()) |
|
|
|
|
|
.Query(query) |
|
|
|
|
|
.Sort(sorts) |
|
|
|
|
|
.Size(maxResultCount) |
|
|
|
|
|
.SearchAfter(searchAfter); |
|
|
|
|
|
}, cancellationToken); |
|
|
|
|
|
|
|
|
|
|
|
if (searchResponse.TryGetErrorMessage(out var errorMessage)) |
|
|
|
|
|
{ |
|
|
|
|
|
Logger.LogWarning("Query log failed: {errorMessage}", errorMessage); |
|
|
|
|
|
return []; |
|
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
|
|
return searchResponse.Documents.ToList(); |
|
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
|
|
private async Task<List<FieldValue>?> GetSearchAfterValue( |
|
|
|
|
|
ElasticsearchClient client, |
|
|
|
|
|
Query query, |
|
|
|
|
|
SortOptions[] sorts, |
|
|
|
|
|
int skipCount, |
|
|
|
|
|
CancellationToken cancellationToken = default) |
|
|
|
|
|
{ |
|
|
|
|
|
// 10000以内直接取最后一条数据
|
|
|
|
|
|
if (skipCount < 10000) |
|
|
|
|
|
{ |
|
|
|
|
|
var response = await client.SearchAsync<SerilogInfo>( |
|
|
|
|
|
dsl => dsl.Indices(CreateIndex()) |
|
|
|
|
|
.Query(query) |
|
|
|
|
|
.Sort(sorts) |
|
|
|
|
|
.SourceIncludes(x => x.Level) |
|
|
|
|
|
.From(skipCount) |
|
|
|
|
|
.Size(1), |
|
|
|
|
|
cancellationToken); |
|
|
|
|
|
|
|
|
|
|
|
if (!response.IsSuccess() || response.Hits == null || !response.Hits.Any()) |
|
|
|
|
|
{ |
|
|
|
|
|
return null; |
|
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
|
|
var hit = response.Hits.FirstOrDefault(); |
|
|
|
|
|
return hit?.Sort?.ToList(); |
|
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
|
|
// 获取第9999条数据Hits作为searchAfter
|
|
|
|
|
|
var firstResponse = await client.SearchAsync<SerilogInfo>( |
|
|
|
|
|
dsl => dsl.Indices(CreateIndex()) |
|
|
|
|
|
.Query(query) |
|
|
|
|
|
.Sort(sorts) |
|
|
|
|
|
.SourceIncludes(x => x.Level) |
|
|
|
|
|
.From(9999) |
|
|
|
|
|
.Size(1), |
|
|
|
|
|
cancellationToken); |
|
|
|
|
|
|
|
|
|
|
|
if (!firstResponse.IsSuccess() || firstResponse.Hits == null || !firstResponse.Hits.Any()) |
|
|
|
|
|
{ |
|
|
{ |
|
|
return null; |
|
|
sorting = $"{ElasticsearchJsonFormatter.TimestampPropertyName} DESC"; |
|
|
} |
|
|
} |
|
|
|
|
|
// 额外处理一下timestamp字段
|
|
|
var firstHit = firstResponse.Hits.FirstOrDefault(); |
|
|
else if (sorting.Contains("timestamp", StringComparison.CurrentCultureIgnoreCase)) |
|
|
if (firstHit?.Sort == null || !firstHit.Sort.Any()) |
|
|
|
|
|
{ |
|
|
{ |
|
|
return null; |
|
|
sorting = sorting |
|
|
|
|
|
.Replace("timestamp", ElasticsearchJsonFormatter.TimestampPropertyName, StringComparison.CurrentCultureIgnoreCase) |
|
|
|
|
|
.Replace("@@", "@"); |
|
|
} |
|
|
} |
|
|
|
|
|
|
|
|
// 获取skipCount最近一条数据作为searchAfter
|
|
|
Expression<Func<SerilogInfo, bool>> expression = _ => true; |
|
|
var secondResponse = await client.SearchAsync<SerilogInfo>( |
|
|
|
|
|
dsl => dsl.Indices(CreateIndex()) |
|
|
expression = expression |
|
|
.Query(query) |
|
|
.AndIf(startTime.HasValue, x => x.TimeStamp >= _clock.Normalize(startTime!.Value)) |
|
|
// 反转排序取第一个数据作为起始索引
|
|
|
.AndIf(endTime.HasValue, x => x.TimeStamp <= _clock.Normalize(endTime!.Value)) |
|
|
.Sort(sorts.ReverseSort()!.ToArray()) |
|
|
.AndIf(level.HasValue, x => x.Level == GetLogEventLevel(level!.Value)) |
|
|
.SourceIncludes(x => x.Level) |
|
|
.AndIf(!machineName.IsNullOrWhiteSpace(), x => x.Fields.MachineName!.Contains(machineName!)) |
|
|
.SearchAfter(firstHit.Sort.ToList()) |
|
|
.AndIf(!environment.IsNullOrWhiteSpace(), x => x.Fields.Environment!.Contains(environment!)) |
|
|
.Size(1), |
|
|
.AndIf(!application.IsNullOrWhiteSpace(), x => x.Fields.Application!.Contains(application!)) |
|
|
cancellationToken); |
|
|
.AndIf(!context.IsNullOrWhiteSpace(), x => x.Fields.Context == context) |
|
|
|
|
|
.AndIf(!requestId.IsNullOrWhiteSpace(), x => x.Fields.RequestId == requestId) |
|
|
if (!secondResponse.IsSuccess() || secondResponse.Hits == null || !secondResponse.Hits.Any()) |
|
|
.AndIf(!requestPath.IsNullOrWhiteSpace(), x => x.Fields.RequestPath!.StartsWith(requestPath!)) |
|
|
{ |
|
|
.AndIf(!correlationId.IsNullOrWhiteSpace(), x => x.Fields.CorrelationId!.Contains(correlationId!)) |
|
|
return null; |
|
|
.AndIf(processId.HasValue, x => x.Fields.ProcessId == processId) |
|
|
} |
|
|
.AndIf(threadId.HasValue, x => x.Fields.ThreadId == threadId) |
|
|
|
|
|
.AndIf(hasException == true, x => x.Exceptions != null) |
|
|
|
|
|
.AndIf(hasException == false, x => x.Exceptions == null); |
|
|
|
|
|
|
|
|
var lastHit = secondResponse.Hits.LastOrDefault(); |
|
|
var serilogLogs = await _expressionQueryService.GetListAsync( |
|
|
if (lastHit?.Sort == null || !lastHit.Sort.Any()) |
|
|
CreateIndex(), |
|
|
{ |
|
|
expression, |
|
|
return null; |
|
|
sorting: sorting, |
|
|
} |
|
|
maxResultCount: maxResultCount, |
|
|
|
|
|
skipCount: skipCount, |
|
|
|
|
|
sourceExcludes: includeDetails == false |
|
|
|
|
|
? Fields.FromFields( |
|
|
|
|
|
[ |
|
|
|
|
|
new Field("exceptions"), |
|
|
|
|
|
]) |
|
|
|
|
|
: null, |
|
|
|
|
|
cancellationToken: cancellationToken); |
|
|
|
|
|
|
|
|
return lastHit.Sort.ToList(); |
|
|
return _objectMapper.Map<List<SerilogInfo>, List<LogInfo>>(serilogLogs); |
|
|
} |
|
|
} |
|
|
|
|
|
|
|
|
protected virtual string CreateIndex(DateTimeOffset? offset = null) |
|
|
protected virtual string CreateIndex(DateTimeOffset? offset = null) |
|
|
@ -620,43 +283,4 @@ public class SerilogElasticsearchLoggingManager : ILoggingManager, ISingletonDep |
|
|
_ => LogEventLevel.Verbose, |
|
|
_ => LogEventLevel.Verbose, |
|
|
}; |
|
|
}; |
|
|
} |
|
|
} |
|
|
|
|
|
|
|
|
private static SortOptions[]? GetOrDefaultSort(IndexMappingInfo indexMappingInfo, string? sorting = null) |
|
|
|
|
|
{ |
|
|
|
|
|
var sortOrder = !sorting.IsNullOrWhiteSpace() && sorting.EndsWith("asc", StringComparison.InvariantCultureIgnoreCase) |
|
|
|
|
|
? SortOrder.Asc : SortOrder.Desc; |
|
|
|
|
|
sorting = !sorting.IsNullOrWhiteSpace() |
|
|
|
|
|
? sorting.Split()[0] |
|
|
|
|
|
: ElasticsearchJsonFormatter.TimestampPropertyName; |
|
|
|
|
|
|
|
|
|
|
|
SortOptions[]? sorts = null; |
|
|
|
|
|
var sortingFieldMap = indexMappingInfo.Fields |
|
|
|
|
|
.Where(x => x.Key.Equals(sorting, StringComparison.CurrentCultureIgnoreCase)) |
|
|
|
|
|
.Select(x => x.Value) |
|
|
|
|
|
.FirstOrDefault(); |
|
|
|
|
|
if (sortingFieldMap != null) |
|
|
|
|
|
{ |
|
|
|
|
|
sorting = sortingFieldMap.Path; |
|
|
|
|
|
} |
|
|
|
|
|
if (!sorting.IsNullOrWhiteSpace()) |
|
|
|
|
|
{ |
|
|
|
|
|
sorts = new SortOptions[1] |
|
|
|
|
|
{ |
|
|
|
|
|
new SortOptions |
|
|
|
|
|
{ |
|
|
|
|
|
Field = new FieldSort(new Field(sorting)) |
|
|
|
|
|
{ |
|
|
|
|
|
Order = sortOrder, |
|
|
|
|
|
}, |
|
|
|
|
|
} |
|
|
|
|
|
}; |
|
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
|
|
return sorts; |
|
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
|
|
private static string GetField(IndexMappingInfo indexMappingInfo, string fieldFullPath) |
|
|
|
|
|
{ |
|
|
|
|
|
return indexMappingInfo.GetExactFieldPath(fieldFullPath); |
|
|
|
|
|
} |
|
|
|
|
|
} |
|
|
} |
|
|
|
colin
3 weeks ago