From 259b8178a4d326a4b482ef361df06e7c7c4a7717 Mon Sep 17 00:00:00 2001 From: cKey <35512826+colinin@users.noreply.github.com> Date: Sat, 30 Oct 2021 11:47:13 +0800 Subject: [PATCH] feat(logging): add log query parameters. --- .../ElasticsearchAuditLogManager.cs | 36 +++++++++++--- .../ElasticsearchSecurityLogManager.cs | 33 ++++++++++--- .../Auditing/Localization/Resources/en.json | 23 ++++++++- .../Localization/Resources/zh-Hans.json | 23 ++++++++- .../Auditing/Logging/Dto/LogGetByPagedDto.cs | 4 +- .../SerilogElasticsearchLoggingManager.cs | 47 ++++++++++++++++++- .../Abp/AuditLogging/DefaultLoggingManager.cs | 4 +- .../Abp/AuditLogging/ILoggingManager.cs | 5 +- .../appsettings.Development.json | 5 ++ .../appsettings.Development.json | 5 ++ .../appsettings.Development.json | 5 ++ .../appsettings.Development.json | 5 ++ .../appsettings.Development.json | 5 ++ 13 files changed, 181 insertions(+), 19 deletions(-) diff --git a/aspnet-core/modules/auditing/LINGYUN.Abp.AuditLogging.Elasticsearch/LINGYUN/Abp/AuditLogging/Elasticsearch/ElasticsearchAuditLogManager.cs b/aspnet-core/modules/auditing/LINGYUN.Abp.AuditLogging.Elasticsearch/LINGYUN/Abp/AuditLogging/Elasticsearch/ElasticsearchAuditLogManager.cs index 56f1e1703..e4caf10d2 100644 --- a/aspnet-core/modules/auditing/LINGYUN.Abp.AuditLogging.Elasticsearch/LINGYUN/Abp/AuditLogging/Elasticsearch/ElasticsearchAuditLogManager.cs +++ b/aspnet-core/modules/auditing/LINGYUN.Abp.AuditLogging.Elasticsearch/LINGYUN/Abp/AuditLogging/Elasticsearch/ElasticsearchAuditLogManager.cs @@ -119,11 +119,6 @@ namespace LINGYUN.Abp.AuditLogging.Elasticsearch ? sorting.Split()[0] : nameof(AuditLog.ExecutionTime); - if (_elasticsearchOptions.FieldCamelCase) - { - sorting = sorting.ToCamelCase(); - } - var querys = BuildQueryDescriptor( startTime, endTime, @@ -159,7 +154,7 @@ namespace LINGYUN.Abp.AuditLogging.Elasticsearch dsl.Index(CreateIndex()) .Query(log => log.Bool(b => b.Must(querys.ToArray()))) .Source(ConvertFileSystem) - .Sort(log => log.Field(sorting, sortOrder)) + .Sort(log => log.Field(GetField(sorting), sortOrder)) .From(skipCount) .Size(maxResultCount), cancellationToken); @@ -343,5 +338,34 @@ namespace LINGYUN.Abp.AuditLogging.Elasticsearch ? "audit-log" : $"{_options.IndexPrefix}-audit-log"; } + + private readonly static IDictionary _fieldMaps = new Dictionary(StringComparer.InvariantCultureIgnoreCase) + { + { "Id", "Id.keyword" }, + { "ApplicationName", "ApplicationName.keyword" }, + { "UserId", "UserId.keyword" }, + { "UserName", "UserName.keyword" }, + { "TenantId", "TenantId.keyword" }, + { "TenantName", "TenantName.keyword" }, + { "ImpersonatorUserId", "ImpersonatorUserId.keyword" }, + { "ImpersonatorTenantId", "ImpersonatorTenantId.keyword" }, + { "ClientName", "ClientName.keyword" }, + { "ClientIpAddress", "ClientIpAddress.keyword" }, + { "ClientId", "ClientId.keyword" }, + { "CorrelationId", "CorrelationId.keyword" }, + { "BrowserInfo", "BrowserInfo.keyword" }, + { "HttpMethod", "HttpMethod.keyword" }, + { "Url", "Url.keyword" } + }; + protected virtual string GetField(string field) + { + field = _elasticsearchOptions.FieldCamelCase ? field.ToCamelCase() : field.ToPascalCase(); + if (_fieldMaps.TryGetValue(field, out string mapField)) + { + return mapField; + } + + return field; + } } } diff --git a/aspnet-core/modules/auditing/LINGYUN.Abp.AuditLogging.Elasticsearch/LINGYUN/Abp/AuditLogging/Elasticsearch/ElasticsearchSecurityLogManager.cs b/aspnet-core/modules/auditing/LINGYUN.Abp.AuditLogging.Elasticsearch/LINGYUN/Abp/AuditLogging/Elasticsearch/ElasticsearchSecurityLogManager.cs index 03275f9de..e427021c3 100644 --- a/aspnet-core/modules/auditing/LINGYUN.Abp.AuditLogging.Elasticsearch/LINGYUN/Abp/AuditLogging/Elasticsearch/ElasticsearchSecurityLogManager.cs +++ b/aspnet-core/modules/auditing/LINGYUN.Abp.AuditLogging.Elasticsearch/LINGYUN/Abp/AuditLogging/Elasticsearch/ElasticsearchSecurityLogManager.cs @@ -120,11 +120,6 @@ namespace LINGYUN.Abp.AuditLogging.Elasticsearch ? sorting.Split()[0] : nameof(SecurityLog.CreationTime); - if (_elasticsearchOptions.FieldCamelCase) - { - sorting = sorting.ToCamelCase(); - } - var querys = BuildQueryDescriptor( startTime, endTime, @@ -141,7 +136,7 @@ namespace LINGYUN.Abp.AuditLogging.Elasticsearch dsl.Index(CreateIndex()) .Query(log => log.Bool(b => b.Must(querys.ToArray()))) .Source(log => log.IncludeAll()) - .Sort(log => log.Field(sorting, sortOrder)) + .Sort(log => log.Field(GetField(sorting), sortOrder)) .From(skipCount) .Size(maxResultCount), cancellationToken); @@ -254,5 +249,31 @@ namespace LINGYUN.Abp.AuditLogging.Elasticsearch ? "security-log" : $"{_options.IndexPrefix}-security-log"; } + + private readonly static IDictionary _fieldMaps = new Dictionary(StringComparer.InvariantCultureIgnoreCase) + { + { "Id", "Id.keyword" }, + { "ApplicationName", "ApplicationName.keyword" }, + { "UserId", "UserId.keyword" }, + { "UserName", "UserName.keyword" }, + { "TenantId", "TenantId.keyword" }, + { "TenantName", "TenantName.keyword" }, + { "Identity", "Identity.keyword" }, + { "Action", "Action.keyword" }, + { "BrowserInfo", "BrowserInfo.keyword" }, + { "ClientIpAddress", "ClientIpAddress.keyword" }, + { "ClientId", "ClientId.keyword" }, + { "CorrelationId", "CorrelationId.keyword" }, + }; + protected virtual string GetField(string field) + { + field = _elasticsearchOptions.FieldCamelCase ? field.ToCamelCase() : field.ToPascalCase(); + if (_fieldMaps.TryGetValue(field, out string mapField)) + { + return mapField; + } + + return field; + } } } diff --git a/aspnet-core/modules/auditing/LINGYUN.Abp.Auditing.Application.Contracts/LINGYUN/Abp/Auditing/Localization/Resources/en.json b/aspnet-core/modules/auditing/LINGYUN.Abp.Auditing.Application.Contracts/LINGYUN/Abp/Auditing/Localization/Resources/en.json index 22046fb52..6e15b1eb2 100644 --- a/aspnet-core/modules/auditing/LINGYUN.Abp.Auditing.Application.Contracts/LINGYUN/Abp/Auditing/Localization/Resources/en.json +++ b/aspnet-core/modules/auditing/LINGYUN.Abp.Auditing.Application.Contracts/LINGYUN/Abp/Auditing/Localization/Resources/en.json @@ -12,6 +12,7 @@ "Features:Description:SecurityLog": "Whether to enable security logging", "SecurityLog": "Security log", "AuditLog": "Audit log", + "Logging": "System Log", "Application": "Application", "ApplicationName": "Application name", "TenantId": "Tenant id", @@ -62,6 +63,26 @@ "Updated": "Updated", "Deleted": "Deleted", "ShowLogDialog": "Show log", - "DeleteLog": "Delete log" + "DeleteLog": "Delete log", + "MachineName": "Machine", + "Environment": "Environment", + "Context": "Context", + "RequestId": "Request Id", + "RequestPath": "Request Path", + "ProcessId": "Process Id", + "ThreadId": "Thread Id", + "ActionId": "Action Id", + "ConnectionId": "Connection Id", + "Depth": "Depth", + "Class": "Class", + "Message": "Message", + "Source": "Source", + "StackTrace": "Stack Trace", + "HResult": "HResult", + "HelpURL": "Help Url", + "TimeStamp": "Timestamp", + "Level": "Level", + "Fields": "Fields", + "Exceptions": "Exceptions" } } \ No newline at end of file diff --git a/aspnet-core/modules/auditing/LINGYUN.Abp.Auditing.Application.Contracts/LINGYUN/Abp/Auditing/Localization/Resources/zh-Hans.json b/aspnet-core/modules/auditing/LINGYUN.Abp.Auditing.Application.Contracts/LINGYUN/Abp/Auditing/Localization/Resources/zh-Hans.json index a741e3af8..4f86c8238 100644 --- a/aspnet-core/modules/auditing/LINGYUN.Abp.Auditing.Application.Contracts/LINGYUN/Abp/Auditing/Localization/Resources/zh-Hans.json +++ b/aspnet-core/modules/auditing/LINGYUN.Abp.Auditing.Application.Contracts/LINGYUN/Abp/Auditing/Localization/Resources/zh-Hans.json @@ -12,6 +12,7 @@ "Features:Description:SecurityLog": "是否启用安全日志功能", "SecurityLog": "安全日志", "AuditLog": "审计日志", + "Logging": "系统日志", "Application": "应用信息", "ApplicationName": "应用名称", "TenantId": "租户标识", @@ -62,6 +63,26 @@ "Updated": "修改", "Deleted": "删除", "ShowLogDialog": "查看日志", - "DeleteLog": "删除日志" + "DeleteLog": "删除日志", + "MachineName": "机器名称", + "Environment": "应用环境", + "Context": "上下文", + "RequestId": "请求标识", + "RequestPath": "请求路径", + "ProcessId": "进程标识", + "ThreadId": "线程标识", + "ActionId": "方法标识", + "ConnectionId": "连接标识", + "Depth": "深度", + "Class": "类型", + "Message": "信息", + "Source": "来源", + "StackTrace": "堆栈", + "HResult": "代码", + "HelpURL": "帮助", + "TimeStamp": "时间戳", + "Level": "级别", + "Fields": "字段", + "Exceptions": "错误信息" } } \ No newline at end of file diff --git a/aspnet-core/modules/auditing/LINGYUN.Abp.Auditing.Application.Contracts/LINGYUN/Abp/Auditing/Logging/Dto/LogGetByPagedDto.cs b/aspnet-core/modules/auditing/LINGYUN.Abp.Auditing.Application.Contracts/LINGYUN/Abp/Auditing/Logging/Dto/LogGetByPagedDto.cs index 76e89e440..022704baa 100644 --- a/aspnet-core/modules/auditing/LINGYUN.Abp.Auditing.Application.Contracts/LINGYUN/Abp/Auditing/Logging/Dto/LogGetByPagedDto.cs +++ b/aspnet-core/modules/auditing/LINGYUN.Abp.Auditing.Application.Contracts/LINGYUN/Abp/Auditing/Logging/Dto/LogGetByPagedDto.cs @@ -1,4 +1,5 @@ -using System; +using Microsoft.Extensions.Logging; +using System; using Volo.Abp.Application.Dtos; namespace LINGYUN.Abp.Auditing.Logging @@ -7,6 +8,7 @@ namespace LINGYUN.Abp.Auditing.Logging { public DateTime? StartTime { get; set; } public DateTime? EndTime { get; set; } + public LogLevel? Level { get; set; } public string MachineName { get; set; } public string Environment { get; set; } public string Application { get; set; } diff --git a/aspnet-core/modules/logging/LINGYUN.Abp.Logging.Serilog.Elasticsearch/LINGYUN/Abp/AuditLogging/Serilog/Elasticsearch/SerilogElasticsearchLoggingManager.cs b/aspnet-core/modules/logging/LINGYUN.Abp.Logging.Serilog.Elasticsearch/LINGYUN/Abp/AuditLogging/Serilog/Elasticsearch/SerilogElasticsearchLoggingManager.cs index c6ea6b7cc..2172348dd 100644 --- a/aspnet-core/modules/logging/LINGYUN.Abp.Logging.Serilog.Elasticsearch/LINGYUN/Abp/AuditLogging/Serilog/Elasticsearch/SerilogElasticsearchLoggingManager.cs +++ b/aspnet-core/modules/logging/LINGYUN.Abp.Logging.Serilog.Elasticsearch/LINGYUN/Abp/AuditLogging/Serilog/Elasticsearch/SerilogElasticsearchLoggingManager.cs @@ -1,4 +1,5 @@ using LINGYUN.Abp.Elasticsearch; +using LINGYUN.Abp.Serilog.Enrichers.Application; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Logging.Abstractions; using Microsoft.Extensions.Options; @@ -65,6 +66,7 @@ namespace LINGYUN.Abp.Logging.Serilog.Elasticsearch public virtual async Task GetCountAsync( DateTime? startTime = null, DateTime? endTime = null, + Microsoft.Extensions.Logging.LogLevel? level = null, string machineName = null, string environment = null, string application = null, @@ -82,6 +84,7 @@ namespace LINGYUN.Abp.Logging.Serilog.Elasticsearch var querys = BuildQueryDescriptor( startTime, endTime, + level, machineName, environment, application, @@ -104,11 +107,12 @@ namespace LINGYUN.Abp.Logging.Serilog.Elasticsearch /// /// 获取日志列表 /// - /// 排序字段，注意：忽略排序字段仅使用timestamp排序，根据传递的ASC、DESC字段区分倒序还是正序 + /// 排序字段 /// /// /// /// + /// /// /// /// @@ -128,6 +132,7 @@ namespace LINGYUN.Abp.Logging.Serilog.Elasticsearch int skipCount = 0, DateTime? startTime = null, DateTime? endTime = null, + Microsoft.Extensions.Logging.LogLevel? level = null, string machineName = null, string environment = null, string application = null, @@ -149,6 +154,7 @@ namespace LINGYUN.Abp.Logging.Serilog.Elasticsearch var querys = BuildQueryDescriptor( startTime, endTime, + level, machineName, environment, application, @@ -176,7 +182,7 @@ namespace LINGYUN.Abp.Logging.Serilog.Elasticsearch dsl.Index(CreateIndex()) .Query(log => log.Bool(b => b.Must(querys.ToArray()))) .Source(ConvertFileSystem) - .Sort(log => log.Field("@timestamp", sortOrder)) + .Sort(log => log.Field(GetField(sorting), sortOrder)) .From(skipCount) .Size(maxResultCount), cancellationToken); @@ -187,6 +193,7 @@ namespace LINGYUN.Abp.Logging.Serilog.Elasticsearch protected virtual List, QueryContainer>> BuildQueryDescriptor( DateTime? startTime = null, DateTime? endTime = null, + Microsoft.Extensions.Logging.LogLevel? level = null, string machineName = null, string environment = null, string application = null, @@ -208,6 +215,10 @@ namespace LINGYUN.Abp.Logging.Serilog.Elasticsearch { querys.Add((log) => log.DateRange((q) => q.Field(f => f.TimeStamp).LessThanOrEquals(endTime))); } + if (level.HasValue) + { + querys.Add((log) => log.Term((q) => q.Field(f => f.Level.Suffix("keyword")).Value(level))); + } if (!machineName.IsNullOrWhiteSpace()) { querys.Add((log) => log.Term((q) => q.Field((f) => f.Fields.MachineName.Suffix("keyword")).Value(machineName))); @@ -291,5 +302,37 @@ namespace LINGYUN.Abp.Logging.Serilog.Elasticsearch } return string.Format(_options.IndexFormat, offset.Value).ToLowerInvariant(); } + + private readonly static IDictionary _fieldMaps = new Dictionary(StringComparer.InvariantCultureIgnoreCase) + { + { "timestamp", "@timestamp" }, + { "level", "level.keyword" }, + { "machinename", $"fields.{AbpLoggingEnricherPropertyNames.MachineName}.keyword" }, + { "environment", $"fields.{AbpLoggingEnricherPropertyNames.EnvironmentName}.keyword" }, + { "application", $"fields.{AbpSerilogEnrichersConsts.ApplicationNamePropertyName}.keyword" }, + { "context", "fields.Context.keyword" }, + { "actionid", "fields.ActionId.keyword" }, + { "actionname", "fields.ActionName.keyword" }, + { "requestid", "fields.RequestId.keyword" }, + { "requestpath", "fields.RequestPath.keyword" }, + { "connectionid", "fields.ConnectionId.keyword" }, + { "correlationid", "fields.CorrelationId.keyword" }, + { "clientid", "fields.ClientId.keyword" }, + { "userid", "fields.UserId.keyword" }, + { "processid", "fields.ProcessId" }, + { "threadid", "fields.ThreadId" }, + }; + protected virtual string GetField(string field) + { + foreach (var fieldMap in _fieldMaps) + { + if (field.ToLowerInvariant().Contains(fieldMap.Key)) + { + return fieldMap.Value; + } + } + + return field; + } } } diff --git a/aspnet-core/modules/logging/LINGYUN.Abp.Logging/LINGYUN/Abp/AuditLogging/DefaultLoggingManager.cs b/aspnet-core/modules/logging/LINGYUN.Abp.Logging/LINGYUN/Abp/AuditLogging/DefaultLoggingManager.cs index 602df4682..c5d363a11 100644 --- a/aspnet-core/modules/logging/LINGYUN.Abp.Logging/LINGYUN/Abp/AuditLogging/DefaultLoggingManager.cs +++ b/aspnet-core/modules/logging/LINGYUN.Abp.Logging/LINGYUN/Abp/AuditLogging/DefaultLoggingManager.cs @@ -27,7 +27,8 @@ namespace LINGYUN.Abp.Logging public Task GetCountAsync( DateTime? startTime = null, - DateTime? endTime = null, + DateTime? endTime = null, + LogLevel? level = null, string machineName = null, string environment = null, string application = null, @@ -50,6 +51,7 @@ namespace LINGYUN.Abp.Logging int skipCount = 0, DateTime? startTime = null, DateTime? endTime = null, + LogLevel? level = null, string machineName = null, string environment = null, string application = null, diff --git a/aspnet-core/modules/logging/LINGYUN.Abp.Logging/LINGYUN/Abp/AuditLogging/ILoggingManager.cs b/aspnet-core/modules/logging/LINGYUN.Abp.Logging/LINGYUN/Abp/AuditLogging/ILoggingManager.cs index d15e26010..cad65eea8 100644 --- a/aspnet-core/modules/logging/LINGYUN.Abp.Logging/LINGYUN/Abp/AuditLogging/ILoggingManager.cs +++ b/aspnet-core/modules/logging/LINGYUN.Abp.Logging/LINGYUN/Abp/AuditLogging/ILoggingManager.cs @@ -1,4 +1,5 @@ -using System; +using Microsoft.Extensions.Logging; +using System; using System.Collections.Generic; using System.Threading; using System.Threading.Tasks; @@ -14,6 +15,7 @@ namespace LINGYUN.Abp.Logging Task GetCountAsync( DateTime? startTime = null, DateTime? endTime = null, + LogLevel? level = null, string machineName = null, string environment = null, string application = null, @@ -32,6 +34,7 @@ namespace LINGYUN.Abp.Logging int skipCount = 0, DateTime? startTime = null, DateTime? endTime = null, + LogLevel? level = null, string machineName = null, string environment = null, string application = null, diff --git a/aspnet-core/services/apigateway/LINGYUN.ApiGateway.HttpApi.Host/appsettings.Development.json b/aspnet-core/services/apigateway/LINGYUN.ApiGateway.HttpApi.Host/appsettings.Development.json index c9bf7bbde..45365e3c4 100644 --- a/aspnet-core/services/apigateway/LINGYUN.ApiGateway.HttpApi.Host/appsettings.Development.json +++ b/aspnet-core/services/apigateway/LINGYUN.ApiGateway.HttpApi.Host/appsettings.Development.json @@ -38,6 +38,11 @@ "Authority": "http://localhost:44385/", "ApiName": "lingyun-abp-application" }, + "AuditLogging": { + "Elasticsearch": { + "IndexPrefix": "lingyun.abp.auditing" + } + }, "Elasticsearch": { "NodeUris": "http://localhost:9200" }, diff --git a/aspnet-core/services/identity-server/LINGYUN.Abp.IdentityServer4.HttpApi.Host/appsettings.Development.json b/aspnet-core/services/identity-server/LINGYUN.Abp.IdentityServer4.HttpApi.Host/appsettings.Development.json index bcff4003a..759c70842 100644 --- a/aspnet-core/services/identity-server/LINGYUN.Abp.IdentityServer4.HttpApi.Host/appsettings.Development.json +++ b/aspnet-core/services/identity-server/LINGYUN.Abp.IdentityServer4.HttpApi.Host/appsettings.Development.json @@ -40,6 +40,11 @@ "Authority": "http://localhost:44385/", "ApiName": "lingyun-abp-application" }, + "AuditLogging": { + "Elasticsearch": { + "IndexPrefix": "lingyun.abp.auditing" + } + }, "Elasticsearch": { "NodeUris": "http://localhost:9200" }, diff --git a/aspnet-core/services/localization/LINGYUN.Abp.LocalizationManagement.HttpApi.Host/appsettings.Development.json b/aspnet-core/services/localization/LINGYUN.Abp.LocalizationManagement.HttpApi.Host/appsettings.Development.json index 17bc0b83e..61deb00b4 100644 --- a/aspnet-core/services/localization/LINGYUN.Abp.LocalizationManagement.HttpApi.Host/appsettings.Development.json +++ b/aspnet-core/services/localization/LINGYUN.Abp.LocalizationManagement.HttpApi.Host/appsettings.Development.json @@ -37,6 +37,11 @@ "Authority": "http://localhost:44385/", "ApiName": "lingyun-abp-application" }, + "AuditLogging": { + "Elasticsearch": { + "IndexPrefix": "lingyun.abp.auditing" + } + }, "Elasticsearch": { "NodeUris": "http://localhost:9200" }, diff --git a/aspnet-core/services/messages/LINGYUN.Abp.MessageService.HttpApi.Host/appsettings.Development.json b/aspnet-core/services/messages/LINGYUN.Abp.MessageService.HttpApi.Host/appsettings.Development.json index e68dc5a4f..62bde980b 100644 --- a/aspnet-core/services/messages/LINGYUN.Abp.MessageService.HttpApi.Host/appsettings.Development.json +++ b/aspnet-core/services/messages/LINGYUN.Abp.MessageService.HttpApi.Host/appsettings.Development.json @@ -67,6 +67,11 @@ "VirtualHost": "/" } }, + "AuditLogging": { + "Elasticsearch": { + "IndexPrefix": "lingyun.abp.auditing" + } + }, "Elasticsearch": { "NodeUris": "http://localhost:9200" }, diff --git a/aspnet-core/services/platform/LINGYUN.Platform.HttpApi.Host/appsettings.Development.json b/aspnet-core/services/platform/LINGYUN.Platform.HttpApi.Host/appsettings.Development.json index 43f2079b7..34f82c9e6 100644 --- a/aspnet-core/services/platform/LINGYUN.Platform.HttpApi.Host/appsettings.Development.json +++ b/aspnet-core/services/platform/LINGYUN.Platform.HttpApi.Host/appsettings.Development.json @@ -68,6 +68,11 @@ "Authority": "http://localhost:44385/", "ApiName": "lingyun-abp-application" }, + "AuditLogging": { + "Elasticsearch": { + "IndexPrefix": "lingyun.abp.auditing" + } + }, "Elasticsearch": { "NodeUris": "http://localhost:9200" },