tsai
/
abp-next-admin
mirror of https://github.com/colinin/abp-next-admin.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity
Browse Source

feat: Optimize es query indexes 

- Retrieve the last sorting index by reversing the sorting order.
- Obtain the index mapping using pattern matching
- Add the extension for obtaining ES error messages
pull/1556/head
colin 1 month ago
parent
0bc450796a
commit
b510e4f422
9 changed files with 137 additions and 145 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 44
      aspnet-core/framework/auditing/LINGYUN.Abp.AuditLogging.Elasticsearch/LINGYUN/Abp/AuditLogging/Elasticsearch/AuditLoggingIndexInitializer.cs
  2. 53
      aspnet-core/framework/auditing/LINGYUN.Abp.AuditLogging.Elasticsearch/LINGYUN/Abp/AuditLogging/Elasticsearch/ElasticsearchAuditLogManager.cs
  3. 12
      aspnet-core/framework/auditing/LINGYUN.Abp.AuditLogging.Elasticsearch/LINGYUN/Abp/AuditLogging/Elasticsearch/ElasticsearchAuditLogWriter.cs
  4. 12
      aspnet-core/framework/auditing/LINGYUN.Abp.AuditLogging.Elasticsearch/LINGYUN/Abp/AuditLogging/Elasticsearch/ElasticsearchSecurityLogWriter.cs
  5. 32
      aspnet-core/framework/elasticsearch/LINGYUN.Abp.Elasticsearch/Elastic/Clients/Elasticsearch/ElasticsearchResponseErrorExtensions.cs
  6. 51
      aspnet-core/framework/elasticsearch/LINGYUN.Abp.Elasticsearch/LINGYUN/Abp/Elasticsearch/ElasticsearchIndexMappingProvider.cs
  7. 11
      aspnet-core/framework/elasticsearch/LINGYUN.Abp.Elasticsearch/LINGYUN/Abp/Elasticsearch/ExpressionQueryService.cs
  8. 4
      aspnet-core/framework/elasticsearch/LINGYUN.Abp.Elasticsearch/LINGYUN/Abp/Elasticsearch/IIndexMappingProvider.cs
  9. 63
      aspnet-core/framework/logging/LINGYUN.Abp.Logging.Serilog.Elasticsearch/LINGYUN/Abp/AuditLogging/Serilog/Elasticsearch/SerilogElasticsearchLoggingManager.cs

44
aspnet-core/framework/auditing/LINGYUN.Abp.AuditLogging.Elasticsearch/LINGYUN/Abp/AuditLogging/Elasticsearch/AuditLoggingIndexInitializer.cs
View File

@ -145,31 +145,15 @@ public class AuditLoggingIndexInitializer : IAuditLoggingIndexInitializer, ISing
});
}, cancellationToken);
if (!putTemplateResponse.IsValidResponse)
if (putTemplateResponse.TryGetErrorMessage(out var errorMessage))
{
var errorBuilder = new StringBuilder();
if (putTemplateResponse.TryGetOriginalException(out var ex) && ex != null)
{
errorBuilder.AppendLine(ex.Message);
Logger.LogWarning(ex, "Failed to initialize index and audit log may not be retrieved.");
return;
}
else if (putTemplateResponse.TryGetElasticsearchServerError(out var error) && error != null)
{
errorBuilder.AppendLine(error.ToString());
}
else
{
errorBuilder.AppendLine(putTemplateResponse.DebugInformation);
}
if (_elasticsearchOptions.ThrowIfIndexInitFailed)
{
throw new AbpInitializationException($"Failed to initialize audit log index template, the error: {errorBuilder.ToString()}");
throw new AbpInitializationException($"Failed to initialize audit log index template, the error: {errorMessage}");
}
Logger.LogWarning("Failed to initialize index and audit log may not be retrieved.");
Logger.LogWarning("The error: {error}", errorBuilder.ToString());
Logger.LogWarning("The error: {error}", errorMessage);
}
}
@ -217,31 +201,15 @@ public class AuditLoggingIndexInitializer : IAuditLoggingIndexInitializer, ISing
});
}, cancellationToken);
if (!putTemplateResponse.IsValidResponse)
if (putTemplateResponse.TryGetErrorMessage(out var errorMessage))
{
var errorBuilder = new StringBuilder();
if (putTemplateResponse.TryGetOriginalException(out var ex) && ex != null)
{
errorBuilder.AppendLine(ex.Message);
Logger.LogWarning(ex, "Failed to initialize index and security log may not be retrieved.");
return;
}
else if (putTemplateResponse.TryGetElasticsearchServerError(out var error) && error != null)
{
errorBuilder.AppendLine(error.ToString());
}
else
{
errorBuilder.AppendLine(putTemplateResponse.DebugInformation);
}
if (_elasticsearchOptions.ThrowIfIndexInitFailed)
{
throw new AbpInitializationException($"Failed to initialize security log index template, the error: {errorBuilder.ToString()}");
throw new AbpInitializationException($"Failed to initialize security log index template, the error: {errorMessage}");
}
Logger.LogWarning("Failed to initialize index and security log may not be retrieved.");
Logger.LogWarning("The error: {error}", errorBuilder.ToString());
Logger.LogWarning("The error: {error}", errorMessage);
}
}
}

53
aspnet-core/framework/auditing/LINGYUN.Abp.AuditLogging.Elasticsearch/LINGYUN/Abp/AuditLogging/Elasticsearch/ElasticsearchAuditLogManager.cs
View File

@ -145,6 +145,11 @@ public class ElasticsearchAuditLogManager : IAuditLogManager, ITransientDependen
}),
cancellationToken);
if (response.TryGetErrorMessage(out var errorMessage))
{
Logger.LogWarning("Query audit log count failed: {errorMessage}", errorMessage);
}
return response.Count;
}
@ -407,8 +412,9 @@ public class ElasticsearchAuditLogManager : IAuditLogManager, ITransientDependen
}
}, cancellationToken);
if (!searchResponse.IsSuccess())
if (searchResponse.TryGetErrorMessage(out var errorMessage))
{
Logger.LogWarning("Query audit log failed: {errorMessage}", errorMessage);
return [];
}
@ -456,8 +462,9 @@ public class ElasticsearchAuditLogManager : IAuditLogManager, ITransientDependen
}
}, cancellationToken);
if (!searchResponse.IsSuccess())
if (searchResponse.TryGetErrorMessage(out var errorMessage))
{
Logger.LogWarning("Query audit log failed: {errorMessage}", errorMessage);
return [];
}
@ -514,15 +521,15 @@ public class ElasticsearchAuditLogManager : IAuditLogManager, ITransientDependen
return null;
}
var remaining = skipCount - 10000;
// 获取skipCount最近一条数据作为searchAfter
var secondResponse = await client.SearchAsync<AuditLog>(
dsl => dsl.Indices(indexName)
.Query(query)
.Sort(sorts)
// 反转排序取第一个数据作为起始索引
.Sort(sorts.Select(x => x).Reverse().ToArray())
.SourceIncludes(x => x.Id)
.SearchAfter(firstHit.Sort.ToList())
.Size(remaining),
.Size(1),
cancellationToken);
if (!secondResponse.IsSuccess() || secondResponse.Hits == null || !secondResponse.Hits.Any())
@ -530,11 +537,6 @@ public class ElasticsearchAuditLogManager : IAuditLogManager, ITransientDependen
return null;
}
if (secondResponse.Hits.Count < remaining)
{
return null;
}
var lastHit = secondResponse.Hits.LastOrDefault();
if (lastHit?.Sort == null || !lastHit.Sort.Any())
{
@ -558,29 +560,26 @@ public class ElasticsearchAuditLogManager : IAuditLogManager, ITransientDependen
: nameof(AuditLog.ExecutionTime);
SortOptions[]? sorts = null;
if (sorting.IsNullOrWhiteSpace())
{
var sortingFieldMap = indexMappingInfo.Fields
var sortingFieldMap = indexMappingInfo.Fields
.Where(x => x.Key.Equals(sorting, StringComparison.CurrentCultureIgnoreCase))
.Select(x => x.Value)
.FirstOrDefault();
if (sortingFieldMap != null)
{
sorting = sortingFieldMap.Path;
}
if (!sorting.IsNullOrWhiteSpace())
if (sortingFieldMap != null)
{
sorting = sortingFieldMap.Path;
}
if (!sorting.IsNullOrWhiteSpace())
{
sorts = new SortOptions[1]
{
sorts = new SortOptions[1]
new SortOptions
{
new SortOptions
Field = new FieldSort(new Field(sorting))
{
Field = new FieldSort(new Field(sorting))
{
Order = sortOrder,
},
}
};
}
Order = sortOrder,
},
}
};
}
return sorts;

12
aspnet-core/framework/auditing/LINGYUN.Abp.AuditLogging.Elasticsearch/LINGYUN/Abp/AuditLogging/Elasticsearch/ElasticsearchAuditLogWriter.cs
View File

@ -1,6 +1,5 @@
using Elastic.Clients.Elasticsearch;
using Elastic.Clients.Elasticsearch.Core.Bulk;
using Elastic.Transport.Products.Elasticsearch;
using LINGYUN.Abp.Elasticsearch;
using Microsoft.Extensions.Logging;
using System;
@ -44,17 +43,10 @@ public class ElasticsearchAuditLogWriter : IAuditLogWriter, ITransientDependency
.Id(auditLog.Id),
cancellationToken);
if (!response.IsSuccess())
if (response.TryGetErrorMessage(out var errorMessage))
{
_logger.LogWarning("Could not save the audit log object: " + Environment.NewLine + auditLog.ToString());
if (response.TryGetOriginalException(out var ex) && ex != null)
{
_logger.LogWarning(ex, ex.Message);
}
else if (response.ElasticsearchServerError != null)
{
_logger.LogWarning(response.ElasticsearchServerError.ToString());
}
_logger.LogWarning(errorMessage);
return "";
}

12
aspnet-core/framework/auditing/LINGYUN.Abp.AuditLogging.Elasticsearch/LINGYUN/Abp/AuditLogging/Elasticsearch/ElasticsearchSecurityLogWriter.cs
View File

@ -1,5 +1,6 @@
using Elastic.Clients.Elasticsearch;
using Elastic.Clients.Elasticsearch.Core.Bulk;
using Elastic.Clients.Elasticsearch.IndexManagement;
using LINGYUN.Abp.Elasticsearch;
using Microsoft.Extensions.Logging;
using System;
@ -48,17 +49,10 @@ public class ElasticsearchSecurityLogWriter : ISecurityLogWriter, ITransientDepe
.Id(securityLog.Id),
cancellationToken);
if (!response.IsValidResponse)
if (response.TryGetErrorMessage(out var errorMessage))
{
_logger.LogWarning("Could not save the security log object: " + Environment.NewLine + securityLog.ToString());
if (response.TryGetOriginalException(out var ex) && ex != null)
{
_logger.LogWarning(ex, ex.Message);
}
else if (response.ElasticsearchServerError != null)
{
_logger.LogWarning(response.ElasticsearchServerError.ToString());
}
_logger.LogWarning(errorMessage);
}
}

32
aspnet-core/framework/elasticsearch/LINGYUN.Abp.Elasticsearch/Elastic/Clients/Elasticsearch/ElasticsearchResponseErrorExtensions.cs
View File

@ -0,0 +1,32 @@
using Elastic.Transport.Products.Elasticsearch;
using System.Text;
namespace Elastic.Clients.Elasticsearch;
public static class ElasticsearchResponseErrorExtensions
{
public static bool TryGetErrorMessage(this ElasticsearchResponse response, out string? errorMessage)
{
if (!response.IsSuccess())
{
var errorBuilder = new StringBuilder();
if (response.TryGetOriginalException(out var ex) && ex != null)
{
errorBuilder.AppendLine(ex.Message);
}
if (response.TryGetElasticsearchServerError(out var error) && error != null)
{
errorBuilder.AppendLine(error.ToString());
}
if (errorBuilder.Length == 0)
{
errorBuilder.AppendLine(response.DebugInformation);
}
errorMessage = errorBuilder.ToString();
return true;
}
errorMessage = null;
return false;
}
}

51
aspnet-core/framework/elasticsearch/LINGYUN.Abp.Elasticsearch/LINGYUN/Abp/Elasticsearch/ElasticsearchIndexMappingProvider.cs
View File

@ -1,13 +1,14 @@
using Elastic.Clients.Elasticsearch;
using Elastic.Clients.Elasticsearch.IndexManagement;
using Elastic.Clients.Elasticsearch.Mapping;
using Elastic.Transport.Products.Elasticsearch;
using Microsoft.Extensions.Caching.Memory;
using Microsoft.Extensions.FileSystemGlobbing.Internal;
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading;
using System.Threading.Tasks;
using Volo.Abp;
using Volo.Abp.DependencyInjection;
namespace LINGYUN.Abp.Elasticsearch;
@ -26,40 +27,46 @@ public class ElasticsearchIndexMappingProvider : IIndexMappingProvider, ITransie
_cache = cache;
}
public async Task<IndexMappingInfo> GetMappingAsync(string indexName, CancellationToken cancellationToken = default)
public async Task<IndexMappingInfo> GetMappingAsync(string indexPattern, CancellationToken cancellationToken = default)
{
var cacheKey = $"es_mapping_{indexName}";
var cacheKey = $"es_mapping_{indexPattern}";
var cacheItem = _cache.Get<IndexMappingInfo>(cacheKey);
if (cacheItem == null)
{
var client = _clientFactory.Create();
var response = await client.Indices.GetMappingAsync(indexName, cancellationToken);
var request = new GetMappingRequest(indexPattern)
{
IgnoreUnavailable = true,
AllowNoIndices = true,
ExpandWildcards = [ExpandWildcard.Open]
};
var response = await client.Indices.GetMappingAsync(request, cancellationToken);
if (!response.IsSuccess())
if (response.TryGetErrorMessage(out var errorMessage))
{
var errorBuilder = new StringBuilder();
if (response.TryGetOriginalException(out var ex) && ex != null)
{
errorBuilder.AppendLine(ex.Message);
}
else if (response.TryGetElasticsearchServerError(out var error) && error != null)
throw new AbpException($"Failed to get mapping for index {indexPattern}: {errorMessage}");
}
var indexName = indexPattern.EndsWith("*") ? indexPattern.Substring(0, indexPattern.Length - 1) : indexPattern;
var indexMappings = response.GetMappingFor(indexName);
if (indexMappings == null)
{
foreach (var indexMappingRecord in response.Mappings)
{
errorBuilder.AppendLine(error.ToString());
if (indexMappingRecord.Key.StartsWith(indexName))
{
indexMappings = indexMappingRecord.Value.Mappings;
break;
}
}
else
if (indexMappings == null)
{
errorBuilder.AppendLine(response.DebugInformation);
throw new AbpException($"Index {indexPattern} not found in response");
}
throw new Exception($"Failed to get mapping for index {indexName}: {errorBuilder.ToString()}");
}
if (!response.Mappings.TryGetValue(indexName, out var indexMappingRecord))
{
throw new Exception($"Index {indexName} not found in response");
}
cacheItem = ParseMapping(indexMappingRecord.Mappings, indexName);
cacheItem = ParseMapping(indexMappings, indexPattern);
_cache.Set(cacheKey, cacheItem, _cacheDuration);
}

11
aspnet-core/framework/elasticsearch/LINGYUN.Abp.Elasticsearch/LINGYUN/Abp/Elasticsearch/ExpressionQueryService.cs
View File

@ -241,15 +241,15 @@ public class ExpressionQueryService : IExpressionQueryService, ITransientDepende
return null;
}
var remaining = skipCount - 10000;
// 获取skipCount最近一条数据作为searchAfter
var secondResponse = await client.SearchAsync<TDocument>(
dsl => dsl.Indices(indexName)
.Query(query)
.Sort(sorts)
// 反转排序取第一个数据作为起始索引
.Sort(sorts.Select(x => x).Reverse().ToArray())
.SourceIncludes([])
.SearchAfter(firstHit.Sort.ToList())
.Size(remaining),
.Size(1),
cancellationToken);
if (!secondResponse.IsSuccess() || secondResponse.Hits == null || !secondResponse.Hits.Any())
@ -257,11 +257,6 @@ public class ExpressionQueryService : IExpressionQueryService, ITransientDepende
return null;
}
if (secondResponse.Hits.Count < remaining)
{
return null;
}
var lastHit = secondResponse.Hits.LastOrDefault();
if (lastHit?.Sort == null || !lastHit.Sort.Any())
{

4
aspnet-core/framework/elasticsearch/LINGYUN.Abp.Elasticsearch/LINGYUN/Abp/Elasticsearch/IIndexMappingProvider.cs
View File

@ -5,5 +5,7 @@ namespace LINGYUN.Abp.Elasticsearch;
public interface IIndexMappingProvider
{
Task<IndexMappingInfo> GetMappingAsync(string indexName, CancellationToken cancellationToken = default);
Task<IndexMappingInfo> GetMappingAsync(
string indexPattern,
CancellationToken cancellationToken = default);
}

63
aspnet-core/framework/logging/LINGYUN.Abp.Logging.Serilog.Elasticsearch/LINGYUN/Abp/AuditLogging/Serilog/Elasticsearch/SerilogElasticsearchLoggingManager.cs
View File

@ -6,6 +6,7 @@ using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Logging.Abstractions;
using Microsoft.Extensions.Options;
using Serilog.Events;
using Serilog.Formatting.Elasticsearch;
using System;
using System.Collections.Generic;
using System.Linq;
@ -195,6 +196,10 @@ public class SerilogElasticsearchLoggingManager : ILoggingManager, ISingletonDep
(t) => t.Field(GetField(indexMapping, "fields.UniqueId")).Value(id)))))
.Size(1),
cancellationToken);
if (response.TryGetErrorMessage(out var errorMessage))
{
Logger.LogWarning("Query logs failed: {errorMessage}", errorMessage);
}
}
return _objectMapper.Map<SerilogInfo?, LogInfo?>(response.Documents.FirstOrDefault());
@ -240,6 +245,10 @@ public class SerilogElasticsearchLoggingManager : ILoggingManager, ISingletonDep
dsl.Indices(indexName)
.Query(log => log.Bool(b => b.Must(querys.ToArray()))),
cancellationToken);
if (response.TryGetErrorMessage(out var errorMessage))
{
Logger.LogWarning("Query log count failed: {errorMessage}", errorMessage);
}
return response.Count;
}
@ -469,8 +478,9 @@ public class SerilogElasticsearchLoggingManager : ILoggingManager, ISingletonDep
}
}, cancellationToken);
if (!searchResponse.IsSuccess())
if (searchResponse.TryGetErrorMessage(out var errorMessage))
{
Logger.LogWarning("Query log failed: {errorMessage}", errorMessage);
return [];
}
@ -506,8 +516,9 @@ public class SerilogElasticsearchLoggingManager : ILoggingManager, ISingletonDep
.SearchAfter(searchAfter);
}, cancellationToken);
if (!searchResponse.IsSuccess())
if (searchResponse.TryGetErrorMessage(out var errorMessage))
{
Logger.LogWarning("Query log failed: {errorMessage}", errorMessage);
return [];
}
@ -563,15 +574,15 @@ public class SerilogElasticsearchLoggingManager : ILoggingManager, ISingletonDep
return null;
}
var remaining = skipCount - 10000;
// 获取skipCount最近一条数据作为searchAfter
var secondResponse = await client.SearchAsync<SerilogInfo>(
dsl => dsl.Indices(CreateIndex())
.Query(query)
.Sort(sorts)
// 反转排序取第一个数据作为起始索引
.Sort(sorts.Select(x => x).Reverse().ToArray())
.SourceIncludes(x => x.Level)
.SearchAfter(firstHit.Sort.ToList())
.Size(remaining),
.Size(1),
cancellationToken);
if (!secondResponse.IsSuccess() || secondResponse.Hits == null || !secondResponse.Hits.Any())
@ -579,11 +590,6 @@ public class SerilogElasticsearchLoggingManager : ILoggingManager, ISingletonDep
return null;
}
if (secondResponse.Hits.Count < remaining)
{
return null;
}
var lastHit = secondResponse.Hits.LastOrDefault();
if (lastHit?.Sort == null || !lastHit.Sort.Any())
{
@ -621,32 +627,29 @@ public class SerilogElasticsearchLoggingManager : ILoggingManager, ISingletonDep
? SortOrder.Asc : SortOrder.Desc;
sorting = !sorting.IsNullOrWhiteSpace()
? sorting.Split()[0]
: nameof(SerilogInfo.TimeStamp);
: ElasticsearchJsonFormatter.TimestampPropertyName;
SortOptions[]? sorts = null;
if (sorting.IsNullOrWhiteSpace())
var sortingFieldMap = indexMappingInfo.Fields
.Where(x => x.Key.Equals(sorting, StringComparison.CurrentCultureIgnoreCase))
.Select(x => x.Value)
.FirstOrDefault();
if (sortingFieldMap != null)
{
var sortingFieldMap = indexMappingInfo.Fields
.Where(x => x.Key.Equals(sorting, StringComparison.CurrentCultureIgnoreCase))
.Select(x => x.Value)
.FirstOrDefault();
if (sortingFieldMap != null)
{
sorting = sortingFieldMap.Path;
}
if (!sorting.IsNullOrWhiteSpace())
sorting = sortingFieldMap.Path;
}
if (!sorting.IsNullOrWhiteSpace())
{
sorts = new SortOptions[1]
{
sorts = new SortOptions[1]
new SortOptions
{
new SortOptions
Field = new FieldSort(new Field(sorting))
{
Field = new FieldSort(new Field(sorting))
{
Order = sortOrder,
},
}
};
}
Order = sortOrder,
},
}
};
}
return sorts;

Write Preview
Loading…
Cancel
Save