Browse Source

fix(oss): fix MemoryStream outsize for large file uploads

pull/343/head
cKey 4 years ago
parent
commit
41ef26bab9
  1. 4
      aspnet-core/LINGYUN.MicroService.Common.sln
  2. 31
      aspnet-core/configuration/admin/LINGYUN.Abp.BackendAdmin.HttpApi.Host/appsettings.Development.json
  3. 38
      aspnet-core/modules/auditing/LINGYUN.Abp.AuditLogging.Elasticsearch/README.md
  4. 35
      aspnet-core/modules/elasticsearch/LINGYUN.Abp.Elasticsearch/README.md
  5. 14
      aspnet-core/modules/oss-management/LINGYUN.Abp.OssManagement.Application/LINGYUN/Abp/OssManagement/FileUploader.cs
  6. 2
      aspnet-core/modules/oss-management/LINGYUN.Abp.OssManagement.FileSystem/LINGYUN/Abp/OssManagement/FileSystem/FileSystemOssContainer.cs

4
aspnet-core/LINGYUN.MicroService.Common.sln

@ -172,9 +172,9 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "LINGYUN.Abp.AuditLogging.El
EndProject
Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "logging", "logging", "{23F4260D-87C1-4AA6-A302-0A8A76D53BA1}"
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "LINGYUN.Abp.Logging", "modules\logging\LINGYUN.Abp.Logging\LINGYUN.Abp.Logging.csproj", "{CE7E525F-8628-4076-8A2E-B615B944D140}"
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "LINGYUN.Abp.Logging", "modules\logging\LINGYUN.Abp.Logging\LINGYUN.Abp.Logging.csproj", "{CE7E525F-8628-4076-8A2E-B615B944D140}"
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "LINGYUN.Abp.Logging.Serilog.Elasticsearch", "modules\logging\LINGYUN.Abp.Logging.Serilog.Elasticsearch\LINGYUN.Abp.Logging.Serilog.Elasticsearch.csproj", "{CD556F2A-A96B-43A2-8BB3-6C0EBA27EB02}"
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "LINGYUN.Abp.Logging.Serilog.Elasticsearch", "modules\logging\LINGYUN.Abp.Logging.Serilog.Elasticsearch\LINGYUN.Abp.Logging.Serilog.Elasticsearch.csproj", "{CD556F2A-A96B-43A2-8BB3-6C0EBA27EB02}"
EndProject
Global
GlobalSection(SolutionConfigurationPlatforms) = preSolution

31
aspnet-core/configuration/admin/LINGYUN.Abp.BackendAdmin.HttpApi.Host/appsettings.Development.json

@ -12,6 +12,21 @@
"AbpPermissionManagement": "Server=127.0.0.1;Database=Platform;User Id=root;Password=123456",
"AbpLocalizationManagement": "Server=127.0.0.1;Database=Platform;User Id=root;Password=123456"
},
"Logging": {
"Serilog": {
"Elasticsearch": {
"IndexFormat": "linyun.abp.logging-{0:yyyy.MM.dd}"
}
}
},
"AuditLogging": {
"Elasticsearch": {
"IndexPrefix": "lingyun.abp.auditing"
}
},
"Elasticsearch": {
"NodeUris": "http://localhost:9200"
},
"Location": {
"Baidu": {
"AccessKey": "你自己的百度地图WebAPI Key",
@ -62,6 +77,22 @@
},
"Enrich": [ "FromLogContext", "WithProcessId", "WithThreadId" ],
"WriteTo": [
{
"Name": "Console",
"Args": {
"restrictedToMinimumLevel": "Debug",
"outputTemplate": "{Timestamp:yyyy-MM-dd HH:mm:ss} [{Level:u3}] [{SourceContext}] [{ProcessId}] [{ThreadId}] - {Message:lj}{NewLine}{Exception}"
}
},
{
"Name": "Elasticsearch",
"Args": {
"nodeUris": "http://localhost:9200",
"indexFormat": "linyun.abp.logging-{0:yyyy.MM.dd}",
"autoRegisterTemplate": true,
"autoRegisterTemplateVersion": "ESv7"
}
},
{
"Name": "File",
"Args": {

38
aspnet-core/modules/auditing/LINGYUN.Abp.AuditLogging.Elasticsearch/README.md

@ -0,0 +1,38 @@
# LINGYUN.Abp.AuditLogging.Elasticsearch
审计模块 Elasticsearch 实现
ElasticsearchAuditLogManager 实现了 IAuditLogManager, 审计日志由ES管理
ElasticsearchSecurityLogManager 实现了 ISecurityLogManager, 安全日志由ES管理
## 模块引用
```csharp
[DependsOn(typeof(AbpAuditLoggingElasticsearchModule))]
public class YouProjectModule : AbpModule
{
// other
}
```
## 配置项
* AbpAuditLoggingElasticsearchOptions.IndexPrefix 索引前缀, 默认 auditlogging
## 注意事项
与租户模块集成, 跨租户时将会切换索引
## appsettings.json
```json
{
"AuditLogging": {
"Elasticsearch": {
"IndexPrefix": "auditlogging"
}
}
}
```

35
aspnet-core/modules/elasticsearch/LINGYUN.Abp.Elasticsearch/README.md

@ -0,0 +1,35 @@
# LINGYUN.Abp.Elasticsearch
Abp Elasticsearch集成,提供全局唯一IElasticClient访问接口
## 模块引用
```csharp
[DependsOn(typeof(AbpElasticsearchModule))]
public class YouProjectModule : AbpModule
{
// other
}
```
## 配置项
* AbpElasticsearchOptions.FieldCamelCase 字段是否采用 camelCase 格式, 默认false
* AbpElasticsearchOptions.NodeUris ES端点,多个端点以,或;分隔
* AbpElasticsearchOptions.TypeName 文档名称,默认_doc
* AbpElasticsearchOptions.ConnectionLimit 最大连接数,详情见 NEST 文档
* AbpElasticsearchOptions.UserName 连接用户,详情见 NEST 文档
* AbpElasticsearchOptions.Password 用户密码,详情见 NEST 文档
* AbpElasticsearchOptions.ConnectionTimeout 连接超时时间,详情见 NEST 文档
## appsettings.json
```json
{
"Elasticsearch": {
"NodeUris": "http://localhost:9200"
}
}
```

14
aspnet-core/modules/oss-management/LINGYUN.Abp.OssManagement.Application/LINGYUN/Abp/OssManagement/FileUploader.cs

@ -38,7 +38,7 @@ namespace LINGYUN.Abp.OssManagement
if (cancellationToken.IsCancellationRequested)
{
// 如果取消请求,删除临时目录
Directory.Delete(tempFilePath, true);
DirectoryHelper.DeleteIfExists(tempFilePath, true);
return;
}
@ -54,6 +54,9 @@ namespace LINGYUN.Abp.OssManagement
if (input.ChunkNumber == input.TotalChunks)
{
var mergeTmpFile = Path.Combine(tempFilePath, input.FileName);
// 创建临时合并文件流
using var mergeFileStream = new FileStream(mergeTmpFile, FileMode.Create, FileAccess.ReadWrite);
var createOssObjectInput = new CreateOssObjectInput
{
Bucket = input.Bucket,
@ -63,24 +66,23 @@ namespace LINGYUN.Abp.OssManagement
// 合并文件
var mergeSavedFile = Path.Combine(tempFilePath, $"{input.FileName}");
// 获取并排序所有分片文件
var mergeFiles = Directory.GetFiles(tempFilePath).OrderBy(f => f.Length).ThenBy(f => f);
var mergeFiles = Directory.GetFiles(tempFilePath, "*.upd").OrderBy(f => f.Length).ThenBy(f => f);
// 创建临时合并文件
using var memoryStream = new MemoryStream();
foreach (var mergeFile in mergeFiles)
{
// 读取当前文件字节
var mergeFileBytes = await FileHelper.ReadAllBytesAsync(mergeFile);
// 写入到合并文件流
await memoryStream.WriteAsync(mergeFileBytes, 0, mergeFileBytes.Length, cancellationToken);
await mergeFileStream.WriteAsync(mergeFileBytes, 0, mergeFileBytes.Length, cancellationToken);
Array.Clear(mergeFileBytes, 0, mergeFileBytes.Length);
// 删除已参与合并的临时文件分片
FileHelper.DeleteIfExists(mergeFile);
}
createOssObjectInput.SetContent(memoryStream);
createOssObjectInput.SetContent(mergeFileStream);
// 分离出合并接口,合并时计算上传次数
await _fileUploadMerger.MergeAsync(createOssObjectInput);
// 文件保存之后删除临时文件目录
Directory.Delete(tempFilePath, true);
DirectoryHelper.DeleteIfExists(tempFilePath, true);
}
}
catch

2
aspnet-core/modules/oss-management/LINGYUN.Abp.OssManagement.FileSystem/LINGYUN/Abp/OssManagement/FileSystem/FileSystemOssContainer.cs

@ -118,7 +118,7 @@ namespace LINGYUN.Abp.OssManagement.FileSystem
DirectoryHelper.CreateIfNotExists(Path.GetDirectoryName(filePath));
FileMode fileMode = request.Overwrite ? FileMode.Create : FileMode.CreateNew;
using (var fileStream = File.Open(filePath, fileMode, FileAccess.Write))
using (var fileStream = File.Open(filePath, fileMode, FileAccess.ReadWrite))
{
await request.Content.CopyToAsync(fileStream);

Loading…
Cancel
Save