tsai
/
abp-next-admin
mirror of https://github.com/colinin/abp-next-admin.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity
Browse Source

fix(oss): fix MemoryStream outsize for large file uploads

pull/343/head
cKey 4 years ago
parent
04a300e965
commit
41ef26bab9
6 changed files with 227 additions and 121 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Unified View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 4
      aspnet-core/LINGYUN.MicroService.Common.sln
  2. 255
      aspnet-core/configuration/admin/LINGYUN.Abp.BackendAdmin.HttpApi.Host/appsettings.Development.json
  3. 38
      aspnet-core/modules/auditing/LINGYUN.Abp.AuditLogging.Elasticsearch/README.md
  4. 35
      aspnet-core/modules/elasticsearch/LINGYUN.Abp.Elasticsearch/README.md
  5. 14
      aspnet-core/modules/oss-management/LINGYUN.Abp.OssManagement.Application/LINGYUN/Abp/OssManagement/FileUploader.cs
  6. 2
      aspnet-core/modules/oss-management/LINGYUN.Abp.OssManagement.FileSystem/LINGYUN/Abp/OssManagement/FileSystem/FileSystemOssContainer.cs

4
aspnet-core/LINGYUN.MicroService.Common.sln
View File

@ -172,9 +172,9 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "LINGYUN.Abp.AuditLogging.El
EndProject EndProject
Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "logging", "logging", "{23F4260D-87C1-4AA6-A302-0A8A76D53BA1}" Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "logging", "logging", "{23F4260D-87C1-4AA6-A302-0A8A76D53BA1}"
EndProject EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "LINGYUN.Abp.Logging", "modules\logging\LINGYUN.Abp.Logging\LINGYUN.Abp.Logging.csproj", "{CE7E525F-8628-4076-8A2E-B615B944D140}" Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "LINGYUN.Abp.Logging", "modules\logging\LINGYUN.Abp.Logging\LINGYUN.Abp.Logging.csproj", "{CE7E525F-8628-4076-8A2E-B615B944D140}"
EndProject EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "LINGYUN.Abp.Logging.Serilog.Elasticsearch", "modules\logging\LINGYUN.Abp.Logging.Serilog.Elasticsearch\LINGYUN.Abp.Logging.Serilog.Elasticsearch.csproj", "{CD556F2A-A96B-43A2-8BB3-6C0EBA27EB02}" Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "LINGYUN.Abp.Logging.Serilog.Elasticsearch", "modules\logging\LINGYUN.Abp.Logging.Serilog.Elasticsearch\LINGYUN.Abp.Logging.Serilog.Elasticsearch.csproj", "{CD556F2A-A96B-43A2-8BB3-6C0EBA27EB02}"
EndProject EndProject
Global Global
GlobalSection(SolutionConfigurationPlatforms) = preSolution GlobalSection(SolutionConfigurationPlatforms) = preSolution

255
aspnet-core/configuration/admin/LINGYUN.Abp.BackendAdmin.HttpApi.Host/appsettings.Development.json
View File

@ -1,112 +1,143 @@
{ {
"App": { "App": {
"TrackingEntitiesChanged": true, "TrackingEntitiesChanged": true,
"SelfUrl": "http://localhost:44385/" "SelfUrl": "http://localhost:44385/"
}, },
"ConnectionStrings": { "ConnectionStrings": {
"Default": "Server=127.0.0.1;Database=Platform;User Id=root;Password=123456", "Default": "Server=127.0.0.1;Database=Platform;User Id=root;Password=123456",
"AbpIdentity": "Server=127.0.0.1;Database=IdentityServer;User Id=root;Password=123456", "AbpIdentity": "Server=127.0.0.1;Database=IdentityServer;User Id=root;Password=123456",
"AbpIdentityServer": "Server=127.0.0.1;Database=IdentityServer;User Id=root;Password=123456", "AbpIdentityServer": "Server=127.0.0.1;Database=IdentityServer;User Id=root;Password=123456",
"AbpTenantManagement": "Server=127.0.0.1;Database=Platform;User Id=root;Password=123456", "AbpTenantManagement": "Server=127.0.0.1;Database=Platform;User Id=root;Password=123456",
"AbpSettingManagement": "Server=127.0.0.1;Database=Platform;User Id=root;Password=123456", "AbpSettingManagement": "Server=127.0.0.1;Database=Platform;User Id=root;Password=123456",
"AbpPermissionManagement": "Server=127.0.0.1;Database=Platform;User Id=root;Password=123456", "AbpPermissionManagement": "Server=127.0.0.1;Database=Platform;User Id=root;Password=123456",
"AbpLocalizationManagement": "Server=127.0.0.1;Database=Platform;User Id=root;Password=123456" "AbpLocalizationManagement": "Server=127.0.0.1;Database=Platform;User Id=root;Password=123456"
}, },
"Location": { "Logging": {
"Baidu": { "Serilog": {
"AccessKey": "你自己的百度地图WebAPI Key", "Elasticsearch": {
"ExtensionsRoad": true, "IndexFormat": "linyun.abp.logging-{0:yyyy.MM.dd}"
"ExtensionsTown": true, }
"ExtensionsPoi": "1", }
"VisableErrorToClient": true },
}, "AuditLogging": {
"Tencent": { "Elasticsearch": {
"AccessKey": "你自己的腾讯地图WebAPI Key", "IndexPrefix": "lingyun.abp.auditing"
"SecretKey": "你自己的腾讯地图WebAPI SecretKey Key, 不填的话不计算签名", }
"VisableErrorToClient": true },
} "Elasticsearch": {
}, "NodeUris": "http://localhost:9200"
"CAP": { },
"EventBus": { "Location": {
"DefaultGroupName": "BackendAdmin", "Baidu": {
"Version": "v1", "AccessKey": "你自己的百度地图WebAPI Key",
"FailedRetryInterval": 300, "ExtensionsRoad": true,
"FailedRetryCount": 10 "ExtensionsTown": true,
}, "ExtensionsPoi": "1",
"RabbitMQ": { "VisableErrorToClient": true
"HostName": "Your RabbitMQ server connection address", },
"Port": 5672, "Tencent": {
"UserName": "Your RabbitMQ server connection user", "AccessKey": "你自己的腾讯地图WebAPI Key",
"Password": "Your RabbitMQ server connection user password", "SecretKey": "你自己的腾讯地图WebAPI SecretKey Key, 不填的话不计算签名",
"ExchangeName": "The name of your RabbitMQ server switch", "VisableErrorToClient": true
"VirtualHost": "Name of your RabbitMQ server VirtualHost" }
} },
}, "CAP": {
"Redis": { "EventBus": {
"Configuration": "127.0.0.1", "DefaultGroupName": "BackendAdmin",
"InstanceName": "LINGYUN.AbpApplication", "Version": "v1",
"DefaultDatabase": 10 "FailedRetryInterval": 300,
}, "FailedRetryCount": 10
"AuthServer": { },
"Authority": "http://localhost:44385/", "RabbitMQ": {
"ApiName": "lingyun-abp-application" "HostName": "Your RabbitMQ server connection address",
}, "Port": 5672,
"Serilog": { "UserName": "Your RabbitMQ server connection user",
"MinimumLevel": { "Password": "Your RabbitMQ server connection user password",
"Default": "Debug", "ExchangeName": "The name of your RabbitMQ server switch",
"Override": { "VirtualHost": "Name of your RabbitMQ server VirtualHost"
"Microsoft.EntityFrameworkCore": "Debug", }
"System": "Warning", },
"Microsoft": "Warning" "Redis": {
} "Configuration": "127.0.0.1",
}, "InstanceName": "LINGYUN.AbpApplication",
"Enrich": [ "FromLogContext", "WithProcessId", "WithThreadId" ], "DefaultDatabase": 10
"WriteTo": [ },
{ "AuthServer": {
"Name": "File", "Authority": "http://localhost:44385/",
"Args": { "ApiName": "lingyun-abp-application"
"path": "Logs/Debug-.log", },
"restrictedToMinimumLevel": "Debug", "Serilog": {
"rollingInterval": "Day", "MinimumLevel": {
"outputTemplate": "{Timestamp:yyyy-MM-dd HH:mm:ss} [{Level:u3}] [{SourceContext}] [{ProcessId}] [{ThreadId}] - {Message:lj}{NewLine}{Exception}" "Default": "Debug",
} "Override": {
}, "Microsoft.EntityFrameworkCore": "Debug",
{ "System": "Warning",
"Name": "File", "Microsoft": "Warning"
"Args": { }
"path": "Logs/Info-.log", },
"restrictedToMinimumLevel": "Information", "Enrich": [ "FromLogContext", "WithProcessId", "WithThreadId" ],
"rollingInterval": "Day", "WriteTo": [
"outputTemplate": "{Timestamp:yyyy-MM-dd HH:mm:ss} [{Level:u3}] [{SourceContext}] [{ProcessId}] [{ThreadId}] - {Message:lj}{NewLine}{Exception}" {
} "Name": "Console",
}, "Args": {
{ "restrictedToMinimumLevel": "Debug",
"Name": "File", "outputTemplate": "{Timestamp:yyyy-MM-dd HH:mm:ss} [{Level:u3}] [{SourceContext}] [{ProcessId}] [{ThreadId}] - {Message:lj}{NewLine}{Exception}"
"Args": { }
"path": "Logs/Warn-.log", },
"restrictedToMinimumLevel": "Warning", {
"rollingInterval": "Day", "Name": "Elasticsearch",
"outputTemplate": "{Timestamp:yyyy-MM-dd HH:mm:ss} [{Level:u3}] [{SourceContext}] [{ProcessId}] [{ThreadId}] - {Message:lj}{NewLine}{Exception}" "Args": {
} "nodeUris": "http://localhost:9200",
}, "indexFormat": "linyun.abp.logging-{0:yyyy.MM.dd}",
{ "autoRegisterTemplate": true,
"Name": "File", "autoRegisterTemplateVersion": "ESv7"
"Args": { }
"path": "Logs/Error-.log", },
"restrictedToMinimumLevel": "Error", {
"rollingInterval": "Day", "Name": "File",
"outputTemplate": "{Timestamp:yyyy-MM-dd HH:mm:ss} [{Level:u3}] [{SourceContext}] [{ProcessId}] [{ThreadId}] - {Message:lj}{NewLine}{Exception}" "Args": {
} "path": "Logs/Debug-.log",
}, "restrictedToMinimumLevel": "Debug",
{ "rollingInterval": "Day",
"Name": "File", "outputTemplate": "{Timestamp:yyyy-MM-dd HH:mm:ss} [{Level:u3}] [{SourceContext}] [{ProcessId}] [{ThreadId}] - {Message:lj}{NewLine}{Exception}"
"Args": { }
"path": "Logs/Fatal-.log", },
"restrictedToMinimumLevel": "Fatal", {
"rollingInterval": "Day", "Name": "File",
"outputTemplate": "{Timestamp:yyyy-MM-dd HH:mm:ss} [{Level:u3}] [{SourceContext}] [{ProcessId}] [{ThreadId}] - {Message:lj}{NewLine}{Exception}" "Args": {
} "path": "Logs/Info-.log",
} "restrictedToMinimumLevel": "Information",
] "rollingInterval": "Day",
} "outputTemplate": "{Timestamp:yyyy-MM-dd HH:mm:ss} [{Level:u3}] [{SourceContext}] [{ProcessId}] [{ThreadId}] - {Message:lj}{NewLine}{Exception}"
} }
},
{
"Name": "File",
"Args": {
"path": "Logs/Warn-.log",
"restrictedToMinimumLevel": "Warning",
"rollingInterval": "Day",
"outputTemplate": "{Timestamp:yyyy-MM-dd HH:mm:ss} [{Level:u3}] [{SourceContext}] [{ProcessId}] [{ThreadId}] - {Message:lj}{NewLine}{Exception}"
}
},
{
"Name": "File",
"Args": {
"path": "Logs/Error-.log",
"restrictedToMinimumLevel": "Error",
"rollingInterval": "Day",
"outputTemplate": "{Timestamp:yyyy-MM-dd HH:mm:ss} [{Level:u3}] [{SourceContext}] [{ProcessId}] [{ThreadId}] - {Message:lj}{NewLine}{Exception}"
}
},
{
"Name": "File",
"Args": {
"path": "Logs/Fatal-.log",
"restrictedToMinimumLevel": "Fatal",
"rollingInterval": "Day",
"outputTemplate": "{Timestamp:yyyy-MM-dd HH:mm:ss} [{Level:u3}] [{SourceContext}] [{ProcessId}] [{ThreadId}] - {Message:lj}{NewLine}{Exception}"
}
}
]
}
}

38
aspnet-core/modules/auditing/LINGYUN.Abp.AuditLogging.Elasticsearch/README.md
View File

@ -0,0 +1,38 @@
# LINGYUN.Abp.AuditLogging.Elasticsearch
审计模块 Elasticsearch 实现
ElasticsearchAuditLogManager 实现了 IAuditLogManager, 审计日志由ES管理
ElasticsearchSecurityLogManager 实现了 ISecurityLogManager, 安全日志由ES管理
## 模块引用
```csharp
[DependsOn(typeof(AbpAuditLoggingElasticsearchModule))]
public class YouProjectModule : AbpModule
{
// other
}
```
## 配置项
* AbpAuditLoggingElasticsearchOptions.IndexPrefix 索引前缀, 默认 auditlogging
## 注意事项
与租户模块集成, 跨租户时将会切换索引
## appsettings.json
```json
{
"AuditLogging": {
"Elasticsearch": {
"IndexPrefix": "auditlogging"
}
}
}
```

35
aspnet-core/modules/elasticsearch/LINGYUN.Abp.Elasticsearch/README.md
View File

@ -0,0 +1,35 @@
# LINGYUN.Abp.Elasticsearch
Abp Elasticsearch集成,提供全局唯一IElasticClient访问接口
## 模块引用
```csharp
[DependsOn(typeof(AbpElasticsearchModule))]
public class YouProjectModule : AbpModule
{
// other
}
```
## 配置项
* AbpElasticsearchOptions.FieldCamelCase 字段是否采用 camelCase 格式, 默认false
* AbpElasticsearchOptions.NodeUris ES端点,多个端点以,或;分隔
* AbpElasticsearchOptions.TypeName 文档名称,默认_doc
* AbpElasticsearchOptions.ConnectionLimit 最大连接数,详情见 NEST 文档
* AbpElasticsearchOptions.UserName 连接用户,详情见 NEST 文档
* AbpElasticsearchOptions.Password 用户密码,详情见 NEST 文档
* AbpElasticsearchOptions.ConnectionTimeout 连接超时时间,详情见 NEST 文档
## appsettings.json
```json
{
"Elasticsearch": {
"NodeUris": "http://localhost:9200"
}
}
```

14
aspnet-core/modules/oss-management/LINGYUN.Abp.OssManagement.Application/LINGYUN/Abp/OssManagement/FileUploader.cs
View File

@ -38,7 +38,7 @@ namespace LINGYUN.Abp.OssManagement
if (cancellationToken.IsCancellationRequested) if (cancellationToken.IsCancellationRequested)
{ {
// 如果取消请求,删除临时目录 // 如果取消请求,删除临时目录
Directory.Delete(tempFilePath, true); DirectoryHelper.DeleteIfExists(tempFilePath, true);
return; return;
} }
@ -54,6 +54,9 @@ namespace LINGYUN.Abp.OssManagement
if (input.ChunkNumber == input.TotalChunks) if (input.ChunkNumber == input.TotalChunks)
{ {
var mergeTmpFile = Path.Combine(tempFilePath, input.FileName);
// 创建临时合并文件流
using var mergeFileStream = new FileStream(mergeTmpFile, FileMode.Create, FileAccess.ReadWrite);
var createOssObjectInput = new CreateOssObjectInput var createOssObjectInput = new CreateOssObjectInput
{ {
Bucket = input.Bucket, Bucket = input.Bucket,
@ -63,24 +66,23 @@ namespace LINGYUN.Abp.OssManagement
// 合并文件 // 合并文件
var mergeSavedFile = Path.Combine(tempFilePath, $"{input.FileName}"); var mergeSavedFile = Path.Combine(tempFilePath, $"{input.FileName}");
// 获取并排序所有分片文件 // 获取并排序所有分片文件
var mergeFiles = Directory.GetFiles(tempFilePath).OrderBy(f => f.Length).ThenBy(f => f); var mergeFiles = Directory.GetFiles(tempFilePath, "*.upd").OrderBy(f => f.Length).ThenBy(f => f);
// 创建临时合并文件 // 创建临时合并文件
using var memoryStream = new MemoryStream();
foreach (var mergeFile in mergeFiles) foreach (var mergeFile in mergeFiles)
{ {
// 读取当前文件字节 // 读取当前文件字节
var mergeFileBytes = await FileHelper.ReadAllBytesAsync(mergeFile); var mergeFileBytes = await FileHelper.ReadAllBytesAsync(mergeFile);
// 写入到合并文件流 // 写入到合并文件流
await memoryStream.WriteAsync(mergeFileBytes, 0, mergeFileBytes.Length, cancellationToken); await mergeFileStream.WriteAsync(mergeFileBytes, 0, mergeFileBytes.Length, cancellationToken);
Array.Clear(mergeFileBytes, 0, mergeFileBytes.Length); Array.Clear(mergeFileBytes, 0, mergeFileBytes.Length);
// 删除已参与合并的临时文件分片 // 删除已参与合并的临时文件分片
FileHelper.DeleteIfExists(mergeFile); FileHelper.DeleteIfExists(mergeFile);
} }
createOssObjectInput.SetContent(memoryStream); createOssObjectInput.SetContent(mergeFileStream);
// 分离出合并接口,合并时计算上传次数 // 分离出合并接口,合并时计算上传次数
await _fileUploadMerger.MergeAsync(createOssObjectInput); await _fileUploadMerger.MergeAsync(createOssObjectInput);
// 文件保存之后删除临时文件目录 // 文件保存之后删除临时文件目录
Directory.Delete(tempFilePath, true); DirectoryHelper.DeleteIfExists(tempFilePath, true);
} }
} }
catch catch

2
aspnet-core/modules/oss-management/LINGYUN.Abp.OssManagement.FileSystem/LINGYUN/Abp/OssManagement/FileSystem/FileSystemOssContainer.cs
View File

@ -118,7 +118,7 @@ namespace LINGYUN.Abp.OssManagement.FileSystem
DirectoryHelper.CreateIfNotExists(Path.GetDirectoryName(filePath)); DirectoryHelper.CreateIfNotExists(Path.GetDirectoryName(filePath));
FileMode fileMode = request.Overwrite ? FileMode.Create : FileMode.CreateNew; FileMode fileMode = request.Overwrite ? FileMode.Create : FileMode.CreateNew;
using (var fileStream = File.Open(filePath, fileMode, FileAccess.Write)) using (var fileStream = File.Open(filePath, fileMode, FileAccess.ReadWrite))
{ {
await request.Content.CopyToAsync(fileStream); await request.Content.CopyToAsync(fileStream);

Write Preview
Loading…
Cancel
Save