Merge pull request #1347 from colinin/fix-long-object

fix(blobstoring): fix the issue of downloading large objects

aspnet-core/framework/cloud-tencent/LINGYUN.Abp.BlobStoring.Tencent/LINGYUN.Abp.BlobStoring.Tencent.csproj

@@ -15,20 +15,21 @@
 	</PropertyGroup>
 
 	<ItemGroup>
-	  <None Remove="LINGYUN\Abp\BlobStoring\Tencent\Localization\*.json" />
+		<None Remove="LINGYUN\Abp\BlobStoring\Tencent\Localization\*.json" />
 	</ItemGroup>
 
 	<ItemGroup>
-	  <EmbeddedResource Include="LINGYUN\Abp\BlobStoring\Tencent\Localization\*.json" />
+		<EmbeddedResource Include="LINGYUN\Abp\BlobStoring\Tencent\Localization\*.json" />
 	</ItemGroup>
 
 	<ItemGroup>
 		<PackageReference Include="Volo.Abp.BlobStoring" />
 		<PackageReference Include="Tencent.QCloud.Cos.Sdk" />
+		<PackageReference Include="Microsoft.Extensions.Http" />
 	</ItemGroup>
 
 	<ItemGroup>
-	  <ProjectReference Include="..\LINGYUN.Abp.Tencent\LINGYUN.Abp.Tencent.csproj" />
+		<ProjectReference Include="..\LINGYUN.Abp.Tencent\LINGYUN.Abp.Tencent.csproj" />
 	</ItemGroup>
 
 </Project>

aspnet-core/framework/cloud-tencent/LINGYUN.Abp.BlobStoring.Tencent/LINGYUN/Abp/BlobStoring/Tencent/AbpBlobStoringTencentCloudModule.cs

@@ -1,5 +1,6 @@
 using LINGYUN.Abp.Tencent;
 using LINGYUN.Abp.Tencent.Localization;
+using Microsoft.Extensions.DependencyInjection;
 using Volo.Abp.BlobStoring;
 using Volo.Abp.Localization;
 using Volo.Abp.Modularity;
@@ -25,5 +26,7 @@ public class AbpBlobStoringTencentCloudModule : AbpModule
                 .Get<TencentCloudResource>()
                 .AddVirtualJson("/LINGYUN/Abp/BlobStoring/Tencent/Localization");
         });
+
+        context.Services.AddTenantOssClient();
     }
 }

aspnet-core/framework/cloud-tencent/LINGYUN.Abp.BlobStoring.Tencent/LINGYUN/Abp/BlobStoring/Tencent/BlobStoringTencentConsts.cs

@@ -0,0 +1,5 @@
+namespace LINGYUN.Abp.BlobStoring.Tencent;
+internal static class BlobStoringTencentConsts
+{
+    public const string HttpClient = "BlobStoring.Tencent";
+}

aspnet-core/framework/cloud-tencent/LINGYUN.Abp.BlobStoring.Tencent/LINGYUN/Abp/BlobStoring/Tencent/TencentCloudBlobProvider.cs

@@ -6,6 +6,7 @@ using LINGYUN.Abp.Tencent.Features;
 using System;
 using System.Collections.Generic;
 using System.IO;
+using System.Net.Http;
 using System.Threading.Tasks;
 using Volo.Abp;
 using Volo.Abp.BlobStoring;
@@ -19,15 +20,18 @@ public class TencentCloudBlobProvider : BlobProviderBase, ITransientDependency
 {
     protected IFeatureChecker FeatureChecker { get; }
     protected ICosClientFactory CosClientFactory { get; }
+    protected IHttpClientFactory HttpClientFactory { get; }
     protected ITencentBlobNameCalculator TencentBlobNameCalculator { get; }
 
     public TencentCloudBlobProvider(
         IFeatureChecker featureChecker,
         ICosClientFactory cosClientFactory,
+        IHttpClientFactory httpClientFactory,
         ITencentBlobNameCalculator tencentBlobNameCalculator)
     {
         FeatureChecker = featureChecker;
         CosClientFactory = cosClientFactory;
+        HttpClientFactory = httpClientFactory;
         TencentBlobNameCalculator = tencentBlobNameCalculator;
     }
 
@@ -65,12 +69,13 @@ public class TencentCloudBlobProvider : BlobProviderBase, ITransientDependency
 
         // TODO: 未经验证
 
-        var request = new GetObjectBytesRequest(GetBucketName(args), blobName);
+        var ossDownloadUrl = ossClient.GetObjectUrl(GetBucketName(args), blobName); 
-        var ossObject = ossClient.GetObject(request);
+        
-        var memoryStream = new MemoryStream();
+        var client = HttpClientFactory.CreateTenantOssClient();
-        await memoryStream.WriteAsync(ossObject.content, 0, ossObject.content.Length);
+
-        memoryStream.Seek(0, SeekOrigin.Begin);
+        var ossContent = await client.GetStreamAsync(ossDownloadUrl);
-        return memoryStream;
+
+        return ossContent;
     }
 
     public override async Task SaveAsync(BlobProviderSaveArgs args)

aspnet-core/framework/cloud-tencent/LINGYUN.Abp.BlobStoring.Tencent/Microsoft/Extensions/DependencyInjection/BlobStoringTencentHttpClientFactoryServiceCollectionExtensions.cs

@@ -0,0 +1,12 @@
+using LINGYUN.Abp.BlobStoring.Tencent;
+
+namespace Microsoft.Extensions.DependencyInjection;
+internal static class BlobStoringTencentHttpClientFactoryServiceCollectionExtensions
+{
+    public static IServiceCollection AddTenantOssClient(this IServiceCollection services)
+    {
+        services.AddHttpClient(BlobStoringTencentConsts.HttpClient);
+
+        return services;
+    }
+}

aspnet-core/framework/cloud-tencent/LINGYUN.Abp.BlobStoring.Tencent/System/Net/Http/BlobStoringTencentHttpClientFactoryExtenssions.cs

@@ -0,0 +1,11 @@
+using LINGYUN.Abp.BlobStoring.Tencent;
+
+namespace System.Net.Http;
+public static class BlobStoringTencentHttpClientFactoryExtenssions
+{
+    public static HttpClient CreateTenantOssClient(
+        this IHttpClientFactory httpClientFactory)
+    {
+        return httpClientFactory.CreateClient(BlobStoringTencentConsts.HttpClient); ;
+    }
+}

aspnet-core/framework/common/LINGYUN.Abp.BlobStoring.Aliyun/LINGYUN/Abp/BlobStoring/Aliyun/AliyunBlobProvider.cs

@@ -56,9 +56,7 @@ public class AliyunBlobProvider : BlobProviderBase, ITransientDependency
         }
 
         var ossObject = ossClient.GetObject(GetBucketName(args), blobName);
-        var memoryStream = new MemoryStream();
+        return ossObject.Content;
-        await ossObject.Content.CopyToAsync(memoryStream);
-        return memoryStream;
     }
 
     public override async Task SaveAsync(BlobProviderSaveArgs args)

aspnet-core/modules/oss-management/LINGYUN.Abp.OssManagement.Domain/LINGYUN/Abp/OssManagement/OssObject.cs

@@ -44,7 +44,7 @@ public class OssObject
     public void SetContent(Stream stream)
     {
         _content = stream;
-        if (!_content.IsNullOrEmpty())
+        if (!_content.IsNullOrEmpty() && _content.CanSeek && _content.Position != 0)
         {
             _content.Seek(0, SeekOrigin.Begin);
         }

aspnet-core/modules/oss-management/LINGYUN.Abp.OssManagement.Minio/LINGYUN.Abp.OssManagement.Minio.csproj

@@ -15,6 +15,7 @@
 
 	<ItemGroup>
 		<PackageReference Include="Volo.Abp.BlobStoring.Minio" />
+		<PackageReference Include="Microsoft.Extensions.Http" />
 	</ItemGroup>
 
 	<ItemGroup>

aspnet-core/modules/oss-management/LINGYUN.Abp.OssManagement.Minio/LINGYUN/Abp/OssManagement/Minio/AbpOssManagementMinioModule.cs

@@ -1,4 +1,5 @@
-using Volo.Abp.BlobStoring.Minio;
+using Microsoft.Extensions.DependencyInjection;
+using Volo.Abp.BlobStoring.Minio;
 using Volo.Abp.Modularity;
 
 namespace LINGYUN.Abp.OssManagement.Minio;
@@ -8,4 +9,8 @@ namespace LINGYUN.Abp.OssManagement.Minio;
     typeof(AbpOssManagementDomainModule))]
 public class AbpOssManagementMinioModule : AbpModule
 {
+    public override void ConfigureServices(ServiceConfigurationContext context)
+    {
+        context.Services.AddMinioHttpClient();
+    }
 }

aspnet-core/modules/oss-management/LINGYUN.Abp.OssManagement.Minio/LINGYUN/Abp/OssManagement/Minio/MinioOssContainer.cs

@@ -11,6 +11,7 @@ using System.Collections.Generic;
 using System.IO;
 using System.Linq;
 using System.Linq.Dynamic.Core;
+using System.Net.Http;
 using System.Reactive.Linq;
 using System.Threading.Tasks;
 using Volo.Abp;
@@ -29,6 +30,7 @@ public class MinioOssContainer : OssContainerBase, IOssObjectExpireor
     protected IMinioBlobNameCalculator MinioBlobNameCalculator { get; }
     protected IBlobNormalizeNamingService BlobNormalizeNamingService { get; }
     protected IBlobContainerConfigurationProvider ConfigurationProvider { get; }
+    protected IHttpClientFactory HttpClientFactory { get; }
 
     protected IClock Clock { get; }
     protected ICurrentTenant CurrentTenant { get; }
@@ -42,12 +44,14 @@ public class MinioOssContainer : OssContainerBase, IOssObjectExpireor
         IBlobNormalizeNamingService blobNormalizeNamingService, 
         IBlobContainerConfigurationProvider configurationProvider,
         IServiceScopeFactory serviceScopeFactory,
+        IHttpClientFactory httpClientFactory,
         IOptions<AbpOssManagementOptions> options)
         : base(options, serviceScopeFactory)
     {
         Clock = clock;
         Logger = logger;
         CurrentTenant = currentTenant;
+        HttpClientFactory = httpClientFactory;
         MinioBlobNameCalculator = minioBlobNameCalculator;
         BlobNormalizeNamingService = blobNormalizeNamingService;
         ConfigurationProvider = configurationProvider;
@@ -440,23 +444,10 @@ public class MinioOssContainer : OssContainerBase, IOssObjectExpireor
             throw new BusinessException(code: OssManagementErrorCodes.ObjectNotFound);
         }
 
-        var memoryStream = new MemoryStream();
+        var getObjectResult = await client.StatObjectAsync(
-        var getObjectArgs = new GetObjectArgs()
+            new StatObjectArgs()
                 .WithBucket(bucket)
-                .WithObject(objectName)
+                .WithObject(objectName));
-                .WithCallbackStream((stream) =>
-                {
-                    if (stream != null)
-                    {
-                        stream.CopyTo(memoryStream);
-                        memoryStream.Seek(0, SeekOrigin.Begin);
-                    }
-                    else
-                    {
-                        memoryStream = null;
-                    }
-                });
-        var getObjectResult = await client.GetObjectAsync(getObjectArgs);
 
         var ossObject = new OssObject(
             !objectPath.IsNullOrWhiteSpace()
@@ -465,7 +456,7 @@ public class MinioOssContainer : OssContainerBase, IOssObjectExpireor
             request.Path,
             getObjectResult.ETag,
             getObjectResult.LastModified,
-            memoryStream.Length,
+            getObjectResult.Size,
             getObjectResult.LastModified,
             getObjectResult.MetaData,
             getObjectResult.ObjectName.EndsWith("/"))
@@ -473,9 +464,16 @@ public class MinioOssContainer : OssContainerBase, IOssObjectExpireor
             FullName = getObjectResult.ObjectName.Replace(prefixPath, "")
         };
 
-        if (memoryStream.Length > 0)
+        if (getObjectResult.Size > 0)
         {
-            ossObject.SetContent(memoryStream);
+            var objectUrl = await client.PresignedGetObjectAsync(
+                new PresignedGetObjectArgs()
+                    .WithBucket(bucket)
+                    .WithObject(objectName)
+                    .WithExpiry(3600));
+            var httpClient = HttpClientFactory.CreateMinioHttpClient();
+
+            ossObject.SetContent(await httpClient.GetStreamAsync(objectUrl));
         }
 
         return ossObject;

aspnet-core/modules/oss-management/LINGYUN.Abp.OssManagement.Minio/Microsoft/Extensions/DependencyInjection/MinioHttpClientFactoryServiceCollectionExtensions.cs

@@ -0,0 +1,18 @@
+using System.Net.Http;
+
+namespace Microsoft.Extensions.DependencyInjection;
+internal static class MinioHttpClientFactoryServiceCollectionExtensions
+{
+    private const string HttpClientName = "__AbpMinioHttpClient";
+    public static IServiceCollection AddMinioHttpClient(this IServiceCollection services)
+    {
+        services.AddHttpClient(HttpClientName);
+
+        return services;
+    }
+
+    public static HttpClient CreateMinioHttpClient(this IHttpClientFactory httpClientFactory)
+    {
+        return httpClientFactory.CreateClient(HttpClientName);
+    }
+}

aspnet-core/modules/oss-management/LINGYUN.Abp.OssManagement.Nexus/LINGYUN/Abp/OssManagement/Nexus/NexusOssContainer.cs

@@ -301,7 +301,7 @@ internal class NexusOssContainer : OssContainerBase, IOssObjectExpireor
             }
         }
 
-        return new OssObject(
+        var ossObject = new OssObject(
             blobName,
             blobPath,
             checksum?.GetOrDefault("md5")?.ToString(),
@@ -309,7 +309,13 @@ internal class NexusOssContainer : OssContainerBase, IOssObjectExpireor
             coreUIResponse.Result.Data.Size,
             coreUIResponse.Result.Data.BlobUpdated,
             metadata
-            );
+        );
+
+        var ossContent = await NexusAssetManager.GetContentOrNullAsync(nexusAssetItem);
+
+        ossObject.SetContent(ossContent);
+
+        return ossObject;
     }
 
     protected virtual NexusBlobProviderConfiguration GetNexusConfiguration()

aspnet-core/modules/oss-management/LINGYUN.Abp.OssManagement.Tencent/LINGYUN/Abp/OssManagement/Tencent/TencentOssContainer.cs

@@ -9,6 +9,7 @@ using System;
 using System.Collections.Generic;
 using System.IO;
 using System.Linq;
+using System.Net.Http;
 using System.Threading.Tasks;
 using Volo.Abp;
 using Volo.Abp.MultiTenancy;
@@ -24,10 +25,12 @@ internal class TencentOssContainer : OssContainerBase, IOssObjectExpireor
     protected IClock Clock { get; }
     protected ICurrentTenant CurrentTenant { get; }
     protected ICosClientFactory CosClientFactory { get; }
+    protected IHttpClientFactory HttpClientFactory { get; }
     public TencentOssContainer(
         IClock clock,
         ICurrentTenant currentTenant,
         ICosClientFactory cosClientFactory,
+        IHttpClientFactory httpClientFactory,
         IServiceScopeFactory serviceScopeFactory,
         IOptions<AbpOssManagementOptions> options)
         : base(options, serviceScopeFactory)
@@ -35,6 +38,7 @@ internal class TencentOssContainer : OssContainerBase, IOssObjectExpireor
         Clock = clock;
         CurrentTenant = currentTenant;
         CosClientFactory = cosClientFactory;
+        HttpClientFactory = httpClientFactory;
     }
     public async override Task BulkDeleteObjectsAsync(BulkDeleteObjectRequest request)
     {
@@ -386,6 +390,11 @@ internal class TencentOssContainer : OssContainerBase, IOssObjectExpireor
             getObjectRequest.SetQueryParameter(request.Process, null);
         }
         var objectResult = ossClient.GetObject(getObjectRequest);
+        var objectDownloadUrl = ossClient.GetObjectUrl(request.Bucket, objectName);
+
+        var client = HttpClientFactory.CreateTenantOssClient();
+        var objectContent = await client.GetStreamAsync(objectDownloadUrl);
+
         var ossObject = new OssObject(
             !objectPath.IsNullOrWhiteSpace()
                 ? objectResult.Key.Replace(objectPath, "")
@@ -393,7 +402,7 @@ internal class TencentOssContainer : OssContainerBase, IOssObjectExpireor
             request.Path,
             objectResult.eTag,
             null,
-            objectResult.content.Length,
+            objectContent.Length,
             null,
             new Dictionary<string, string>(),
             objectResult.Key.EndsWith("/"))
@@ -401,12 +410,9 @@ internal class TencentOssContainer : OssContainerBase, IOssObjectExpireor
             FullName = objectResult.Key
         };
 
-        if (objectResult.content.Length > 0)
+        if (objectContent.Length > 0)
         {
-            var memoryStream = new MemoryStream();
+            ossObject.SetContent(objectContent);
-            await memoryStream.WriteAsync(objectResult.content, 0, objectResult.content.Length);
-            memoryStream.Seek(0, SeekOrigin.Begin);
-            ossObject.SetContent(memoryStream);
         }
 
         return ossObject;

aspnet-core/modules/oss-management/LINGYUN.Abp.OssManagement.Tencent/LINGYUN/Abp/OssManagement/Tencent/TencentOssContainerFactory.cs

@@ -1,6 +1,7 @@
 using LINGYUN.Abp.BlobStoring.Tencent;
 using Microsoft.Extensions.DependencyInjection;
 using Microsoft.Extensions.Options;
+using System.Net.Http;
 using Volo.Abp.MultiTenancy;
 using Volo.Abp.Timing;
 
@@ -11,7 +12,7 @@ public class TencentOssContainerFactory : IOssContainerFactory
     protected IClock Clock { get; }
     protected ICurrentTenant CurrentTenant { get; }
     protected ICosClientFactory CosClientFactory { get; }
-
+    protected IHttpClientFactory HttpClientFactory { get; }
     protected IServiceScopeFactory ServiceScopeFactory { get; }
     protected IOptions<AbpOssManagementOptions> Options { get; }
 
@@ -19,12 +20,14 @@ public class TencentOssContainerFactory : IOssContainerFactory
         IClock clock,
         ICurrentTenant currentTenant,
         ICosClientFactory cosClientFactory,
+        IHttpClientFactory httpClientFactory,
         IServiceScopeFactory serviceScopeFactory,
         IOptions<AbpOssManagementOptions> options)
     {
         Clock = clock;
         CurrentTenant = currentTenant;
         CosClientFactory = cosClientFactory;
+        HttpClientFactory = httpClientFactory;
         Options = options;
         ServiceScopeFactory = serviceScopeFactory;
     }
@@ -35,6 +38,7 @@ public class TencentOssContainerFactory : IOssContainerFactory
             Clock,
             CurrentTenant,
             CosClientFactory,
+            HttpClientFactory,
             ServiceScopeFactory,
             Options);
     }