From 11c293b01061dd51d42ee40a5c7fbfafef15fed4 Mon Sep 17 00:00:00 2001 From: Sebastian Date: Tue, 5 May 2020 16:59:52 +0200 Subject: [PATCH] Support uploading streams without length for S3 --- .../Assets/AmazonS3AssetStore.cs | 43 ++++++++++++++--- .../Config/IdentityServerPathMiddleware.cs | 1 - .../Controllers/Rules/Models/RuleEventDto.cs | 12 ++--- .../Controllers/Rules/Models/RuleEventsDto.cs | 12 +++-- .../Api/Controllers/Rules/RulesController.cs | 2 +- .../Authentication/IdentityServerServices.cs | 47 ++++--------------- .../src/Squidex/Config/MyIdentityOptions.cs | 2 - .../src/Squidex/Config/Web/WebExtensions.cs | 2 +- .../Assets/AssetStoreTests.cs | 36 ++++++++++++++ 9 files changed, 97 insertions(+), 60 deletions(-) diff --git a/backend/src/Squidex.Infrastructure.Amazon/Assets/AmazonS3AssetStore.cs b/backend/src/Squidex.Infrastructure.Amazon/Assets/AmazonS3AssetStore.cs index 2a829b752..feb585075 100644 --- a/backend/src/Squidex.Infrastructure.Amazon/Assets/AmazonS3AssetStore.cs +++ b/backend/src/Squidex.Infrastructure.Amazon/Assets/AmazonS3AssetStore.cs @@ -179,9 +179,35 @@ namespace Squidex.Infrastructure.Assets Key = key }; - SetStream(stream, request); + if (!HasContentLength(stream)) + { + var tempFileName = Path.GetTempFileName(); + + var tempStream = new FileStream(tempFileName, + FileMode.Create, + FileAccess.ReadWrite, + FileShare.Delete, 1024 * 16, + FileOptions.Asynchronous | + FileOptions.DeleteOnClose | + FileOptions.SequentialScan); + + using (tempStream) + { + await stream.CopyToAsync(tempStream); + + request.InputStream = tempStream; + + await transferUtility.UploadAsync(request, ct); + } + } + else + { + request.InputStream = new SeekFakerStream(stream); - await transferUtility.UploadAsync(request, ct); + request.AutoCloseStream = false; + + await transferUtility.UploadAsync(request, ct); + } } catch (AmazonS3Exception ex) when (ex.StatusCode == HttpStatusCode.PreconditionFailed) { @@ -237,11 +263,16 @@ namespace Squidex.Infrastructure.Assets throw new AssetAlreadyExistsException(fileName); } - private static void SetStream(Stream stream, TransferUtilityUploadRequest request) + private static bool HasContentLength(Stream stream) { - // Amazon S3 requires a seekable stream, but does not seek anything. - request.InputStream = new SeekFakerStream(stream); - request.AutoCloseStream = false; + try + { + return stream.Length > 0; + } + catch + { + return false; + } } } } \ No newline at end of file diff --git a/backend/src/Squidex/Areas/Api/Config/IdentityServerPathMiddleware.cs b/backend/src/Squidex/Areas/Api/Config/IdentityServerPathMiddleware.cs index 2ded8318b..9e63d229d 100644 --- a/backend/src/Squidex/Areas/Api/Config/IdentityServerPathMiddleware.cs +++ b/backend/src/Squidex/Areas/Api/Config/IdentityServerPathMiddleware.cs @@ -8,7 +8,6 @@ using System.Threading.Tasks; using IdentityServer4.Extensions; using Microsoft.AspNetCore.Http; -using Microsoft.Extensions.Options; using Squidex.Web; namespace Squidex.Areas.Api.Config diff --git a/backend/src/Squidex/Areas/Api/Controllers/Rules/Models/RuleEventDto.cs b/backend/src/Squidex/Areas/Api/Controllers/Rules/Models/RuleEventDto.cs index 731fddec7..3866c88b7 100644 --- a/backend/src/Squidex/Areas/Api/Controllers/Rules/Models/RuleEventDto.cs +++ b/backend/src/Squidex/Areas/Api/Controllers/Rules/Models/RuleEventDto.cs @@ -64,25 +64,25 @@ namespace Squidex.Areas.Api.Controllers.Rules.Models /// public RuleJobResult JobResult { get; set; } - public static RuleEventDto FromRuleEvent(IRuleEventEntity ruleEvent, ApiController controller, string app) + public static RuleEventDto FromRuleEvent(IRuleEventEntity ruleEvent, Resources resources) { var result = new RuleEventDto(); SimpleMapper.Map(ruleEvent, result); SimpleMapper.Map(ruleEvent.Job, result); - return result.CreateLinks(controller, app); + return result.CreateLinks(resources); } - private RuleEventDto CreateLinks(ApiController controller, string app) + private RuleEventDto CreateLinks(Resources resources) { - var values = new { app, id = Id }; + var values = new { app = resources.App, id = Id }; - AddPutLink("update", controller.Url(x => nameof(x.PutEvent), values)); + AddPutLink("update", resources.Url(x => nameof(x.PutEvent), values)); if (NextAttempt.HasValue) { - AddDeleteLink("delete", controller.Url(x => nameof(x.DeleteEvent), values)); + AddDeleteLink("delete", resources.Url(x => nameof(x.DeleteEvent), values)); } return this; diff --git a/backend/src/Squidex/Areas/Api/Controllers/Rules/Models/RuleEventsDto.cs b/backend/src/Squidex/Areas/Api/Controllers/Rules/Models/RuleEventsDto.cs index 4f961e858..d5a77fa1d 100644 --- a/backend/src/Squidex/Areas/Api/Controllers/Rules/Models/RuleEventsDto.cs +++ b/backend/src/Squidex/Areas/Api/Controllers/Rules/Models/RuleEventsDto.cs @@ -26,20 +26,22 @@ namespace Squidex.Areas.Api.Controllers.Rules.Models /// public long Total { get; set; } - public static RuleEventsDto FromRuleEvents(IResultList ruleEvents, ApiController controller, string app) + public static RuleEventsDto FromRuleEvents(IResultList ruleEvents, Resources resources) { var result = new RuleEventsDto { Total = ruleEvents.Total, - Items = ruleEvents.Select(x => RuleEventDto.FromRuleEvent(x, controller, app)).ToArray() + Items = ruleEvents.Select(x => RuleEventDto.FromRuleEvent(x, resources)).ToArray() }; - return result.CreateLinks(controller, app); + return result.CreateLinks(resources); } - private RuleEventsDto CreateLinks(ApiController controller, string app) + private RuleEventsDto CreateLinks(Resources resources) { - AddSelfLink(controller.Url(x => nameof(x.GetEvents), new { app })); + var values = new { app = resources.App }; + + AddSelfLink(resources.Url(x => nameof(x.GetEvents), values)); return this; } diff --git a/backend/src/Squidex/Areas/Api/Controllers/Rules/RulesController.cs b/backend/src/Squidex/Areas/Api/Controllers/Rules/RulesController.cs index bcc01d537..d34c674d0 100644 --- a/backend/src/Squidex/Areas/Api/Controllers/Rules/RulesController.cs +++ b/backend/src/Squidex/Areas/Api/Controllers/Rules/RulesController.cs @@ -297,7 +297,7 @@ namespace Squidex.Areas.Api.Controllers.Rules { var ruleEvents = await ruleEventsRepository.QueryByAppAsync(AppId, ruleId, skip, take); - var response = RuleEventsDto.FromRuleEvents(ruleEvents, this, app); + var response = RuleEventsDto.FromRuleEvents(ruleEvents, Resources); return Ok(response); } diff --git a/backend/src/Squidex/Config/Authentication/IdentityServerServices.cs b/backend/src/Squidex/Config/Authentication/IdentityServerServices.cs index 8702948ae..5640ccc63 100644 --- a/backend/src/Squidex/Config/Authentication/IdentityServerServices.cs +++ b/backend/src/Squidex/Config/Authentication/IdentityServerServices.cs @@ -5,14 +5,10 @@ // All rights reserved. Licensed under the MIT license. // ========================================================================== -using IdentityModel.AspNetCore.OAuth2Introspection; -using IdentityServer4.AccessTokenValidation; using Microsoft.AspNetCore.Authentication; using Microsoft.AspNetCore.Authentication.Cookies; -using Microsoft.AspNetCore.Builder; using Microsoft.Extensions.Configuration; using Microsoft.Extensions.DependencyInjection; -using Squidex.Infrastructure; using Squidex.Web; namespace Squidex.Config.Authentication @@ -21,41 +17,9 @@ namespace Squidex.Config.Authentication { public static AuthenticationBuilder AddSquidexIdentityServerAuthentication(this AuthenticationBuilder authBuilder, MyIdentityOptions identityOptions, IConfiguration config) { - var apiScope = Constants.ApiScope; - - var urlsOptions = config.GetSection("urls").Get(); - - if (!string.IsNullOrWhiteSpace(urlsOptions.BaseUrl)) + if (!string.IsNullOrWhiteSpace(identityOptions.AuthorityUrl)) { - string apiAuthorityUrl; - - if (!string.IsNullOrWhiteSpace(identityOptions.AuthorityUrl)) - { - apiAuthorityUrl = identityOptions.AuthorityUrl.BuildFullUrl(Constants.IdentityServerPrefix); - } - else - { - apiAuthorityUrl = urlsOptions.BuildUrl(Constants.IdentityServerPrefix); - } - - if (identityOptions.LocalApi) - { - authBuilder.AddLocalApi(Constants.ApiSecurityScheme, options => - { - options.ExpectedScope = apiScope; - }); - } - else - { - authBuilder.AddIdentityServerAuthentication(Constants.ApiSecurityScheme, options => - { - options.Authority = apiAuthorityUrl; - options.ApiName = apiScope; - options.ApiSecret = null; - options.RequireHttpsMetadata = identityOptions.RequiresHttps; - options.SupportedTokens = SupportedTokens.Jwt; - }); - } + var apiAuthorityUrl = identityOptions.AuthorityUrl; authBuilder.AddOpenIdConnect(options => { @@ -71,6 +35,13 @@ namespace Squidex.Config.Authentication options.SignInScheme = CookieAuthenticationDefaults.AuthenticationScheme; }); } + else + { + authBuilder.AddLocalApi(Constants.ApiSecurityScheme, options => + { + options.ExpectedScope = Constants.ApiScope; + }); + } return authBuilder; } diff --git a/backend/src/Squidex/Config/MyIdentityOptions.cs b/backend/src/Squidex/Config/MyIdentityOptions.cs index 29e90d9b9..a89555413 100644 --- a/backend/src/Squidex/Config/MyIdentityOptions.cs +++ b/backend/src/Squidex/Config/MyIdentityOptions.cs @@ -55,8 +55,6 @@ namespace Squidex.Config public bool AllowPasswordAuth { get; set; } - public bool LocalApi { get; set; } = true; - public bool LockAutomatically { get; set; } public bool NoConsent { get; set; } diff --git a/backend/src/Squidex/Config/Web/WebExtensions.cs b/backend/src/Squidex/Config/Web/WebExtensions.cs index 8e860d1f2..1eef8611e 100644 --- a/backend/src/Squidex/Config/Web/WebExtensions.cs +++ b/backend/src/Squidex/Config/Web/WebExtensions.cs @@ -36,8 +36,8 @@ namespace Squidex.Config.Web public static IApplicationBuilder UseSquidexTracking(this IApplicationBuilder app) { app.UseMiddleware(); - app.UseMiddleware(); app.UseMiddleware(); + app.UseMiddleware(); return app; } diff --git a/backend/tests/Squidex.Infrastructure.Tests/Assets/AssetStoreTests.cs b/backend/tests/Squidex.Infrastructure.Tests/Assets/AssetStoreTests.cs index ece7ff1e9..60de1ecb3 100644 --- a/backend/tests/Squidex.Infrastructure.Tests/Assets/AssetStoreTests.cs +++ b/backend/tests/Squidex.Infrastructure.Tests/Assets/AssetStoreTests.cs @@ -7,6 +7,7 @@ using System; using System.IO; +using System.IO.Compression; using System.Threading.Tasks; using Xunit; @@ -121,6 +122,20 @@ namespace Squidex.Infrastructure.Assets Assert.Equal(assetLarge.ToArray(), readData.ToArray()); } + [Fact] + public async Task Should_upload_compressed_file() + { + var source = CreateDeflateStream(20_000); + + await Sut.UploadAsync(fileName, source); + + var readData = new MemoryStream(); + + await Sut.DownloadAsync(fileName, readData); + + Assert.True(readData.Length > 0); + } + [Fact] public async Task Should_write_and_read_file_with_range() { @@ -215,5 +230,26 @@ namespace Squidex.Infrastructure.Assets return memoryStream; } + + private static Stream CreateDeflateStream(int length) + { + var memoryStream = new MemoryStream(); + + using (var archive1 = new ZipArchive(memoryStream, ZipArchiveMode.Create, true)) + { + using (var file = archive1.CreateEntry("test").Open()) + { + var test = CreateFile(length); + + test.CopyTo(file); + } + } + + memoryStream.Position = 0; + + var archive2 = new ZipArchive(memoryStream, ZipArchiveMode.Read); + + return archive2.GetEntry("test").Open(); + } } }