Browse Source

Bugfixes (#1194)

* FFMPEG support

* Fix redirects.

* Install ffmpeg.

* Rebuild options fixed.

* Update tests

* Catch exception

* Disable test
pull/1197/head
Sebastian Stehle 1 year ago
committed by GitHub
parent
commit
88882b2190
No known key found for this signature in database GPG Key ID: B5690EEEBB952194
  1. 6
      Dockerfile
  2. 4
      backend/src/Migrations/Migrations/RebuildApps.cs
  3. 4
      backend/src/Migrations/Migrations/RebuildAssetFolders.cs
  4. 4
      backend/src/Migrations/Migrations/RebuildAssets.cs
  5. 4
      backend/src/Migrations/Migrations/RebuildContents.cs
  6. 4
      backend/src/Migrations/Migrations/RebuildRules.cs
  7. 4
      backend/src/Migrations/Migrations/RebuildSchemas.cs
  8. 17
      backend/src/Migrations/Migrations/RebuildSnapshots.cs
  9. 2
      backend/src/Migrations/RebuildOptions.cs
  10. 21
      backend/src/Migrations/RebuildRunner.cs
  11. 10
      backend/src/Migrations/RebuilderExtensions.cs
  12. 9
      backend/src/Squidex.Data.EntityFramework/Providers/MySql/Migrations/.editorconfig
  13. 9
      backend/src/Squidex.Data.EntityFramework/Providers/Postgres/Migrations/.editorconfig
  14. 9
      backend/src/Squidex.Data.EntityFramework/Providers/SqlServer/Migrations/.editorconfig
  15. 2
      backend/src/Squidex.Data.MongoDb/Domain/Apps/Entities/Contents/MongoContentRepository_SnapshotStore.cs
  16. 77
      backend/src/Squidex.Domain.Apps.Entities/Assets/FFMpegAssetMetadataSource.cs
  17. 121
      backend/src/Squidex.Domain.Apps.Entities/Assets/FileTagAssetMetadataSource.cs
  18. 1
      backend/src/Squidex.Domain.Apps.Entities/Squidex.Domain.Apps.Entities.csproj
  19. 2
      backend/src/Squidex.Web/Constants.cs
  20. 14
      backend/src/Squidex.Web/ContextExtensions.cs
  21. 1
      backend/src/Squidex.Web/Pipeline/AppResolver.cs
  22. 15
      backend/src/Squidex/Areas/IdentityServer/Config/IdentityServerServices.cs
  23. 20
      backend/src/Squidex/Areas/IdentityServer/Controllers/Account/AccountController.cs
  24. 13
      backend/src/Squidex/Areas/IdentityServer/Controllers/Connect/AuthorizationController.cs
  25. 5
      backend/src/Squidex/Config/Authentication/OidcHandler.cs
  26. 7
      backend/src/Squidex/Config/Authentication/OidcServices.cs
  27. 3
      backend/src/Squidex/Config/Domain/AssetServices.cs
  28. 9
      backend/src/Squidex/appsettings.json
  29. 2
      backend/tests/Squidex.Data.Tests.CodeGenerator/Squidex.Data.Tests.CodeGenerator.csproj
  30. 4
      backend/tests/Squidex.Data.Tests/MongoDb/Domain/Contents/Text/AtlasTextIndexTests.cs
  31. 4
      backend/tests/Squidex.Data.Tests/MongoDb/Infrastructure/EventSourcing/MongoEventConsumerProcessorIntegrationTests_Direct.cs
  32. 22
      backend/tests/Squidex.Data.Tests/MongoDb/Infrastructure/EventSourcing/MongoEventStoreParallelInsertTests.cs
  33. 2
      backend/tests/Squidex.Data.Tests/Squidex.Data.Tests.csproj
  34. 98
      backend/tests/Squidex.Domain.Apps.Entities.Tests/Assets/FFMpegAssetMetadataSourceTests.cs
  35. BIN
      backend/tests/Squidex.Domain.Apps.Entities.Tests/Assets/TestFiles/SampleVideo_Broken.mp4
  36. 4
      backend/tests/Squidex.Domain.Apps.Entities.Tests/Contents/Text/AzureTextIndexTests.cs
  37. 4
      backend/tests/Squidex.Domain.Apps.Entities.Tests/Contents/Text/ElasticSearchTextIndexTests.cs
  38. 4
      backend/tests/Squidex.Domain.Apps.Entities.Tests/Contents/Text/OpenSearchTextIndexTests.cs
  39. 3
      backend/tests/Squidex.Domain.Apps.Entities.Tests/Squidex.Domain.Apps.Entities.Tests.csproj
  40. 11
      tools/TestSuite/TestSuite.ApiTests/AssetFormatTests.cs
  41. 8
      tools/TestSuite/TestSuite.ApiTests/Verify/AssetFormatTests.Should_upload_video_3gp.verified.txt
  42. 9
      tools/TestSuite/TestSuite.ApiTests/Verify/AssetFormatTests.Should_upload_video_flv.verified.txt
  43. 9
      tools/TestSuite/TestSuite.ApiTests/Verify/AssetFormatTests.Should_upload_video_mkv.verified.txt

6
Dockerfile

@ -4,6 +4,10 @@
FROM --platform=$BUILDPLATFORM mcr.microsoft.com/dotnet/sdk:8.0 AS backend
# FFMPEG for tests
RUN apt-get update \
&& apt-get install -y ffmpeg
ARG SQUIDEX__BUILD__VERSION=7.0.0
WORKDIR /src
@ -73,7 +77,7 @@ ARG SQUIDEX__RUNTIME__VERSION=7.0.0
# Curl for debugging and libc-dev for protobuf
RUN apt-get update \
&& apt-get install -y curl libc-dev
&& apt-get install -y curl libc-dev ffmpeg
# Default tool directory
WORKDIR /tools

4
backend/src/Migrations/Migrations/RebuildApps.cs

@ -16,11 +16,9 @@ public sealed class RebuildApps(
IOptions<RebuildOptions> rebuildOptions)
: IMigration
{
private readonly RebuildOptions rebuildOptions = rebuildOptions.Value;
public Task UpdateAsync(
CancellationToken ct)
{
return rebuilder.RebuildAppsAsync(rebuildOptions.BatchSize, ct);
return rebuilder.RebuildAppsAsync(rebuildOptions.Value.CalculateBatchSize(), ct);
}
}

4
backend/src/Migrations/Migrations/RebuildAssetFolders.cs

@ -16,11 +16,9 @@ public sealed class RebuildAssetFolders(
IOptions<RebuildOptions> rebuildOptions)
: IMigration
{
private readonly RebuildOptions rebuildOptions = rebuildOptions.Value;
public Task UpdateAsync(
CancellationToken ct)
{
return rebuilder.RebuildAssetFoldersAsync(rebuildOptions.BatchSize, ct);
return rebuilder.RebuildAssetFoldersAsync(rebuildOptions.Value.CalculateBatchSize(), ct);
}
}

4
backend/src/Migrations/Migrations/RebuildAssets.cs

@ -16,11 +16,9 @@ public sealed class RebuildAssets(
IOptions<RebuildOptions> rebuildOptions)
: IMigration
{
private readonly RebuildOptions rebuildOptions = rebuildOptions.Value;
public Task UpdateAsync(
CancellationToken ct)
{
return rebuilder.RebuildAssetsAsync(rebuildOptions.BatchSize, ct);
return rebuilder.RebuildAssetsAsync(rebuildOptions.Value.CalculateBatchSize(), ct);
}
}

4
backend/src/Migrations/Migrations/RebuildContents.cs

@ -16,11 +16,9 @@ public sealed class RebuildContents(
IOptions<RebuildOptions> rebuildOptions)
: IMigration
{
private readonly RebuildOptions rebuildOptions = rebuildOptions.Value;
public Task UpdateAsync(
CancellationToken ct)
{
return rebuilder.RebuildContentAsync(rebuildOptions.BatchSize, ct);
return rebuilder.RebuildContentAsync(rebuildOptions.Value.CalculateBatchSize(), ct);
}
}

4
backend/src/Migrations/Migrations/RebuildRules.cs

@ -16,11 +16,9 @@ public sealed class RebuildRules(
IOptions<RebuildOptions> rebuildOptions)
: IMigration
{
private readonly RebuildOptions rebuildOptions = rebuildOptions.Value;
public Task UpdateAsync(
CancellationToken ct)
{
return rebuilder.RebuildRulesAsync(rebuildOptions.BatchSize, ct);
return rebuilder.RebuildRulesAsync(rebuildOptions.Value.CalculateBatchSize(), ct);
}
}

4
backend/src/Migrations/Migrations/RebuildSchemas.cs

@ -16,11 +16,9 @@ public sealed class RebuildSchemas(
IOptions<RebuildOptions> rebuildOptions)
: IMigration
{
private readonly RebuildOptions rebuildOptions = rebuildOptions.Value;
public Task UpdateAsync(
CancellationToken ct)
{
return rebuilder.RebuildSchemasAsync(rebuildOptions.BatchSize, ct);
return rebuilder.RebuildSchemasAsync(rebuildOptions.Value.CalculateBatchSize(), ct);
}
}

17
backend/src/Migrations/Migrations/RebuildSnapshots.cs

@ -16,16 +16,17 @@ public sealed class RebuildSnapshots(
IOptions<RebuildOptions> rebuildOptions)
: IMigration
{
private readonly RebuildOptions rebuildOptions = rebuildOptions.Value;
public async Task UpdateAsync(
CancellationToken ct)
{
await rebuilder.RebuildAppsAsync(rebuildOptions.BatchSize, ct);
await rebuilder.RebuildSchemasAsync(rebuildOptions.BatchSize, ct);
await rebuilder.RebuildRulesAsync(rebuildOptions.BatchSize, ct);
await rebuilder.RebuildContentAsync(rebuildOptions.BatchSize, ct);
await rebuilder.RebuildAssetsAsync(rebuildOptions.BatchSize, ct);
await rebuilder.RebuildAssetFoldersAsync(rebuildOptions.BatchSize, ct);
var batchSize = rebuildOptions.Value.CalculateBatchSize();
await rebuilder.RebuildAppsAsync(batchSize, ct);
await rebuilder.RebuildSchemasAsync(batchSize, ct);
await rebuilder.RebuildRulesAsync(batchSize, ct);
await rebuilder.RebuildContentAsync(batchSize, ct);
await rebuilder.RebuildAssetsAsync(batchSize, ct);
await rebuilder.RebuildAssetFoldersAsync(batchSize, ct);
await rebuilder.RebuildTeamsAsync(batchSize, ct);
}
}

2
backend/src/Migrations/RebuildOptions.cs

@ -23,6 +23,8 @@ public sealed class RebuildOptions
public bool Schemas { get; set; }
public bool Teams { get; set; }
public int BatchSize { get; set; } = 100;
public int CalculateBatchSize()

21
backend/src/Migrations/RebuildRunner.cs

@ -16,40 +16,43 @@ public sealed class RebuildRunner(
Rebuilder rebuilder,
RebuildFiles rebuildFiles)
{
private readonly RebuildOptions rebuildOptions = rebuildOptions.Value;
public async Task RunAsync(
CancellationToken ct)
{
var batchSize = rebuildOptions.CalculateBatchSize();
var batchSize = rebuildOptions.Value.CalculateBatchSize();
if (rebuildOptions.Apps)
if (rebuildOptions.Value.Apps)
{
await rebuilder.RebuildAppsAsync(batchSize, ct);
}
if (rebuildOptions.Schemas)
if (rebuildOptions.Value.Teams)
{
await rebuilder.RebuildTeamsAsync(batchSize, ct);
}
if (rebuildOptions.Value.Schemas)
{
await rebuilder.RebuildSchemasAsync(batchSize, ct);
}
if (rebuildOptions.Rules)
if (rebuildOptions.Value.Rules)
{
await rebuilder.RebuildRulesAsync(batchSize, ct);
}
if (rebuildOptions.Assets)
if (rebuildOptions.Value.Assets)
{
await rebuilder.RebuildAssetsAsync(batchSize, ct);
await rebuilder.RebuildAssetFoldersAsync(batchSize, ct);
}
if (rebuildOptions.AssetFiles)
if (rebuildOptions.Value.AssetFiles)
{
await rebuildFiles.RepairAsync(ct);
}
if (rebuildOptions.Contents)
if (rebuildOptions.Value.Contents)
{
await rebuilder.RebuildContentAsync(batchSize, ct);
}

10
backend/src/Migrations/RebuilderExtensions.cs

@ -10,11 +10,13 @@ using Squidex.Domain.Apps.Core.Assets;
using Squidex.Domain.Apps.Core.Contents;
using Squidex.Domain.Apps.Core.Rules;
using Squidex.Domain.Apps.Core.Schemas;
using Squidex.Domain.Apps.Core.Teams;
using Squidex.Domain.Apps.Entities.Apps.DomainObject;
using Squidex.Domain.Apps.Entities.Assets.DomainObject;
using Squidex.Domain.Apps.Entities.Contents.DomainObject;
using Squidex.Domain.Apps.Entities.Rules.DomainObject;
using Squidex.Domain.Apps.Entities.Schemas.DomainObject;
using Squidex.Domain.Apps.Entities.Teams.DomainObject;
using Squidex.Events;
using Squidex.Infrastructure.Commands;
@ -71,4 +73,12 @@ public static class RebuilderExtensions
return rebuilder.RebuildAsync<ContentDomainObject, WriteContent>(streamFilter, batchSize, AllowedErrorRate, ct);
}
public static Task RebuildTeamsAsync(this Rebuilder rebuilder, int batchSize,
CancellationToken ct = default)
{
var streamFilter = StreamFilter.Prefix("team-");
return rebuilder.RebuildAsync<TeamDomainObject, Team>(streamFilter, batchSize, AllowedErrorRate, ct);
}
}

9
backend/src/Squidex.Data.EntityFramework/Providers/MySql/Migrations/.editorconfig

@ -0,0 +1,9 @@
[*.cs]
# MA0007: Add a comma after the last value
dotnet_diagnostic.MA0007.severity = none
# MA0048: File name must match type name
dotnet_diagnostic.MA0048.severity = none
# SA1633: File must have header
dotnet_diagnostic.SA1633.severity = none

9
backend/src/Squidex.Data.EntityFramework/Providers/Postgres/Migrations/.editorconfig

@ -0,0 +1,9 @@
[*.cs]
# MA0007: Add a comma after the last value
dotnet_diagnostic.MA0007.severity = none
# MA0048: File name must match type name
dotnet_diagnostic.MA0048.severity = none
# SA1633: File must have header
dotnet_diagnostic.SA1633.severity = none

9
backend/src/Squidex.Data.EntityFramework/Providers/SqlServer/Migrations/.editorconfig

@ -0,0 +1,9 @@
[*.cs]
# MA0007: Add a comma after the last value
dotnet_diagnostic.MA0007.severity = none
# MA0048: File name must match type name
dotnet_diagnostic.MA0048.severity = none
# SA1633: File must have header
dotnet_diagnostic.SA1633.severity = none

2
backend/src/Squidex.Data.MongoDb/Domain/Apps/Entities/Contents/MongoContentRepository_SnapshotStore.cs

@ -108,8 +108,6 @@ public partial class MongoContentRepository : ISnapshotStore<WriteContent>, IDel
async Task ISnapshotStore<WriteContent>.WriteManyAsync(IEnumerable<SnapshotWriteJob<WriteContent>> jobs,
CancellationToken ct)
{
var validJobs = jobs.Where(x => IsValid(x.Value)).ToList();
using (Telemetry.Activities.StartActivity("MongoContentRepository/WriteManyAsync"))
{
var collectionUpdates = new Dictionary<IMongoCollection<MongoContentEntity>, List<MongoContentEntity>>();

77
backend/src/Squidex.Domain.Apps.Entities/Assets/FFMpegAssetMetadataSource.cs

@ -0,0 +1,77 @@
// ==========================================================================
// Squidex Headless CMS
// ==========================================================================
// Copyright (c) Squidex UG (haftungsbeschraenkt)
// All rights reserved. Licensed under the MIT license.
// ==========================================================================
using FFMpegCore;
using Squidex.Domain.Apps.Core.Assets;
using Squidex.Domain.Apps.Entities.Assets.Commands;
namespace Squidex.Domain.Apps.Entities.Assets;
public sealed class FFMpegAssetMetadataSource : IAssetMetadataSource
{
public async Task EnhanceAsync(UploadAssetCommand command,
CancellationToken ct)
{
if (command.Type != AssetType.Unknown)
{
return;
}
try
{
var analysis = await FFProbe.AnalyseAsync(command.File.OpenRead(), cancellationToken: ct);
void TryAddInt(string name, long? value)
{
if (value > 0)
{
command.Metadata[name] = value.Value;
}
}
void TryAddTimeSpan(string name, TimeSpan value)
{
if (value != TimeSpan.Zero)
{
command.Metadata[name] = value.ToString();
}
}
var audioStream = analysis.AudioStreams.FirstOrDefault();
if (audioStream != null)
{
TryAddTimeSpan(KnownMetadataKeys.Duration, audioStream.Duration);
TryAddInt(KnownMetadataKeys.AudioBitrate, audioStream.BitRate / 1000);
TryAddInt(KnownMetadataKeys.AudioSampleRate, audioStream.SampleRateHz);
command.Type = AssetType.Audio;
}
var videoStream = analysis.VideoStreams.FirstOrDefault();
if (videoStream != null)
{
TryAddTimeSpan(KnownMetadataKeys.Duration, videoStream.Duration);
TryAddInt(KnownMetadataKeys.VideoWidth, videoStream.Width);
TryAddInt(KnownMetadataKeys.VideoHeight, videoStream.Height);
command.Type = AssetType.Video;
}
}
catch (Exception)
{
// Throws for invalid file types.
return;
}
}
public IEnumerable<string> Format(Asset asset)
{
yield break;
}
}

121
backend/src/Squidex.Domain.Apps.Entities/Assets/FileTagAssetMetadataSource.cs

@ -45,90 +45,89 @@ public sealed class FileTagAssetMetadataSource : IAssetMetadataSource
{
try
{
using (var file = Create(new FileAbstraction(command.File), ReadStyle.Average))
using var file = Create(new FileAbstraction(command.File), ReadStyle.Average);
if (file.Properties == null)
{
if (file.Properties == null)
{
return Task.CompletedTask;
}
return Task.CompletedTask;
}
var type = file.Properties.MediaTypes;
var type = file.Properties.MediaTypes;
if (type == MediaTypes.Audio)
{
command.Type = AssetType.Audio;
}
else if (type == MediaTypes.Photo)
{
command.Type = AssetType.Image;
}
else if (type.HasFlag(MediaTypes.Video))
{
command.Type = AssetType.Video;
}
if (type == MediaTypes.Audio)
{
command.Type = AssetType.Audio;
}
else if (type == MediaTypes.Photo)
{
command.Type = AssetType.Image;
}
else if (type.HasFlag(MediaTypes.Video))
{
command.Type = AssetType.Video;
}
var pw = file.Properties.PhotoWidth;
var ph = file.Properties.PhotoHeight;
var pw = file.Properties.PhotoWidth;
var ph = file.Properties.PhotoHeight;
if (pw > 0 && ph > 0)
{
command.Metadata[KnownMetadataKeys.PixelWidth] = pw;
command.Metadata[KnownMetadataKeys.PixelHeight] = ph;
}
if (pw > 0 && ph > 0)
{
command.Metadata[KnownMetadataKeys.PixelWidth] = pw;
command.Metadata[KnownMetadataKeys.PixelHeight] = ph;
}
void TryAddString(string name, string? value)
void TryAddString(string name, string? value)
{
if (!string.IsNullOrWhiteSpace(value))
{
if (!string.IsNullOrWhiteSpace(value))
{
command.Metadata.Add(name, value);
}
command.Metadata.Add(name, value);
}
}
void TryAddInt(string name, int? value)
void TryAddInt(string name, long? value)
{
if (value > 0)
{
if (value > 0)
{
command.Metadata.Add(name, (double)value.Value);
}
command.Metadata[name] = value.Value;
}
}
void TryAddDouble(string name, double? value)
void TryAddDouble(string name, double? value)
{
if (value > 0)
{
if (value > 0)
{
command.Metadata.Add(name, value.Value);
}
command.Metadata[name] = value.Value;
}
}
void TryAddTimeSpan(string name, TimeSpan value)
void TryAddTimeSpan(string name, TimeSpan value)
{
if (value != TimeSpan.Zero)
{
if (value != TimeSpan.Zero)
{
command.Metadata.Add(name, value.ToString());
}
command.Metadata[name] = value.ToString();
}
}
if (file.Tag is ImageTag imageTag)
{
TryAddDouble(KnownMetadataKeys.Latitude, imageTag.Latitude);
TryAddDouble(KnownMetadataKeys.Longitude, imageTag.Longitude);
if (file.Tag is ImageTag imageTag)
{
TryAddDouble(KnownMetadataKeys.Latitude, imageTag.Latitude);
TryAddDouble(KnownMetadataKeys.Longitude, imageTag.Longitude);
TryAddString(KnownMetadataKeys.Created, imageTag.DateTime?.ToIso8601());
}
TryAddString(KnownMetadataKeys.Created, imageTag.DateTime?.ToIso8601());
}
TryAddTimeSpan(KnownMetadataKeys.Duration, file.Properties.Duration);
TryAddTimeSpan(KnownMetadataKeys.Duration, file.Properties.Duration);
TryAddInt(KnownMetadataKeys.BitsPerSample, file.Properties.BitsPerSample);
TryAddInt(KnownMetadataKeys.AudioBitrate, file.Properties.AudioBitrate);
TryAddInt(KnownMetadataKeys.AudioChannels, file.Properties.AudioChannels);
TryAddInt(KnownMetadataKeys.AudioSampleRate, file.Properties.AudioSampleRate);
TryAddInt(KnownMetadataKeys.ImageQuality, file.Properties.PhotoQuality);
TryAddInt(KnownMetadataKeys.BitsPerSample, file.Properties.BitsPerSample);
TryAddInt(KnownMetadataKeys.AudioBitrate, file.Properties.AudioBitrate);
TryAddInt(KnownMetadataKeys.AudioChannels, file.Properties.AudioChannels);
TryAddInt(KnownMetadataKeys.AudioSampleRate, file.Properties.AudioSampleRate);
TryAddInt(KnownMetadataKeys.ImageQuality, file.Properties.PhotoQuality);
TryAddInt(KnownMetadataKeys.VideoWidth, file.Properties.VideoWidth);
TryAddInt(KnownMetadataKeys.VideoHeight, file.Properties.VideoHeight);
TryAddInt(KnownMetadataKeys.VideoWidth, file.Properties.VideoWidth);
TryAddInt(KnownMetadataKeys.VideoHeight, file.Properties.VideoHeight);
TryAddString(KnownMetadataKeys.Description, file.Properties.Description);
}
TryAddString(KnownMetadataKeys.Description, file.Properties.Description);
return Task.CompletedTask;
}

1
backend/src/Squidex.Domain.Apps.Entities/Squidex.Domain.Apps.Entities.csproj

@ -25,6 +25,7 @@
</ItemGroup>
<ItemGroup>
<PackageReference Include="CsvHelper" Version="33.0.1" />
<PackageReference Include="FFMpegCore" Version="5.1.0" />
<PackageReference Include="GraphQL" Version="8.2.1" />
<PackageReference Include="GraphQL.DataLoader" Version="8.2.1" />
<PackageReference Include="Lucene.Net.QueryParser" Version="4.8.0-beta00016" />

2
backend/src/Squidex.Web/Constants.cs

@ -16,6 +16,8 @@ public static class Constants
public const string ApiSecurityScheme = "API";
public const string ExternalScheme = "ExternalOidc";
public const string PrefixApi = "/api";
public const string PrefixIdentityServer = "/identity-server";

14
backend/src/Squidex.Web/ContextExtensions.cs

@ -5,13 +5,27 @@
// All rights reserved. Licensed under the MIT license.
// ==========================================================================
using Microsoft.AspNetCore.Authentication;
using Microsoft.AspNetCore.Http;
using Microsoft.AspNetCore.Identity;
using Microsoft.Extensions.DependencyInjection;
using RequestContext = Squidex.Domain.Apps.Entities.Context;
namespace Squidex.Web;
public static class ContextExtensions
{
public static async Task<bool> HasSchemeAsync(this HttpContext httpContext, string name)
{
var provider = httpContext.RequestServices.GetService<IAuthenticationSchemeProvider>();
if (provider == null)
{
return false;
}
return await provider.GetSchemeAsync(name) != null;
}
public static RequestContext Context(this HttpContext httpContext)
{
var context = httpContext.Features.Get<RequestContext>();

1
backend/src/Squidex.Web/Pipeline/AppResolver.cs

@ -38,7 +38,6 @@ public sealed class AppResolver(IAppProvider appProvider) : IAsyncActionFilter
var isFrontend = user.IsInClient(DefaultClients.Frontend);
var app = await appProvider.GetAppAsync(appName, !isFrontend, context.HttpContext.RequestAborted);
if (app == null)
{
var log = context.HttpContext.RequestServices?.GetService<ILogger<AppResolver>>();

15
backend/src/Squidex/Areas/IdentityServer/Config/IdentityServerServices.cs

@ -73,6 +73,10 @@ public static class IdentityServerServices
});
services.AddOpenIddict()
.AddValidation(options =>
{
options.EnableAuthorizationEntryValidation();
})
.AddCore(builder =>
{
builder.SetDefaultScopeEntity<ImmutableScope>();
@ -145,17 +149,14 @@ public static class IdentityServerServices
services.Configure<OpenIddictServerOptions>((c, options) =>
{
var urlGenerator = c.GetRequiredService<IUrlGenerator>();
var identityPrefix = Constants.PrefixIdentityServer;
var identityOptions = c.GetRequiredService<IOptions<MyIdentityOptions>>().Value;
var urlBuilder = c.GetRequiredService<IUrlGenerator>();
Uri BuildUrl(string path)
static Uri BuildUrl(string path)
{
return new Uri($"{identityPrefix.TrimStart('/')}/{path}", UriKind.Relative);
return new Uri($"{Constants.PrefixIdentityServer.TrimStart('/')}/{path}", UriKind.Relative);
}
options.Issuer = new Uri(urlGenerator.BuildUrl());
options.Issuer = new Uri(urlBuilder.BuildUrl());
options.AuthorizationEndpointUris.SetEndpoint(
BuildUrl("connect/authorize"));

20
backend/src/Squidex/Areas/IdentityServer/Controllers/Account/AccountController.cs

@ -5,6 +5,7 @@
// All rights reserved. Licensed under the MIT license.
// ==========================================================================
using Microsoft.AspNetCore.Authentication;
using Microsoft.AspNetCore.Identity;
using Microsoft.AspNetCore.Mvc;
using Microsoft.Extensions.Options;
@ -16,6 +17,7 @@ using Squidex.Infrastructure.Security;
using Squidex.Infrastructure.Translations;
using Squidex.Shared.Identity;
using Squidex.Shared.Users;
using Squidex.Web;
namespace Squidex.Areas.IdentityServer.Controllers.Account;
@ -109,20 +111,22 @@ public sealed class AccountController(
[HttpGet]
[Route("account/logout/")]
public async Task<IActionResult> Logout()
public IActionResult Logout()
{
await SignInManager.SignOutAsync();
return Redirect("~/../");
return SignOut(new AuthenticationProperties
{
RedirectUri = Url.Content("~/../"),
});
}
[HttpGet]
[Route("account/logout-redirect/")]
public async Task<IActionResult> LogoutRedirect()
public IActionResult LogoutRedirect()
{
await SignInManager.SignOutAsync();
return RedirectToAction(nameof(LogoutCompleted));
return SignOut(new AuthenticationProperties
{
RedirectUri = Url.Action(nameof(LogoutCompleted)),
});
}
[HttpGet]

13
backend/src/Squidex/Areas/IdentityServer/Controllers/Connect/AuthorizationController.cs

@ -14,7 +14,6 @@ using Microsoft.IdentityModel.Tokens;
using OpenIddict.Abstractions;
using OpenIddict.Server.AspNetCore;
using Squidex.Areas.IdentityServer.Config;
using Squidex.Areas.IdentityServer.Controllers;
using Squidex.Domain.Users;
using Squidex.Infrastructure;
using Squidex.Shared.Identity;
@ -22,7 +21,7 @@ using Squidex.Shared.Users;
using Squidex.Web;
using static OpenIddict.Abstractions.OpenIddictConstants;
namespace Squidex.Areas.Account.Controllers.Connect;
namespace Squidex.Areas.IdentityServer.Controllers.Connect;
public class AuthorizationController(
IOpenIddictScopeManager scopeManager,
@ -31,7 +30,6 @@ public class AuthorizationController(
: IdentityServerController
{
[HttpPost("connect/token")]
[Produces("application/json")]
public async Task<IActionResult> Exchange()
{
var request = HttpContext.GetOpenIddictServerRequest();
@ -160,7 +158,14 @@ public class AuthorizationController(
{
await SignInManager.SignOutAsync();
return SignOut(OpenIddictServerAspNetCoreDefaults.AuthenticationScheme);
List<string> schemes = [OpenIddictServerAspNetCoreDefaults.AuthenticationScheme];
if (await HttpContext.HasSchemeAsync(Constants.ExternalScheme))
{
schemes.Add(Constants.ExternalScheme);
}
return SignOut(schemes.ToArray());
}
private async Task<ClaimsPrincipal> CreatePrincipalAsync(OpenIddictRequest request, IUser user)

5
backend/src/Squidex/Config/Authentication/OidcHandler.cs

@ -34,11 +34,6 @@ public sealed class OidcHandler(MyIdentityOptions options) : OpenIdConnectEvents
return base.TokenValidated(context);
}
public override Task AuthenticationFailed(AuthenticationFailedContext context)
{
return base.AuthenticationFailed(context);
}
public override Task RedirectToIdentityProviderForSignOut(RedirectContext context)
{
if (!string.IsNullOrEmpty(options.OidcOnSignoutRedirectUrl))

7
backend/src/Squidex/Config/Authentication/OidcServices.cs

@ -8,6 +8,7 @@
using Microsoft.AspNetCore.Authentication;
using Microsoft.AspNetCore.Authentication.OpenIdConnect;
using Squidex.Infrastructure;
using Squidex.Web;
namespace Squidex.Config.Authentication;
@ -17,9 +18,11 @@ public static class OidcServices
{
if (identityOptions.IsOidcConfigured())
{
var displayName = !string.IsNullOrWhiteSpace(identityOptions.OidcName) ? identityOptions.OidcName : OpenIdConnectDefaults.DisplayName;
var displayName = !string.IsNullOrWhiteSpace(identityOptions.OidcName) ?
identityOptions.OidcName :
OpenIdConnectDefaults.DisplayName;
authBuilder.AddOpenIdConnect("ExternalOidc", displayName, options =>
authBuilder.AddOpenIdConnect(Constants.ExternalScheme, displayName, options =>
{
options.Events = new OidcHandler(identityOptions);
options.Authority = identityOptions.OidcAuthority;

3
backend/src/Squidex/Config/Domain/AssetServices.cs

@ -94,6 +94,9 @@ public static class AssetServices
services.AddSingletonAs<SvgAssetMetadataSource>()
.As<IAssetMetadataSource>();
services.AddSingletonAs<FFMpegAssetMetadataSource>()
.As<IAssetMetadataSource>();
services.AddAssetTus();
}

9
backend/src/Squidex/appsettings.json

@ -666,9 +666,9 @@
"githubClient": "211ea00e726baf754c78",
"githubSecret": "d0a0d0fe2c26469ae20987ac265b3a339fd73132",
// Settings for Microsoft auth (keep empty to disable).3
// Settings for Microsoft auth (keep empty to disable).
//
// NOTE: Tennant is optional for using a specific AzureAD tenant
// NOTE: Tenant is optional for using a specific AzureAD tenant
"microsoftClient": "b55da740-6648-4502-8746-b9003f29d5f1",
"microsoftSecret": "idWbANxNYEF4cB368WXJhjN",
"microsoftTenant": null,
@ -774,7 +774,10 @@
"rules": false,
// Set to true to rebuild schemas.
"schemas": false
"schemas": false,
// Set to true to rebuild teams.
"teams": false
},
// A list of configuration values that should be exposed from the info endpoint and in the UI.

2
backend/tests/Squidex.Data.Tests.CodeGenerator/Squidex.Data.Tests.CodeGenerator.csproj

@ -18,7 +18,7 @@
<PrivateAssets>all</PrivateAssets>
<IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
</PackageReference>
<PackageReference Include="Microsoft.CodeAnalysis.CSharp" Version="4.12.0" PrivateAssets="all" />
<PackageReference Include="Microsoft.CodeAnalysis.CSharp" Version="4.11.0" PrivateAssets="all" />
<PackageReference Include="Microsoft.CodeAnalysis.Analyzers" Version="3.11.0">
<PrivateAssets>all</PrivateAssets>
<IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>

4
backend/tests/Squidex.Data.Tests/MongoDb/Domain/Contents/Text/AtlasTextIndexTests.cs

@ -16,11 +16,9 @@ public class AtlasTextIndexTests(AtlasTextIndexFixture fixture) : TextIndexerTes
public override bool SupportsGeo => true;
public AtlasTextIndexFixture _ { get; } = fixture;
public override Task<ITextIndex> CreateSutAsync()
{
return Task.FromResult<ITextIndex>(_.Index);
return Task.FromResult<ITextIndex>(fixture.Index);
}
[Fact]

4
backend/tests/Squidex.Data.Tests/MongoDb/Infrastructure/EventSourcing/MongoEventConsumerProcessorIntegrationTests_Direct.cs

@ -12,10 +12,8 @@ namespace Squidex.MongoDb.Infrastructure.EventSourcing;
[Trait("Category", "Dependencies")]
public class MongoEventConsumerProcessorIntegrationTests_Direct(MongoEventStoreFixture_Direct fixture) : EventConsumerProcessorIntegrationTests, IClassFixture<MongoEventStoreFixture_Direct>
{
public MongoEventStoreFixture _ { get; } = fixture;
public override IEventStore CreateStore()
{
return _.EventStore;
return fixture.EventStore;
}
}

22
backend/tests/Squidex.Data.Tests/MongoDb/Infrastructure/EventSourcing/MongoEventStoreParallelInsertTests.cs

@ -16,12 +16,13 @@ using Squidex.Infrastructure.TestHelpers;
namespace Squidex.MongoDb.Infrastructure.EventSourcing;
[Trait("Category", "Dependencies")]
public class MongoEventStoreParallelInsertTests : IClassFixture<MongoEventStoreFixture_Replica>
public class MongoEventStoreParallelInsertTests(MongoEventStoreFixture_Replica fixture) : IClassFixture<MongoEventStoreFixture_Replica>
{
private readonly TestState<EventConsumerState> state = new TestState<EventConsumerState>(DomainId.Empty);
private readonly DefaultEventFormatter eventFormatter;
public MongoEventStoreFixture _ { get; }
private readonly DefaultEventFormatter eventFormatter =
new DefaultEventFormatter(
new TypeRegistry().Add<IEvent, MyEvent>("MyEvent"),
TestUtils.DefaultSerializer);
public class MyEvent : IEvent
{
@ -60,15 +61,6 @@ public class MongoEventStoreParallelInsertTests : IClassFixture<MongoEventStoreF
}
}
public MongoEventStoreParallelInsertTests(MongoEventStoreFixture_Replica fixture)
{
_ = fixture;
var typeRegistry = new TypeRegistry().Add<IEvent, MyEvent>("MyEvent");
eventFormatter = new DefaultEventFormatter(typeRegistry, TestUtils.DefaultSerializer);
}
[Fact]
public async Task Should_insert_and_retrieve_parallel()
{
@ -182,7 +174,7 @@ public class MongoEventStoreParallelInsertTests : IClassFixture<MongoEventStoreF
state.PersistenceFactory,
eventConsumer,
eventFormatter,
_.EventStore,
fixture.EventStore,
A.Fake<ILogger<EventConsumerProcessor>>());
}
@ -206,7 +198,7 @@ public class MongoEventStoreParallelInsertTests : IClassFixture<MongoEventStoreF
commitList.Add(eventFormatter.ToEventData(Envelope.Create<IEvent>(new MyEvent()), commitId));
}
await _.EventStore.AppendAsync(commitId, streamName, EtagVersion.Any, commitList);
await fixture.EventStore.AppendAsync(commitId, streamName, EtagVersion.Any, commitList);
}
if (i < iterations - 1)

2
backend/tests/Squidex.Data.Tests/Squidex.Data.Tests.csproj

@ -6,8 +6,8 @@
<LangVersion>latest</LangVersion>
<ImplicitUsings>enable</ImplicitUsings>
<Nullable>enable</Nullable>
<NeutralLanguage>en</NeutralLanguage>
<EmitCompilerGeneratedFiles>true</EmitCompilerGeneratedFiles>
<NoWarn>SA0001;NETSDK1206</NoWarn>
</PropertyGroup>
<ItemGroup>
<ProjectReference Include="..\..\src\Squidex.Data.EntityFramework\Squidex.Data.EntityFramework.csproj" />

98
backend/tests/Squidex.Domain.Apps.Entities.Tests/Assets/FFMpegAssetMetadataSourceTests.cs

@ -0,0 +1,98 @@
// ==========================================================================
// Squidex Headless CMS
// ==========================================================================
// Copyright (c) Squidex UG (haftungsbeschraenkt)
// All rights reserved. Licensed under the MIT license.
// ==========================================================================
using Squidex.Assets;
using Squidex.Domain.Apps.Core.Assets;
using Squidex.Domain.Apps.Entities.Assets.Commands;
using Squidex.Domain.Apps.Entities.TestHelpers;
using Squidex.Infrastructure.Json.Objects;
namespace Squidex.Domain.Apps.Entities.Assets;
public class FFMpegAssetMetadataSourceTests : GivenContext
{
private readonly FFMpegAssetMetadataSource sut = new FFMpegAssetMetadataSource();
[Fact]
public async Task Should_ignore_files_with_type()
{
var command = FakeCommand(AssetType.Image);
await sut.EnhanceAsync(command, default);
Assert.Equal(AssetType.Image, command.Type);
}
[Fact]
public async Task Should_not_set_image_height_and_width_metadata_when_file_does_not_have_those_values()
{
var command = Command("SampleAudio_0.4mb.mp3");
await sut.EnhanceAsync(command, default);
Assert.Null(command.Metadata.GetInt32(KnownMetadataKeys.PixelWidth));
Assert.Null(command.Metadata.GetInt32(KnownMetadataKeys.PixelHeight));
}
[Fact]
public async Task Should_provide_metadata_for_audio()
{
var command = Command("SampleAudio_0.4mb.mp3");
await sut.EnhanceAsync(command, default);
Assert.Equal(AssetType.Audio, command.Type);
Assert.Equal(JsonValue.Create(128L), command.Metadata[KnownMetadataKeys.AudioBitrate]);
Assert.Equal(JsonValue.Create(44100L), command.Metadata[KnownMetadataKeys.AudioSampleRate]);
}
[Fact]
public async Task Should_provide_metadata_for_broken_video()
{
var command = Command("SampleVideo_Broken.mp4");
await sut.EnhanceAsync(command, default);
Assert.Equal(AssetType.Video, command.Type);
Assert.Equal(JsonValue.Create("00:00:11"), command.Metadata[KnownMetadataKeys.Duration]);
Assert.Equal(JsonValue.Create(317L), command.Metadata[KnownMetadataKeys.AudioBitrate]);
Assert.Equal(JsonValue.Create(48000L), command.Metadata[KnownMetadataKeys.AudioSampleRate]);
Assert.Equal(JsonValue.Create(1080L), command.Metadata[KnownMetadataKeys.VideoHeight]);
Assert.Equal(JsonValue.Create(1920L), command.Metadata[KnownMetadataKeys.VideoWidth]);
}
[Fact]
public void Should_not_format_asset()
{
var source = CreateAsset() with
{
Type = AssetType.Image,
};
var formatted = sut.Format(source);
Assert.Empty(formatted);
}
private static UploadAssetCommand Command(string path)
{
var file = new FileInfo(Path.Combine("Assets", "TestFiles", path));
return new CreateAsset
{
File = new DelegateAssetFile(file.Name, "mime", file.Length, file.OpenRead),
};
}
private static UploadAssetCommand FakeCommand(AssetType type)
{
return new CreateAsset
{
Type = type,
};
}
}

BIN
backend/tests/Squidex.Domain.Apps.Entities.Tests/Assets/TestFiles/SampleVideo_Broken.mp4

Binary file not shown.

4
backend/tests/Squidex.Domain.Apps.Entities.Tests/Contents/Text/AzureTextIndexTests.cs

@ -12,11 +12,9 @@ public class AzureTextIndexTests(AzureTextIndexFixture fixture) : TextIndexerTes
{
public override bool SupportsGeo => true;
public AzureTextIndexFixture _ { get; } = fixture;
public override Task<ITextIndex> CreateSutAsync()
{
return Task.FromResult<ITextIndex>(_.Index);
return Task.FromResult<ITextIndex>(fixture.Index);
}
[Fact]

4
backend/tests/Squidex.Domain.Apps.Entities.Tests/Contents/Text/ElasticSearchTextIndexTests.cs

@ -12,11 +12,9 @@ public class ElasticSearchTextIndexTests(ElasticSearchTextIndexFixture fixture)
{
public override bool SupportsGeo => true;
public ElasticSearchTextIndexFixture _ { get; } = fixture;
public override Task<ITextIndex> CreateSutAsync()
{
return Task.FromResult<ITextIndex>(_.Index);
return Task.FromResult<ITextIndex>(fixture.Index);
}
[Fact]

4
backend/tests/Squidex.Domain.Apps.Entities.Tests/Contents/Text/OpenSearchTextIndexTests.cs

@ -12,11 +12,9 @@ public class OpenSearchTextIndexTests(OpenSearchTextIndexFixture fixture) : Text
{
public override bool SupportsGeo => true;
public OpenSearchTextIndexFixture _ { get; } = fixture;
public override Task<ITextIndex> CreateSutAsync()
{
return Task.FromResult<ITextIndex>(_.Index);
return Task.FromResult<ITextIndex>(fixture.Index);
}
[Fact]

3
backend/tests/Squidex.Domain.Apps.Entities.Tests/Squidex.Domain.Apps.Entities.Tests.csproj

@ -63,6 +63,9 @@
<None Update="Assets\TestFiles\SampleVideo_1280x720_1mb.mp4">
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
</None>
<None Update="Assets\TestFiles\SampleVideo_Broken.mp4">
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
</None>
<None Update="Assets\TestFiles\SvgInvalid.svg">
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
</None>

11
tools/TestSuite/TestSuite.ApiTests/AssetFormatTests.cs

@ -200,9 +200,6 @@ public class AssetFormatTests(CreatedAppFixture fixture) : IClassFixture<Created
{
var asset = await _.Client.Assets.UploadFileAsync("Assets/SampleVideo_1280x720_1mb.flv", "audio/webm");
// Should not parse yet.
Assert.Equal(AssetType.Unknown, asset.Type);
await Verify(asset);
}
@ -211,20 +208,14 @@ public class AssetFormatTests(CreatedAppFixture fixture) : IClassFixture<Created
{
var asset = await _.Client.Assets.UploadFileAsync("Assets/SampleVideo_1280x720_1mb.flv", "audio/x-flv");
// Should not parse yet.
Assert.Equal(AssetType.Unknown, asset.Type);
await Verify(asset);
}
[Fact]
[Fact(Skip = "Platform specific")]
public async Task Should_upload_video_3gp()
{
var asset = await _.Client.Assets.UploadFileAsync("Assets/SampleVideo_176x144_1mb.3gp", "audio/3gpp");
// Should not parse yet.
Assert.Equal(AssetType.Unknown, asset.Type);
await Verify(asset);
}

8
tools/TestSuite/TestSuite.ApiTests/Verify/AssetFormatTests.Should_upload_video_3gp.verified.txt

@ -7,11 +7,17 @@
Slug: samplevideo-176x144-1mb.3gp,
MimeType: audio/3gpp,
FileType: 3gp,
MetadataText: 1014.4 kB,
MetadataText: 00:00:40.6660000, 1014.4 kB,
Metadata: {
audioBitrate: 12,
audioSampleRate: 8000,
duration: 00:00:40.6660000
},
Tags: [
type/3gp
],
FileSize: 1038741,
Type: Video,
Links: {
content: {
Method: GET

9
tools/TestSuite/TestSuite.ApiTests/Verify/AssetFormatTests.Should_upload_video_flv.verified.txt

@ -7,11 +7,18 @@
Slug: samplevideo-1280x720-1mb.flv,
MimeType: audio/x-flv,
FileType: flv,
MetadataText: 1 MB,
MetadataText: 1280x720pt, 1 MB,
Metadata: {
audioBitrate: 384,
audioSampleRate: 48000,
videoHeight: 720,
videoWidth: 1280
},
Tags: [
type/flv
],
FileSize: 1051185,
Type: Video,
Links: {
content: {
Method: GET

9
tools/TestSuite/TestSuite.ApiTests/Verify/AssetFormatTests.Should_upload_video_mkv.verified.txt

@ -7,11 +7,18 @@
Slug: samplevideo-1280x720-1mb.flv,
MimeType: audio/webm,
FileType: flv,
MetadataText: 1 MB,
MetadataText: 1280x720pt, 1 MB,
Metadata: {
audioBitrate: 384,
audioSampleRate: 48000,
videoHeight: 720,
videoWidth: 1280
},
Tags: [
type/flv
],
FileSize: 1051185,
Type: Video,
Links: {
content: {
Method: GET

Loading…
Cancel
Save