diff --git a/CLAUDE.md b/CLAUDE.md new file mode 100644 index 000000000..b8928da15 --- /dev/null +++ b/CLAUDE.md @@ -0,0 +1,55 @@ +# Squidex + +Headless CMS. Angular frontend in `frontend/`, ASP.NET Core backend in `backend/`. + +## Frontend + +- Angular app in `frontend/`, source under `src/app`: + - `framework/` — generic, reusable UI components and utilities (no domain knowledge). + - `shared/` — Squidex-specific services, state stores and components. + - `features/` — the actual screens (apps, assets, content, rules, schemas, settings, teams, ...). + - `shell/` — app frame, navigation, layout. +- State is handled with the state store pattern from `framework/state.ts` (immutable value objects + `State` subclasses), not with a third-party store library. +- Commands: + +```bash +npm start +``` + +```bash +npm test +``` + +```bash +npm run lint +``` + +### Best Practices + +- i18n texts live in `backend/i18n`, translations are generated into the frontend — do not edit generated translation files by hand. +- Do not write JsDoc comments. + +## Backend + +- .NET solution `backend/Squidex.slnx`. Projects under `backend/src`, tests under `backend/tests`, optional integrations under `backend/extensions`. +- Layering: `Squidex.Infrastructure` (generic building blocks) → `Squidex.Domain.Apps.*` (core model, operations, events, entities) → `Squidex.Web` / `Squidex` (API host). +- Event-sourced domain: aggregates emit events from `Squidex.Domain.Apps.Events`, state is projected into MongoDB or EF Core (`Squidex.Data.MongoDb`, `Squidex.Data.EntityFramework`). +- Run tests with the filter below — some tests need external setup (real databases, Docker/Testcontainers) and will fail without it: + +### Tests + +Some tests need test setup or test containers which are slow. Run the tests like this to skip these tests. + +```bash +dotnet test --filter "Category!=Dependencies & Category!=TestContainer" +``` + +### Best Practices + +- Code style is enforced by StyleCop (`backend/stylecop.json`) and `.editorconfig` — follow the surrounding file's conventions. +- Do not write XML comments. + +## Shared best practices + +- Do write precise short comments and only when needed. +- Do not comment a class or a method, only put comments inside functions or above variables. \ No newline at end of file diff --git a/backend/src/Squidex.Data.EntityFramework/Infrastructure/Extensions.cs b/backend/src/Squidex.Data.EntityFramework/Infrastructure/Extensions.cs index dd48c19b6..11f7d61a2 100644 --- a/backend/src/Squidex.Data.EntityFramework/Infrastructure/Extensions.cs +++ b/backend/src/Squidex.Data.EntityFramework/Infrastructure/Extensions.cs @@ -54,9 +54,15 @@ public static class Extensions return options.GetExtension().Prefix; } - public static DbContextOptionsBuilder SetDefaultWarnings(this DbContextOptionsBuilder builder) + public static DbContextOptionsBuilder SetDefaults(this DbContextOptionsBuilder builder) { builder.ConfigureWarnings(w => w.Ignore(CoreEventId.CollectionWithoutComparer)); + + // Almost everything is read only or written by inserting new entities, so tracking would + // only cost a snapshot of every entity that is read. The few stores that update an entity + // they have queried ask for it with AsTracking. This cannot be done in OnConfiguring, + // because the contexts are pooled and pooling forbids to modify the options there. + builder.UseQueryTrackingBehavior(QueryTrackingBehavior.NoTracking); return builder; } diff --git a/backend/src/Squidex.Data.EntityFramework/ServiceExtensions.cs b/backend/src/Squidex.Data.EntityFramework/ServiceExtensions.cs index 8c0ad4c61..b550a3644 100644 --- a/backend/src/Squidex.Data.EntityFramework/ServiceExtensions.cs +++ b/backend/src/Squidex.Data.EntityFramework/ServiceExtensions.cs @@ -82,7 +82,7 @@ public static class ServiceExtensions services.AddPooledDbContextFactory(builder => { - builder.SetDefaultWarnings(); + builder.SetDefaults(); builder.UseMySql(connectionString, version, options => { options.UseNetTopologySuite(); @@ -93,7 +93,7 @@ public static class ServiceExtensions services.AddNamedDbContext((builder, name) => { - builder.SetDefaultWarnings(); + builder.SetDefaults(); builder.UseBulkInsertMySql(); builder.UseMySql(connectionString, version, options => { @@ -118,7 +118,7 @@ public static class ServiceExtensions { services.AddPooledDbContextFactory(builder => { - builder.SetDefaultWarnings(); + builder.SetDefaults(); builder.UseBulkInsertPostgreSql(); builder.UseNpgsql(connectionString, options => { @@ -128,7 +128,7 @@ public static class ServiceExtensions services.AddNamedDbContext((builder, name) => { - builder.SetDefaultWarnings(); + builder.SetDefaults(); builder.UseBulkInsertPostgreSql(); builder.UseNpgsql(connectionString, options => { @@ -152,7 +152,7 @@ public static class ServiceExtensions { services.AddPooledDbContextFactory(builder => { - builder.SetDefaultWarnings(); + builder.SetDefaults(); builder.UseSqlServer(connectionString, options => { options.UseNetTopologySuite(); @@ -162,7 +162,7 @@ public static class ServiceExtensions services.AddNamedDbContext((builder, name) => { - builder.SetDefaultWarnings(); + builder.SetDefaults(); builder.UseBulkInsertSqlServer(); builder.UseSqlServer(connectionString, options => { diff --git a/backend/src/Squidex.Data.EntityFramework/Squidex.Data.EntityFramework.csproj b/backend/src/Squidex.Data.EntityFramework/Squidex.Data.EntityFramework.csproj index b945ed025..b1bb616c7 100644 --- a/backend/src/Squidex.Data.EntityFramework/Squidex.Data.EntityFramework.csproj +++ b/backend/src/Squidex.Data.EntityFramework/Squidex.Data.EntityFramework.csproj @@ -43,13 +43,13 @@ - - - - - - - + + + + + + + diff --git a/backend/src/Squidex.Data.MongoDb/Domain/Apps/Entities/Assets/MongoAssetRepository.cs b/backend/src/Squidex.Data.MongoDb/Domain/Apps/Entities/Assets/MongoAssetRepository.cs index 5122facb9..ed8e5afe8 100644 --- a/backend/src/Squidex.Data.MongoDb/Domain/Apps/Entities/Assets/MongoAssetRepository.cs +++ b/backend/src/Squidex.Data.MongoDb/Domain/Apps/Entities/Assets/MongoAssetRepository.cs @@ -109,7 +109,7 @@ public sealed partial class MongoAssetRepository : MongoRepositoryBase>> collections = - new ConcurrentDictionary<(DomainId, DomainId), Task>>(); + private readonly ConcurrentDictionary<(DomainId AppId, DomainId SchemaId), Lazy>>> collections = + new ConcurrentDictionary<(DomainId AppId, DomainId SchemaId), Lazy>>>(); - public Task> GetCollectionAsync(DomainId appId, DomainId schemaId) + public async Task> GetCollectionAsync(DomainId appId, DomainId schemaId) { - return collections.GetOrAdd((appId, schemaId), CreateCollectionAsync); + var key = (appId, schemaId); + + // The lazy ensures that the indexes are only created once, even when the same collection is + // requested concurrently. GetOrAdd alone can run the factory several times for one key. + var collection = collections.GetOrAdd(key, CreateLazyCollection); + + try + { + return await collection.Value; + } + catch + { + // A failed attempt must not stay in the cache. Creating the indexes can fail for a + // transient reason and the collection would be unusable until the process is restarted. + // Only remove our own entry, so that a newer successful one is not thrown away. + collections.TryRemove(new KeyValuePair<(DomainId AppId, DomainId SchemaId), Lazy>>>(key, collection)); + throw; + } + } + + private Lazy>> CreateLazyCollection((DomainId AppId, DomainId SchemaId) key) + { + return new Lazy>>( + () => CreateCollectionAsync(key), + LazyThreadSafetyMode.ExecutionAndPublication); } - private async Task> CreateCollectionAsync((DomainId, DomainId) key) + private async Task> CreateCollectionAsync((DomainId AppId, DomainId SchemaId) key) { var (appId, schemaId) = key; diff --git a/backend/src/Squidex.Data.MongoDb/Domain/Apps/Entities/Contents/MongoContentRepository_SnapshotStore.cs b/backend/src/Squidex.Data.MongoDb/Domain/Apps/Entities/Contents/MongoContentRepository_SnapshotStore.cs index d4f4903c3..d55ffa343 100644 --- a/backend/src/Squidex.Data.MongoDb/Domain/Apps/Entities/Contents/MongoContentRepository_SnapshotStore.cs +++ b/backend/src/Squidex.Data.MongoDb/Domain/Apps/Entities/Contents/MongoContentRepository_SnapshotStore.cs @@ -135,7 +135,7 @@ public partial class MongoContentRepository : ISnapshotStore, IDel collectionUpdates.GetOrAddNew(collection).Add(entity); }); - foreach (var job in jobs) + foreach (var job in validJobs) { if (job.Value.ShouldWritePublished()) { diff --git a/backend/src/Squidex.Data.MongoDb/Domain/Apps/Entities/Contents/Operations/QueryByIds.cs b/backend/src/Squidex.Data.MongoDb/Domain/Apps/Entities/Contents/Operations/QueryByIds.cs index c30478d63..c5a84615d 100644 --- a/backend/src/Squidex.Data.MongoDb/Domain/Apps/Entities/Contents/Operations/QueryByIds.cs +++ b/backend/src/Squidex.Data.MongoDb/Domain/Apps/Entities/Contents/Operations/QueryByIds.cs @@ -56,7 +56,7 @@ internal sealed class QueryByIds : OperationBase { contentTotal = -1; } - else + else if (query.NeedsTotalById(q.Ids.Count)) { contentTotal = await Collection.Find(filter).CountDocumentsAsync(ct); } diff --git a/backend/src/Squidex.Data.MongoDb/Domain/Apps/Entities/Contents/Operations/QueryByQuery.cs b/backend/src/Squidex.Data.MongoDb/Domain/Apps/Entities/Contents/Operations/QueryByQuery.cs index e4b785b7c..db5523639 100644 --- a/backend/src/Squidex.Data.MongoDb/Domain/Apps/Entities/Contents/Operations/QueryByQuery.cs +++ b/backend/src/Squidex.Data.MongoDb/Domain/Apps/Entities/Contents/Operations/QueryByQuery.cs @@ -64,6 +64,13 @@ internal sealed class QueryByQuery(MongoCountCollection countCollection) : Opera { contentTotal = -1; } + else if (isDefault) + { + // Cache total count by app and schemas because no other filters are applied (aka default). + var totalKey = CreateTotalKey(app, schemas); + + contentTotal = await countCollection.GetOrAddAsync(totalKey, ct => Collection.Find(filter).CountDocumentsAsync(ct), ct); + } else if (query.IsSatisfiedByIndex()) { // It is faster to filter with sorting when there is an index, because it forces the index to be used. @@ -78,6 +85,16 @@ internal sealed class QueryByQuery(MongoCountCollection countCollection) : Opera return ResultList.Create(contentTotal, contentEntities); } + private static string CreateTotalKey(App app, List schemas) + { + // The schemas depend on the permissions of the user and are not in a stable order, so the ids + // are sorted. They are also hashed, because the key is the ID of the count document and there + // can be enough schemas to exceed the maximum key size of MongoDB. + var schemaIds = schemas.Select(x => x.Id.ToString()).Order(StringComparer.Ordinal); + + return $"{app.Id}_Schemas_{string.Join('_', schemaIds).ToSha256Base64()}"; + } + public async Task> QueryAsync(Schema schema, Q q, CancellationToken ct) { diff --git a/backend/src/Squidex.Data.MongoDb/Infrastructure/Queries/LimitExtensions.cs b/backend/src/Squidex.Data.MongoDb/Infrastructure/Queries/LimitExtensions.cs index becc31bd1..7926f00f2 100644 --- a/backend/src/Squidex.Data.MongoDb/Infrastructure/Queries/LimitExtensions.cs +++ b/backend/src/Squidex.Data.MongoDb/Infrastructure/Queries/LimitExtensions.cs @@ -11,6 +11,13 @@ namespace Squidex.Infrastructure.Queries; public static class LimitExtensions { + public static bool NeedsTotalById(this ClrQuery query, int idCount) + { + // A query by ID can never match more documents than the number of requested IDs, so the result + // already contains all of them unless skip, take or the random selection could have cut it off. + return query.Skip > 0 || query.Take < idCount || query.Random > 0; + } + public static IAggregateFluent QueryLimit(this IAggregateFluent cursor, ClrQuery query) { if (query.Take < long.MaxValue) diff --git a/backend/src/Squidex.Data.MongoDb/Squidex.Data.MongoDb.csproj b/backend/src/Squidex.Data.MongoDb/Squidex.Data.MongoDb.csproj index b7ccc948c..8609dfa0f 100644 --- a/backend/src/Squidex.Data.MongoDb/Squidex.Data.MongoDb.csproj +++ b/backend/src/Squidex.Data.MongoDb/Squidex.Data.MongoDb.csproj @@ -25,12 +25,12 @@ - - - - - - + + + + + + diff --git a/backend/src/Squidex.Domain.Apps.Core.Model/Apps/Roles.cs b/backend/src/Squidex.Domain.Apps.Core.Model/Apps/Roles.cs index 58c30bfcc..379d5f4ff 100644 --- a/backend/src/Squidex.Domain.Apps.Core.Model/Apps/Roles.cs +++ b/backend/src/Squidex.Domain.Apps.Core.Model/Apps/Roles.cs @@ -5,6 +5,7 @@ // All rights reserved. Licensed under the MIT license. // ========================================================================== +using System.Collections.Concurrent; using System.Diagnostics.CodeAnalysis; using System.Diagnostics.Contracts; using Squidex.Infrastructure; @@ -17,6 +18,8 @@ namespace Squidex.Domain.Apps.Core.Apps; public sealed class Roles { + private const int MaxResolved = 1000; + private readonly ConcurrentDictionary<(string App, string Name, bool IsFrontend), Role?> resolved = new ConcurrentDictionary<(string, string, bool), Role?>(); private readonly ReadonlyDictionary inner; public static readonly IReadOnlyDictionary Defaults = new Dictionary @@ -159,18 +162,34 @@ public sealed class Roles { Guard.NotNull(app); - value = null!; + // Resolving a role builds a permission for every permission of the role, but the result only + // depends on the key and the roles are immutable, so it is only done once. This is called for + // every request. + value = resolved.GetOrAdd((app, name, isFrontend), static (key, self) => self.Resolve(key.App, key.Name, key.IsFrontend), this)!; + + return value != null; + } + + private Role? Resolve(string app, string name, bool isFrontend) + { + // Apps without custom roles share the same empty instance, so this cache is not bound to a + // single app and could grow with the number of apps. Start over when it gets too large. + if (resolved.Count >= MaxResolved) + { + resolved.Clear(); + } if (Defaults.TryGetValue(name, out var role)) { - value = role.ForApp(app, isFrontend && name != Role.Owner); + return role.ForApp(app, isFrontend && name != Role.Owner); } - else if (inner.TryGetValue(name, out role)) + + if (inner.TryGetValue(name, out role)) { - value = role.ForApp(app, isFrontend); + return role.ForApp(app, isFrontend); } - return value != null; + return null; } private static string WithoutPrefix(string permission) diff --git a/backend/src/Squidex.Domain.Apps.Core.Model/Squidex.Domain.Apps.Core.Model.csproj b/backend/src/Squidex.Domain.Apps.Core.Model/Squidex.Domain.Apps.Core.Model.csproj index 9c50b0ea1..2887139e2 100644 --- a/backend/src/Squidex.Domain.Apps.Core.Model/Squidex.Domain.Apps.Core.Model.csproj +++ b/backend/src/Squidex.Domain.Apps.Core.Model/Squidex.Domain.Apps.Core.Model.csproj @@ -20,7 +20,7 @@ - + diff --git a/backend/src/Squidex.Domain.Apps.Core.Operations/ConvertContent/ContentConverter.cs b/backend/src/Squidex.Domain.Apps.Core.Operations/ConvertContent/ContentConverter.cs index 4d66827e7..fc928104f 100644 --- a/backend/src/Squidex.Domain.Apps.Core.Operations/ConvertContent/ContentConverter.cs +++ b/backend/src/Squidex.Domain.Apps.Core.Operations/ConvertContent/ContentConverter.cs @@ -149,22 +149,27 @@ public sealed class ContentConverter(ResolvedComponents components, Schema schem return (true, default); } - for (int i = 0; i < array.Count; i++) + // Compact in place instead of removing items, because every remove moves all the items + // after it and dropping many items from a large array would be quadratic. + var target = 0; + + for (var i = 0; i < array.Count; i++) { var oldValue = array[i]; var (removed, newValue) = ConvertArrayItem(field, oldValue); if (removed) { - array.RemoveAt(i); - i--; - } - else if (!ReferenceEquals(newValue.Value, oldValue.Value)) - { - array[i] = newValue; + continue; } + + // Faster to check for reference equality than for deep equals. + array[target] = ReferenceEquals(newValue.Value, oldValue.Value) ? oldValue : newValue; + target++; } + array.RemoveRange(target, array.Count - target); + return (false, array); } @@ -175,23 +180,27 @@ public sealed class ContentConverter(ResolvedComponents components, Schema schem return (true, default); } - for (int i = 0; i < array.Count; i++) + // Compact in place instead of removing items, because every remove moves all the items + // after it and dropping many items from a large array would be quadratic. + var target = 0; + + for (var i = 0; i < array.Count; i++) { var oldValue = array[i]; var (removed, newValue) = ConvertComponent(oldValue, parent); if (removed) { - array.RemoveAt(i); - i--; - } - else if (!ReferenceEquals(newValue.Value, oldValue.Value)) - { - // Faster to check for reference equality than for deep equals. - array[i] = newValue; + continue; } + + // Faster to check for reference equality than for deep equals. + array[target] = ReferenceEquals(newValue.Value, oldValue.Value) ? oldValue : newValue; + target++; } + array.RemoveRange(target, array.Count - target); + return (false, array); } diff --git a/backend/src/Squidex.Domain.Apps.Core.Operations/ConvertContent/UpdateValues.cs b/backend/src/Squidex.Domain.Apps.Core.Operations/ConvertContent/UpdateValues.cs index 3cb1465ea..c4140f177 100644 --- a/backend/src/Squidex.Domain.Apps.Core.Operations/ConvertContent/UpdateValues.cs +++ b/backend/src/Squidex.Domain.Apps.Core.Operations/ConvertContent/UpdateValues.cs @@ -14,6 +14,7 @@ namespace Squidex.Domain.Apps.Core.ConvertContent; public sealed class UpdateValues(ContentData existingData, IScriptEngine scriptEngine, bool canUnset) : IContentValueConverter, IContentDataConverter { + private static readonly ScriptOptions Options = new ScriptOptions { Readonly = true }; private ScriptVars? vars; public void ConvertDataBefore(Schema schema, ContentData source) @@ -32,8 +33,6 @@ public sealed class UpdateValues(ContentData existingData, IScriptEngine scriptE if (Updates.IsUpdate(value, out var expression)) { - var options = new ScriptOptions { Readonly = true }; - // Reuse the vars to save allocations. vars ??= new ScriptVars { @@ -44,7 +43,7 @@ public sealed class UpdateValues(ContentData existingData, IScriptEngine scriptE vars["$self"] = value; // Put the expression in brackets to return an object directly. - var result = scriptEngine.Execute(vars, $"({expression})", options); + var result = scriptEngine.Execute(vars, $"({expression})", Options); if (result.Value is JsonObject obj) { @@ -93,8 +92,6 @@ public sealed class UpdateValues(ContentData existingData, IScriptEngine scriptE return (false, source); } - var options = new ScriptOptions { Readonly = true }; - // Reuse the vars to save allocations. vars ??= new ScriptVars { @@ -105,7 +102,7 @@ public sealed class UpdateValues(ContentData existingData, IScriptEngine scriptE vars["$self"] = obj; // Put the expression in brackets to return an object directly. - var result = scriptEngine.Execute(vars, $"({expression})", options); + var result = scriptEngine.Execute(vars, $"({expression})", Options); return (false, result); } diff --git a/backend/src/Squidex.Domain.Apps.Core.Operations/HandleRules/EventEnricher.cs b/backend/src/Squidex.Domain.Apps.Core.Operations/HandleRules/EventEnricher.cs index bffed50c5..f5225bb04 100644 --- a/backend/src/Squidex.Domain.Apps.Core.Operations/HandleRules/EventEnricher.cs +++ b/backend/src/Squidex.Domain.Apps.Core.Operations/HandleRules/EventEnricher.cs @@ -43,7 +43,7 @@ public sealed class EventEnricher(IMemoryCache userCache, IUserResolver userReso private Task FindUserAsync(RefToken actor) { - var cacheKey = $"{typeof(EventEnricher)}_Users_{actor.Identifier}"; + var cacheKey = (typeof(EventEnricher), actor.Identifier); return userCache.GetOrCreateAsync(cacheKey, async x => { diff --git a/backend/src/Squidex.Domain.Apps.Core.Operations/HandleRules/Extensions/EventJintExtension.cs b/backend/src/Squidex.Domain.Apps.Core.Operations/HandleRules/Extensions/EventJintExtension.cs index 3d1cdb3db..a58160614 100644 --- a/backend/src/Squidex.Domain.Apps.Core.Operations/HandleRules/Extensions/EventJintExtension.cs +++ b/backend/src/Squidex.Domain.Apps.Core.Operations/HandleRules/Extensions/EventJintExtension.cs @@ -5,6 +5,7 @@ // All rights reserved. Licensed under the MIT license. // ========================================================================== +using Jint; using Jint.Native; using Squidex.Domain.Apps.Core.Properties; using Squidex.Domain.Apps.Core.Rules.EnrichedEvents; @@ -65,13 +66,13 @@ public sealed class EventJintExtension(IUrlGenerator urlGenerator) : IJintExtens } } - public void Extend(ScriptExecutionContext context) + public void Extend(Engine engine) { - context.Engine.SetValue("console", FlowConsoleWrapper.Instance); + engine.SetValue("console", FlowConsoleWrapper.Instance); - context.Engine.SetValue("contentAction", new EventDelegate(() => + engine.SetValue("contentAction", new EventDelegate(() => { - if (context.TryGetValue("event", out var temp) && temp is EnrichedContentEvent contentEvent) + if (engine.TryGetVar("event", out var contentEvent)) { return contentEvent.Status.ToString(); } @@ -79,9 +80,9 @@ public sealed class EventJintExtension(IUrlGenerator urlGenerator) : IJintExtens return JsValue.Null; })); - context.Engine.SetValue("contentUrl", new EventDelegate(() => + engine.SetValue("contentUrl", new EventDelegate(() => { - if (context.TryGetValue("event", out var temp) && temp is EnrichedContentEvent contentEvent) + if (engine.TryGetVar("event", out var contentEvent)) { return urlGenerator.ContentUI(contentEvent.AppId, contentEvent.SchemaId, contentEvent.Id); } @@ -89,9 +90,9 @@ public sealed class EventJintExtension(IUrlGenerator urlGenerator) : IJintExtens return JsValue.Null; })); - context.Engine.SetValue("assetContentSlugUrl", new EventDelegate(() => + engine.SetValue("assetContentSlugUrl", new EventDelegate(() => { - if (context.TryGetValue("event", out var temp) && temp is EnrichedAssetEvent assetEvent) + if (engine.TryGetVar("event", out var assetEvent)) { return urlGenerator.AssetContent(assetEvent.AppId, assetEvent.FileName.Slugify()); } @@ -101,7 +102,7 @@ public sealed class EventJintExtension(IUrlGenerator urlGenerator) : IJintExtens var assetUrl = new EventDelegate(() => { - if (context.TryGetValue("event", out var temp) && temp is EnrichedAssetEvent assetEvent) + if (engine.TryGetVar("event", out var assetEvent)) { return urlGenerator.AssetContent(assetEvent.AppId, assetEvent.Id.ToString()); } @@ -109,8 +110,8 @@ public sealed class EventJintExtension(IUrlGenerator urlGenerator) : IJintExtens return JsValue.Null; }); - context.Engine.SetValue("assetContentUrl", assetUrl); - context.Engine.SetValue("assetContentAppUrl", assetUrl); + engine.SetValue("assetContentUrl", assetUrl); + engine.SetValue("assetContentAppUrl", assetUrl); } public void Describe(AddDescription describe, ScriptScope scope) diff --git a/backend/src/Squidex.Domain.Apps.Core.Operations/Scripting/EngineExtensions.cs b/backend/src/Squidex.Domain.Apps.Core.Operations/Scripting/EngineExtensions.cs new file mode 100644 index 000000000..ac9312b17 --- /dev/null +++ b/backend/src/Squidex.Domain.Apps.Core.Operations/Scripting/EngineExtensions.cs @@ -0,0 +1,34 @@ +// ========================================================================== +// Squidex Headless CMS +// ========================================================================== +// Copyright (c) Squidex UG (haftungsbeschraenkt) +// All rights reserved. Licensed under the MIT license. +// ========================================================================== + +using System.Diagnostics.CodeAnalysis; +using Jint; + +namespace Squidex.Domain.Apps.Core.Scripting; + +public static class EngineExtensions +{ + public static ScriptExecutionContext GetContext(this Engine engine) + { + return ScriptExecutionContext.GetContext(engine); + } + + public static bool TryGetVar(this Engine engine, string key, [MaybeNullWhen(false)] out T value) + { + return ScriptExecutionContext.GetContext(engine).TryGetValueIfExists(key, out value); + } + + public static void Schedule(this Engine engine, Func action) + { + ScriptExecutionContext.GetContext(engine).Schedule(action); + } + + public static void Schedule(this Engine engine, Func> action, Action? callback) + { + ScriptExecutionContext.GetContext(engine).Schedule(action, callback); + } +} diff --git a/backend/src/Squidex.Domain.Apps.Core.Operations/Scripting/Extensions/HttpJintExtension.cs b/backend/src/Squidex.Domain.Apps.Core.Operations/Scripting/Extensions/HttpJintExtension.cs index 6a683b461..efb3ae4d0 100644 --- a/backend/src/Squidex.Domain.Apps.Core.Operations/Scripting/Extensions/HttpJintExtension.cs +++ b/backend/src/Squidex.Domain.Apps.Core.Operations/Scripting/Extensions/HttpJintExtension.cs @@ -21,48 +21,48 @@ public sealed class HttpJintExtension(IHttpClientFactory httpClientFactory) : IJ private delegate void HttpJsonWithBodyDelegate(string url, JsValue body, Action callback, JsValue? headers = null, bool ignoreError = false); private delegate void HttpRequestDelegate(JsValue requestInit, Action callback); - public void ExtendAsync(ScriptExecutionContext context) + public void ExtendAsync(Engine engine) { - AddBodyMethod(context, HttpMethod.Patch, "patchJSON"); - AddBodyMethod(context, HttpMethod.Post, "postJSON"); - AddBodyMethod(context, HttpMethod.Put, "putJSON"); - AddMethod(context, HttpMethod.Delete, "deleteJSON"); - AddMethod(context, HttpMethod.Get, "getJSON"); - AddMethod(context, "request"); + AddBodyMethod(engine, HttpMethod.Patch, "patchJSON"); + AddBodyMethod(engine, HttpMethod.Post, "postJSON"); + AddBodyMethod(engine, HttpMethod.Put, "putJSON"); + AddMethod(engine, HttpMethod.Delete, "deleteJSON"); + AddMethod(engine, HttpMethod.Get, "getJSON"); + AddMethod(engine, "request"); } - private void AddMethod(ScriptExecutionContext context, string name) + private void AddMethod(Engine engine, string name) { var action = new HttpRequestDelegate((requestInit, callback) => { - var httpRequest = ParseRequestInit(requestInit); - Request(context, httpRequest.Method, httpRequest.Url, httpRequest.Body, callback, httpRequest.Headers, true, true); + var (url, method, headers, body) = ParseRequestInit(requestInit); + Request(engine, method, url, body, callback, headers, true, true); }); - context.Engine.SetValue(name, action); + engine.SetValue(name, action); } - private void AddMethod(ScriptExecutionContext context, HttpMethod method, string name) + private void AddMethod(Engine engine, HttpMethod method, string name) { var action = new HttpJsonDelegate((url, callback, headers, ignoreError) => { - Request(context, method, url, null, callback, headers, ignoreError); + Request(engine, method, url, null, callback, headers, ignoreError); }); - context.Engine.SetValue(name, action); + engine.SetValue(name, action); } - private void AddBodyMethod(ScriptExecutionContext context, HttpMethod method, string name) + private void AddBodyMethod(Engine engine, HttpMethod method, string name) { var action = new HttpJsonWithBodyDelegate((url, body, callback, headers, ignoreError) => { - Request(context, method, url, body, callback, headers, ignoreError); + Request(engine, method, url, body, callback, headers, ignoreError); }); - context.Engine.SetValue(name, action); + engine.SetValue(name, action); } - private void Request(ScriptExecutionContext context, HttpMethod method, string url, JsValue? body, Action callback, JsValue? headers, bool ignoreError, bool forceRawResponse = false) + private void Request(Engine engine, HttpMethod method, string url, JsValue? body, Action callback, JsValue? headers, bool ignoreError, bool forceRawResponse = false) { if (!Uri.TryCreate(url, UriKind.Absolute, out var uri)) { @@ -74,53 +74,67 @@ public sealed class HttpJintExtension(IHttpClientFactory httpClientFactory) : IJ throw new JavaScriptException("Callback is not defined."); } - context.Schedule(async (scheduler, ct) => + // The request reads javascript values and is therefore created while we are still inside the engine. + var request = CreateRequest(engine, method, uri, body, headers); + + engine.Schedule(async ct => { try { - var httpClient = httpClientFactory.CreateClient("Jint"); - - var request = CreateRequest(context, method, uri, body, headers); - var response = await httpClient.SendAsync(request, ct); - - if (!ignoreError) + using (request) { - response.EnsureSuccessStatusCode(); - } + var httpClient = httpClientFactory.CreateClient("Jint"); - JsValue responseObject; + using var response = await httpClient.SendAsync(request, ct); - var responseString = await response.Content.ReadAsStringAsync(ct); - - if (ignoreError && (forceRawResponse || !response.IsSuccessStatusCode || string.IsNullOrEmpty(responseString))) - { - responseObject = JsValue.FromObject(context.Engine, new Dictionary + if (!ignoreError) { - ["statusCode"] = (int)response.StatusCode, - ["headers"] = + response.EnsureSuccessStatusCode(); + } + + var responseString = await response.Content.ReadAsStringAsync(ct); + + return ( + StatusCode: (int)response.StatusCode, + Headers: response.Content.Headers .Concat(response.Headers) .Concat(response.TrailingHeaders) .GroupBy(x => x.Key) .ToDictionary(x => x.Key, x => x.Last().Value.First()), - ["body"] = responseString, - }); - } - else - { - responseObject = ParseResponse(context, responseString, ct); + Body: responseString, + IsRaw: ignoreError && (forceRawResponse || !response.IsSuccessStatusCode || string.IsNullOrEmpty(responseString)) + ); } - - scheduler.Run(callback, responseObject); } - catch (Exception ex) + catch (Exception ex) when (ex is not OperationCanceledException) { throw new JavaScriptException(ex.Message); } + }, + response => + { + JsValue responseObject; + + if (response.IsRaw) + { + responseObject = JsValue.FromObject(engine, new Dictionary + { + ["statusCode"] = response.StatusCode, + ["headers"] = response.Headers, + ["body"] = response.Body, + }); + } + else + { + responseObject = new JsonParser(engine).Parse(response.Body); + } + + callback(responseObject); }); } - private static HttpRequestMessage CreateRequest(ScriptExecutionContext context, + private static HttpRequestMessage CreateRequest(Engine engine, HttpMethod method, Uri uri, JsValue? body, @@ -166,7 +180,7 @@ public sealed class HttpJintExtension(IHttpClientFactory httpClientFactory) : IJ } else { - var jsonWriter = new JsonSerializer(context.Engine); + var jsonWriter = new JsonSerializer(engine); var jsonContent = jsonWriter.Serialize(body, JsValue.Undefined, JsValue.Undefined)?.ToString(); if (jsonContent != null) @@ -179,19 +193,6 @@ public sealed class HttpJintExtension(IHttpClientFactory httpClientFactory) : IJ return request; } - private static JsValue ParseResponse(ScriptExecutionContext context, string responseString, - CancellationToken ct) - { - ct.ThrowIfCancellationRequested(); - - var jsonParser = new JsonParser(context.Engine); - var jsonValue = jsonParser.Parse(responseString); - - ct.ThrowIfCancellationRequested(); - - return jsonValue; - } - public void Describe(AddDescription describe, ScriptScope scope) { if (!scope.HasFlag(ScriptScope.Async)) diff --git a/backend/src/Squidex.Domain.Apps.Core.Operations/Scripting/Extensions/StringAsyncJintExtension.cs b/backend/src/Squidex.Domain.Apps.Core.Operations/Scripting/Extensions/StringAsyncJintExtension.cs index dbd1fcadd..b8d5fb3b7 100644 --- a/backend/src/Squidex.Domain.Apps.Core.Operations/Scripting/Extensions/StringAsyncJintExtension.cs +++ b/backend/src/Squidex.Domain.Apps.Core.Operations/Scripting/Extensions/StringAsyncJintExtension.cs @@ -5,6 +5,7 @@ // All rights reserved. Licensed under the MIT license. // ========================================================================== +using Jint; using Jint.Native; using Jint.Runtime; using Squidex.AI; @@ -20,39 +21,40 @@ public sealed class StringAsyncJintExtension(ITranslator translator, IChatAgent private delegate void TextGenerateDelegate(string prompt, Action callback); private delegate void TextTranslateDelegate(string text, string language, Action callback, string sourceLanguage); - public void ExtendAsync(ScriptExecutionContext context) + public void ExtendAsync(Engine engine) { var generate = new TextGenerateDelegate((prompt, callback) => { - Generate(context, prompt, callback); + Generate(engine, prompt, callback); }); var translate = new TextTranslateDelegate((text, language, callback, sourceLanguage) => { - Translate(context, text, language, callback, sourceLanguage); + Translate(engine, text, language, callback, sourceLanguage); }); - context.Engine.SetValue("generate", generate); - context.Engine.SetValue("translate", translate); + engine.SetValue("generate", generate); + engine.SetValue("translate", translate); } - private void Generate(ScriptExecutionContext context, string prompt, Action callback) + private void Generate(Engine engine, string prompt, Action callback) { if (callback == null) { throw new JavaScriptException("Callback is not defined."); } - context.Schedule(async (scheduler, ct) => + // We are still inside the engine here, therefore the callback can be invoked directly. + if (string.IsNullOrWhiteSpace(prompt)) + { + callback(JsValue.Null); + return; + } + + engine.Schedule(async ct => { try { - if (string.IsNullOrWhiteSpace(prompt)) - { - scheduler.Run(callback, JsValue.Null); - return; - } - var request = new ChatRequest { Prompt = prompt, @@ -60,41 +62,44 @@ public sealed class StringAsyncJintExtension(ITranslator translator, IChatAgent var result = await chatAgent.PromptAsync(request, ct: ct); - scheduler.Run(callback, JsValue.FromObject(context.Engine, result.Content)); + return result.Content; } - catch (Exception ex) + catch (Exception ex) when (ex is not OperationCanceledException) { throw new JavaScriptException(ex.Message); } - }); + }, + content => callback(JsValue.FromObject(engine, content))); } - private void Translate(ScriptExecutionContext context, string text, string language, Action callback, string sourceLanguage) + private void Translate(Engine engine, string text, string language, Action callback, string sourceLanguage) { if (callback == null) { throw new JavaScriptException("Callback is not defined."); } - context.Schedule(async (scheduler, ct) => + // We are still inside the engine here, therefore the callback can be invoked directly. + if (string.IsNullOrWhiteSpace(text) || string.IsNullOrWhiteSpace(language)) + { + callback(JsValue.Null); + return; + } + + engine.Schedule(async ct => { try { - if (string.IsNullOrWhiteSpace(text) || string.IsNullOrWhiteSpace(language)) - { - scheduler.Run(callback, JsValue.Null); - return; - } - var translation = await translator.TranslateAsync(text, language, sourceLanguage, ct); - scheduler.Run(callback, JsValue.FromObject(context.Engine, translation.Text)); + return translation.Text; } - catch (Exception ex) + catch (Exception ex) when (ex is not OperationCanceledException) { throw new JavaScriptException(ex.Message); } - }); + }, + translated => callback(JsValue.FromObject(engine, translated))); } public void Describe(AddDescription describe, ScriptScope scope) diff --git a/backend/src/Squidex.Domain.Apps.Core.Operations/Scripting/IAsyncScript.cs b/backend/src/Squidex.Domain.Apps.Core.Operations/Scripting/IAsyncScript.cs new file mode 100644 index 000000000..be09e4a93 --- /dev/null +++ b/backend/src/Squidex.Domain.Apps.Core.Operations/Scripting/IAsyncScript.cs @@ -0,0 +1,33 @@ +// ========================================================================== +// Squidex Headless CMS +// ========================================================================== +// Copyright (c) Squidex UG (haftungsbeschraenkt) +// All rights reserved. Licensed under the MIT license. +// ========================================================================== + +using Squidex.Domain.Apps.Core.Contents; +using Squidex.Infrastructure.Json.Objects; + +namespace Squidex.Domain.Apps.Core.Scripting; + +public interface IAsyncScript : IDisposable +{ + ValueTask TransformAsync(DataScriptVars vars, + CancellationToken ct = default); + + ValueTask ExecuteAsync(ScriptVars vars, + CancellationToken ct = default); + + async ValueTask EvaluateAsync(ScriptVars vars, + CancellationToken ct = default) + { + try + { + return (await ExecuteAsync(vars, ct)).Equals(true); + } + catch + { + return false; + } + } +} diff --git a/backend/src/Squidex.Domain.Apps.Core.Operations/Scripting/IJintExtension.cs b/backend/src/Squidex.Domain.Apps.Core.Operations/Scripting/IJintExtension.cs index 0f0f6d391..014c92219 100644 --- a/backend/src/Squidex.Domain.Apps.Core.Operations/Scripting/IJintExtension.cs +++ b/backend/src/Squidex.Domain.Apps.Core.Operations/Scripting/IJintExtension.cs @@ -15,11 +15,7 @@ public interface IJintExtension { } - void Extend(ScriptExecutionContext context) - { - } - - void ExtendAsync(ScriptExecutionContext context) + void ExtendAsync(Engine engine) { } } diff --git a/backend/src/Squidex.Domain.Apps.Core.Operations/Scripting/IScript.cs b/backend/src/Squidex.Domain.Apps.Core.Operations/Scripting/IScript.cs new file mode 100644 index 000000000..440e4abd9 --- /dev/null +++ b/backend/src/Squidex.Domain.Apps.Core.Operations/Scripting/IScript.cs @@ -0,0 +1,30 @@ +// ========================================================================== +// Squidex Headless CMS +// ========================================================================== +// Copyright (c) Squidex UG (haftungsbeschraenkt) +// All rights reserved. Licensed under the MIT license. +// ========================================================================== + +using Squidex.Domain.Apps.Core.Contents; +using Squidex.Infrastructure.Json.Objects; + +namespace Squidex.Domain.Apps.Core.Scripting; + +public interface IScript : IDisposable +{ + ContentData Transform(DataScriptVars vars); + + JsonValue Execute(ScriptVars vars); + + bool Evaluate(ScriptVars vars) + { + try + { + return Execute(vars).Equals(true); + } + catch + { + return false; + } + } +} diff --git a/backend/src/Squidex.Domain.Apps.Core.Operations/Scripting/IScriptEngine.cs b/backend/src/Squidex.Domain.Apps.Core.Operations/Scripting/IScriptEngine.cs index 629b49885..e71c56b00 100644 --- a/backend/src/Squidex.Domain.Apps.Core.Operations/Scripting/IScriptEngine.cs +++ b/backend/src/Squidex.Domain.Apps.Core.Operations/Scripting/IScriptEngine.cs @@ -1,4 +1,4 @@ -// ========================================================================== +// ========================================================================== // Squidex Headless CMS // ========================================================================== // Copyright (c) Squidex UG (haftungsbeschraenkt) @@ -12,19 +12,62 @@ namespace Squidex.Domain.Apps.Core.Scripting; public interface IScriptEngine { - Task ExecuteAsync(ScriptVars vars, string script, ScriptOptions options = default, - CancellationToken ct = default); + IScript CreateScript(string script, ScriptOptions options = default); + + IAsyncScript CreateAsyncScript(string script, ScriptOptions options = default); + + ContentData Transform(DataScriptVars vars, string script, ScriptOptions options = default) + { + using var compiled = CreateScript(script, options); - Task TransformAsync(DataScriptVars vars, string script, ScriptOptions options = default, - CancellationToken ct = default); + return compiled.Transform(vars); + } + + JsonValue Execute(ScriptVars vars, string script, ScriptOptions options = default) + { + using var compiled = CreateScript(script, options); - JsonValue Execute(ScriptVars vars, string script, ScriptOptions options = default); + return compiled.Execute(vars); + } bool Evaluate(ScriptVars vars, string script, ScriptOptions options = default) { try { - return Execute(vars, script, options).Equals(true); + using var compiled = CreateScript(script, options); + + return compiled.Evaluate(vars); + } + catch + { + return false; + } + } + + async ValueTask TransformAsync(DataScriptVars vars, string script, ScriptOptions options = default, + CancellationToken ct = default) + { + using var compiled = CreateAsyncScript(script, options); + + return await compiled.TransformAsync(vars, ct); + } + + async ValueTask ExecuteAsync(ScriptVars vars, string script, ScriptOptions options = default, + CancellationToken ct = default) + { + using var compiled = CreateAsyncScript(script, options); + + return await compiled.ExecuteAsync(vars, ct); + } + + async ValueTask EvaluateAsync(ScriptVars vars, string script, ScriptOptions options = default, + CancellationToken ct = default) + { + try + { + using var compiled = CreateAsyncScript(script, options); + + return await compiled.EvaluateAsync(vars, ct); } catch { diff --git a/backend/src/Squidex.Domain.Apps.Core.Operations/Scripting/Internal/AsyncScriptExecutionContext.cs b/backend/src/Squidex.Domain.Apps.Core.Operations/Scripting/Internal/AsyncScriptExecutionContext.cs new file mode 100644 index 000000000..327256a73 --- /dev/null +++ b/backend/src/Squidex.Domain.Apps.Core.Operations/Scripting/Internal/AsyncScriptExecutionContext.cs @@ -0,0 +1,171 @@ +// ========================================================================== +// Squidex Headless CMS +// ========================================================================== +// Copyright (c) Squidex UG (haftungsbeschraenkt) +// All rights reserved. Licensed under the MIT license. +// ========================================================================== + +using Jint; +using Squidex.Infrastructure.Tasks; + +namespace Squidex.Domain.Apps.Core.Scripting.Internal; + +internal sealed class AsyncScriptExecutionContext : ScriptExecutionContext +{ + private readonly TaskCompletionSource tcs = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + private readonly CancellationTokenRegistration cancellationRegistration; + private readonly CancellationToken cancellationToken; + private readonly JintScript script; + private readonly Engine engine; + private int pendingTasks = 1; + + private sealed class CompletedValue + { + public T Value { get; init; } + } + + public bool IsCompleted + { + get => tcs.Task.IsCompleted; + } + + internal AsyncScriptExecutionContext(Engine engine, JintScript script, CancellationToken ct) + : base(engine) + { + this.engine = engine; + + // The lock belongs to the script, because the engine is shared between all executions. + this.script = script; + + cancellationToken = ct; + + // Settle the source on cancellation, so that pending callbacks do not enter the engine anymore. + cancellationRegistration = cancellationToken.Register(static state => + { + var self = (AsyncScriptExecutionContext)state!; + + self.tcs.TrySetCanceled(self.cancellationToken); + }, + this); + } + + public async Task WaitForCompletionAsync(Func fallback) + { + TryComplete(); + try + { + var result = await tcs.Task; + if (result != null) + { + return result.Value; + } + + return await script.RunLockedAsync(() => fallback(), cancellationToken); + } + finally + { + await cancellationRegistration.DisposeAsync(); + } + } + + public void Complete(T value) + { + tcs.TrySetResult(new CompletedValue { Value = value }); + } + + public override void Fail(Exception exception) + { + TryFail(exception); + } + + public override void Schedule(Func action) + { + ScheduleCoreAsync(async ct => + { + await action(ct); + return true; + }, + null); + } + + public override void Schedule(Func> action, Action? callback) + { + ScheduleCoreAsync(async ct => + { + var result = await action(ct); + return result; + }, + callback); + } + + private void ScheduleCoreAsync(Func> action, Action? callback) + { + if (IsCompleted) + { + return; + } + + async Task ScheduleAsync() + { + TryStart(); + try + { + // The action must not touch the engine, so that parallel tasks do not block each other. + var result = await action(cancellationToken); + + // The callback converts javascript values and is therefore the only part that needs the lock. + await RunLockedAsync(() => callback?.Invoke(result)); + + TryComplete(); + } + catch (Exception ex) + { + TryFail(ex); + } + } + + ScheduleAsync().Forget(); + } + + private Task RunLockedAsync(Action action) + { + // The lock is owned by the script, because the engine is shared between all executions. + return script.RunLockedAsync(() => + { + // Late callbacks must not touch the engine anymore, the next execution might have started. + if (IsCompleted) + { + return true; + } + + // The task can take a while, therefore the action gets a fresh timeout. + engine.Constraints.Reset(); + + action(); + + // The evaluation does not wait for the promises anymore, therefore the continuations that + // the callback has unblocked have to be executed here, while the lock is still held. + engine.Advanced.ProcessTasks(); + return true; + }, + cancellationToken); + } + + private void TryFail(Exception exception) + { + tcs.TrySetException(exception); + } + + private void TryStart() + { + Interlocked.Increment(ref pendingTasks); + } + + private void TryComplete(CompletedValue? result = null) + { + if (Interlocked.Decrement(ref pendingTasks) <= 0) + { + tcs.TrySetResult(result); + } + } +} diff --git a/backend/src/Squidex.Domain.Apps.Core.Operations/Scripting/Internal/CacheParser.cs b/backend/src/Squidex.Domain.Apps.Core.Operations/Scripting/Internal/CacheParser.cs index 5e731f32c..5d9bb3bb3 100644 --- a/backend/src/Squidex.Domain.Apps.Core.Operations/Scripting/Internal/CacheParser.cs +++ b/backend/src/Squidex.Domain.Apps.Core.Operations/Scripting/Internal/CacheParser.cs @@ -17,7 +17,9 @@ internal sealed class CacheParser(IMemoryCache cache) public Prepared