Deep detect action.

backend/extensions/Squidex.Extensions/Actions/Algolia/AlgoliaActionHandler.cs

@@ -40,65 +40,65 @@ public sealed class AlgoliaActionHandler : RuleActionHandler<AlgoliaAction, Algo
 
     protected override async Task<(string Description, AlgoliaJob Data)> CreateJobAsync(EnrichedEvent @event, AlgoliaAction action)
     {
-        if (@event is IEnrichedEntityEvent entityEvent)
+        if (@event is not IEnrichedEntityEvent entityEvent)
         {
-            var delete = @event.ShouldDelete(scriptEngine, action.Delete);
+            return ("Ignore", new AlgoliaJob());
+        }
 
-            var ruleDescription = string.Empty;
-            var contentId = entityEvent.Id.ToString();
-            var content = (AlgoliaContent?)null;
+        var delete = @event.ShouldDelete(scriptEngine, action.Delete);
 
-            if (delete)
-            {
-                ruleDescription = $"Delete entry from Algolia index: {action.IndexName}";
-            }
-            else
-            {
-                ruleDescription = $"Add entry to Algolia index: {action.IndexName}";
+        var ruleDescription = string.Empty;
+        var contentId = entityEvent.Id.ToString();
+        var content = (AlgoliaContent?)null;
 
-                try
-                {
-                    string? jsonString;
+        if (delete)
+        {
+            ruleDescription = $"Delete entry from Algolia index: {action.IndexName}";
+        }
+        else
+        {
+            ruleDescription = $"Add entry to Algolia index: {action.IndexName}";
 
-                    if (!string.IsNullOrEmpty(action.Document))
-                    {
-                        jsonString = await FormatAsync(action.Document, @event);
-                        jsonString = jsonString?.Trim();
-                    }
-                    else
-                    {
-                        jsonString = ToJson(@event);
-                    }
+            try
+            {
+                string? jsonString;
 
-                    content = serializer.Deserialize<AlgoliaContent>(jsonString!);
+                if (!string.IsNullOrEmpty(action.Document))
+                {
+                    jsonString = await FormatAsync(action.Document, @event);
+                    jsonString = jsonString?.Trim();
                 }
-                catch (Exception ex)
+                else
                 {
-                    content = new AlgoliaContent
-                    {
-                        More = new Dictionary<string, object>
-                        {
-                            ["error"] = $"Invalid JSON: {ex.Message}"
-                        }
-                    };
+                    jsonString = ToJson(@event);
                 }
 
-                content.ObjectID = contentId;
+                content = serializer.Deserialize<AlgoliaContent>(jsonString!);
             }
-
-            var ruleJob = new AlgoliaJob
+            catch (Exception ex)
             {
-                AppId = action.AppId,
-                ApiKey = action.ApiKey,
-                Content = serializer.Serialize(content, true),
-                ContentId = contentId,
-                IndexName = (await FormatAsync(action.IndexName, @event))!
-            };
-
-            return (ruleDescription, ruleJob);
+                content = new AlgoliaContent
+                {
+                    More = new Dictionary<string, object>
+                    {
+                        ["error"] = $"Invalid JSON: {ex.Message}"
+                    }
+                };
+            }
+
+            content.ObjectID = contentId;
         }
 
-        return ("Ignore", new AlgoliaJob());
+        var ruleJob = new AlgoliaJob
+        {
+            AppId = action.AppId,
+            ApiKey = action.ApiKey,
+            Content = serializer.Serialize(content, true),
+            ContentId = contentId,
+            IndexName = (await FormatAsync(action.IndexName, @event))!
+        };
+
+        return (ruleDescription, ruleJob);
     }
 
     protected override async Task<Result> ExecuteJobAsync(AlgoliaJob job,

backend/extensions/Squidex.Extensions/Actions/Comment/CommentActionHandler.cs

@@ -26,30 +26,37 @@ public sealed class CommentActionHandler : RuleActionHandler<CommentAction, Crea
 
     protected override async Task<(string Description, CreateComment Data)> CreateJobAsync(EnrichedEvent @event, CommentAction action)
     {
-        if (@event is EnrichedContentEvent contentEvent)
+        if (@event is not EnrichedContentEvent contentEvent)
         {
-            var ruleJob = new CreateComment
-            {
-                AppId = contentEvent.AppId
-            };
+            return ("Ignore", new CreateComment());
+        }
+
+        var ruleJob = new CreateComment
+        {
+            AppId = contentEvent.AppId
+        };
 
-            ruleJob.Text = (await FormatAsync(action.Text, @event))!;
+        var text = await FormatAsync(action.Text, @event);
 
-            if (!string.IsNullOrEmpty(action.Client))
-            {
-                ruleJob.Actor = RefToken.Client(action.Client);
-            }
-            else
-            {
-                ruleJob.Actor = contentEvent.Actor;
-            }
+        if (string.IsNullOrWhiteSpace(text))
+        {
+            return ("NoText", new CreateComment());
+        }
 
-            ruleJob.CommentsId = contentEvent.Id;
+        ruleJob.Text = text;
 
-            return (Description, ruleJob);
+        if (!string.IsNullOrEmpty(action.Client))
+        {
+            ruleJob.Actor = RefToken.Client(action.Client);
+        }
+        else
+        {
+            ruleJob.Actor = contentEvent.Actor;
         }
 
-        return ("Ignore", new CreateComment());
+        ruleJob.CommentsId = contentEvent.Id;
+
+        return (Description, ruleJob);
     }
 
     protected override async Task<Result> ExecuteJobAsync(CreateComment job,

backend/extensions/Squidex.Extensions/Actions/CreateContent/CreateContentActionHandler.cs

@@ -47,6 +47,7 @@ public sealed class CreateContentActionHandler : RuleActionHandler<CreateContent
         }
 
         ruleJob.SchemaId = schema.NamedId();
+        ruleJob.FromRule = true;
 
         var json = await FormatAsync(action.Data, @event);
 
@@ -74,8 +75,6 @@ public sealed class CreateContentActionHandler : RuleActionHandler<CreateContent
     {
         var command = job;
 
-        command.FromRule = true;
-
         await commandBus.PublishAsync(command, ct);
 
         return Result.Success($"Created to: {command.SchemaId.Name}");

backend/extensions/Squidex.Extensions/Actions/DeepDetect/DeepDetectAction.cs

@@ -0,0 +1,29 @@
+// ==========================================================================
+//  Squidex Headless CMS
+// ==========================================================================
+//  Copyright (c) Squidex UG (haftungsbeschraenkt)
+//  All rights reserved. Licensed under the MIT license.
+// ==========================================================================
+
+using System.ComponentModel.DataAnnotations;
+using Squidex.Domain.Apps.Core.HandleRules;
+using Squidex.Domain.Apps.Core.Rules;
+
+namespace Squidex.Extensions.Actions.DeepDetect;
+
+[RuleAction(
+    Title = "DeepDetect",
+    IconImage = "<svg viewBox='0 0 28 28' xmlns='http://www.w3.org/2000/svg'><g style='stroke-width:1.24962' fill='none'><path fill='#ff5252' d='M13 21.92H0v-8.032h9.386V10.92h3.57v11zm-9.386-4.889v1.702H9.43v-1.702z' style='stroke-width:1.24962' transform='matrix(.78667 0 0 .81405 2.529 2.668)'/><path fill='#fff' d='M29.164 21.92h-13V14.028H25.7V5.92h3.464zm-9.536-4.804v1.673H25.7v-1.673z' style='stroke-width:1.24962' transform='matrix(.78667 0 0 .81405 2.529 2.668)'/></g></svg>",
+    IconColor = "#526a75",
+    Display = "Annotate image",
+    Description = "Annotate an image using deep detect.")]
+public sealed record DeepDetectAction : RuleAction
+{
+    [Display(Name = "Min Probability", Description = "The minimum probability for objects to be recognized (0 - 100).")]
+    [Editor(RuleFieldEditor.Number)]
+    public long MinimumProbability { get; set; }
+
+    [Display(Name = "Max Tags", Description = "The maximum number of tags to use.")]
+    [Editor(RuleFieldEditor.Number)]
+    public long MaximumTags { get; set; }
+}

backend/extensions/Squidex.Extensions/Actions/DeepDetect/DeepDetectActionHandler.cs

@@ -0,0 +1,197 @@
+// ==========================================================================
+//  Squidex Headless CMS
+// ==========================================================================
+//  Copyright (c) Squidex UG (haftungsbeschraenkt)
+//  All rights reserved. Licensed under the MIT license.
+// ==========================================================================
+
+using System.Net.Http.Json;
+using System.Text.RegularExpressions;
+using Squidex.Domain.Apps.Core;
+using Squidex.Domain.Apps.Core.Assets;
+using Squidex.Domain.Apps.Core.HandleRules;
+using Squidex.Domain.Apps.Core.Rules.EnrichedEvents;
+using Squidex.Domain.Apps.Entities;
+using Squidex.Domain.Apps.Entities.Assets;
+using Squidex.Domain.Apps.Entities.Assets.Commands;
+using Squidex.Infrastructure;
+using Squidex.Infrastructure.Commands;
+using Squidex.Infrastructure.Json;
+using Squidex.Text;
+
+namespace Squidex.Extensions.Actions.DeepDetect;
+
+#pragma warning disable MA0048 // File name must match type name
+
+internal partial class DeepDetectActionHandler : RuleActionHandler<DeepDetectAction, DeepDetectJob>
+{
+    private const string Description = "Analyze Image";
+    private readonly IHttpClientFactory httpClientFactory;
+    private readonly IJsonSerializer jsonSerializer;
+    private readonly IAppProvider appProvider;
+    private readonly IAssetQueryService assetQuery;
+    private readonly ICommandBus commandBus;
+    private readonly IUrlGenerator urlGenerator;
+
+    public DeepDetectActionHandler(RuleEventFormatter formatter, IHttpClientFactory httpClientFactory,
+        IJsonSerializer jsonSerializer,
+        IAppProvider appProvider,
+        IAssetQueryService assetQuery,
+        ICommandBus commandBus,
+        IUrlGenerator urlGenerator)
+        : base(formatter)
+    {
+        this.httpClientFactory = httpClientFactory;
+        this.jsonSerializer = jsonSerializer;
+        this.appProvider = appProvider;
+        this.assetQuery = assetQuery;
+        this.commandBus = commandBus;
+        this.urlGenerator = urlGenerator;
+    }
+
+    protected override Task<(string Description, DeepDetectJob Data)> CreateJobAsync(EnrichedEvent @event, DeepDetectAction action)
+    {
+        if (@event is not EnrichedAssetEvent assetEvent)
+        {
+            return Task.FromResult(("Ignore", new DeepDetectJob()));
+        }
+
+        if (assetEvent.AssetType != AssetType.Image)
+        {
+            return Task.FromResult(("Ignore", new DeepDetectJob()));
+        }
+
+        var ruleJob = new DeepDetectJob
+        {
+            Actor = assetEvent.Actor,
+            AppId = assetEvent.AppId.Id,
+            AssetId = assetEvent.Id,
+            MaximumTags = action.MaximumTags,
+            MinimumPropability = action.MinimumProbability,
+            Url = urlGenerator.AssetContent(assetEvent.AppId, assetEvent.Id.ToString(), assetEvent.FileVersion)
+        };
+
+        return Task.FromResult((Description, ruleJob));
+    }
+
+    protected override async Task<Result> ExecuteJobAsync(DeepDetectJob job,
+        CancellationToken ct = default)
+    {
+        if (string.IsNullOrWhiteSpace(job.Url))
+        {
+            return Result.Ignored();
+        }
+
+        var httpClient = httpClientFactory.CreateClient("DeepDetect");
+
+        var response = await httpClient.PostAsJsonAsync("predict", new
+        {
+            service = "squidexdetector",
+            output = new
+            {
+                best = job.MaximumTags,
+                confidence_threshold = job.MinimumPropability / 100d,
+            },
+            data = new[]
+            {
+                job.Url,
+            }
+        }, ct);
+
+        var body = await response.Content.ReadAsStringAsync(ct);
+
+        if (!response.IsSuccessStatusCode)
+        {
+            return Result.Failed(new InvalidOperationException($"Failed with status code {response.StatusCode}\n\n{body}"));
+        }
+
+        var responseJson = jsonSerializer.Deserialize<DetectResponse>(body);
+
+        var tags = responseJson!.Body.Predictions.SelectMany(x => x.Classes);
+
+        if (!tags.Any())
+        {
+            return Result.Success(body);
+        }
+
+        var app = await appProvider.GetAppAsync(job.AppId, true, ct);
+        if (app == null)
+        {
+            return Result.Failed(new InvalidOperationException("App not found."));
+        }
+
+        var context = Context.Admin(app);
+
+        var asset = await assetQuery.FindAsync(context, job.AssetId, ct: ct);
+        if (asset == null)
+        {
+            return Result.Failed(new InvalidOperationException("Asset not found."));
+        }
+
+        var command = new AnnotateAsset
+        {
+            Tags = asset.TagNames,
+            AssetId = asset.AssetId,
+            AppId = asset.AppId,
+            Actor = job.Actor,
+            FromRule = true
+        };
+
+        foreach (var tag in tags)
+        {
+            var tagParts = tag.Cat.Split(',')[0].Split(' ', StringSplitOptions.RemoveEmptyEntries);
+
+            if (IdRegex().IsMatch(tagParts[0]))
+            {
+                tagParts = tagParts.Skip(1).ToArray();
+            }
+
+            var tagName = string.Join('_', tagParts.Select(x => x.Slugify()));
+
+            command.Tags.Add($"ai/{tagName}");
+        }
+
+        await commandBus.PublishAsync(command, ct);
+        return Result.Success(body);
+    }
+
+    private sealed class DetectResponse
+    {
+        public DetectBody Body { get; set; }
+    }
+
+    private sealed class DetectBody
+    {
+        public DetectPredications[] Predictions { get; set; }
+    }
+
+    private sealed class DetectPredications
+    {
+        public DetectClass[] Classes { get; set; }
+    }
+
+    private sealed class DetectClass
+    {
+        public double Prob { get; set; }
+
+        public string Cat { get; set; }
+    }
+
+    [GeneratedRegex("^n[0-9]+$")]
+    private static partial Regex IdRegex();
+}
+
+public sealed class DeepDetectJob
+{
+    public DomainId AppId { get; set; }
+
+    public DomainId AssetId { get; set; }
+
+    public RefToken Actor { get; set; }
+
+    public long MaximumTags { get; set; }
+
+    public long MinimumPropability { get; set; }
+
+    public string? Url { get; set; }
+}

backend/extensions/Squidex.Extensions/Actions/DeepDetect/DeepDetectPlugin.cs

@@ -0,0 +1,32 @@
+// ==========================================================================
+//  Squidex Headless CMS
+// ==========================================================================
+//  Copyright (c) Squidex UG (haftungsbeschraenkt)
+//  All rights reserved. Licensed under the MIT license.
+// ==========================================================================
+
+using Microsoft.Extensions.Configuration;
+using Microsoft.Extensions.DependencyInjection;
+using Squidex.Infrastructure.Plugins;
+
+namespace Squidex.Extensions.Actions.DeepDetect;
+
+internal class DeepDetectPlugin : IPlugin
+{
+    public void ConfigureServices(IServiceCollection services, IConfiguration config)
+    {
+        var url = config.GetValue<string>("deepdetect:url");
+
+        if (!Uri.TryCreate(url, UriKind.Absolute, out var uri))
+        {
+            return;
+        }
+
+        services.AddHttpClient("DeepDetect", client =>
+        {
+            client.BaseAddress = uri;
+        });
+
+        services.AddRuleAction<DeepDetectAction, DeepDetectActionHandler>();
+    }
+}

backend/extensions/Squidex.Extensions/Actions/Email/EmailActionHandler.cs

@@ -47,7 +47,6 @@ public sealed class EmailActionHandler : RuleActionHandler<EmailAction, EmailJob
         using (var smtpClient = new SmtpClient())
         {
             await smtpClient.ConnectAsync(job.ServerHost, job.ServerPort, cancellationToken: ct);
-
             await smtpClient.AuthenticateAsync(job.ServerUsername, job.ServerPassword, ct);
 
             var smtpMessage = new MimeMessage();

backend/extensions/Squidex.Extensions/Actions/Notification/NotificationActionHandler.cs

@@ -30,46 +30,46 @@ public sealed class NotificationActionHandler : RuleActionHandler<NotificationAc
 
     protected override async Task<(string Description, CreateComment Data)> CreateJobAsync(EnrichedEvent @event, NotificationAction action)
     {
-        if (@event is EnrichedUserEventBase userEvent)
+        if (@event is not EnrichedUserEventBase userEvent)
         {
-            var user = await userResolver.FindByIdOrEmailAsync(action.User);
+            return ("Ignore", new CreateComment());
+        }
 
-            if (user == null)
-            {
-                throw new InvalidOperationException($"Cannot find user by '{action.User}'");
-            }
+        var user = await userResolver.FindByIdOrEmailAsync(action.User);
 
-            var actor = userEvent.Actor;
+        if (user == null)
+        {
+            throw new InvalidOperationException($"Cannot find user by '{action.User}'");
+        }
 
-            if (!string.IsNullOrEmpty(action.Client))
-            {
-                actor = RefToken.Client(action.Client);
-            }
+        var actor = userEvent.Actor;
 
-            var ruleJob = new CreateComment
-            {
-                AppId = CommentsCommand.NoApp,
-                Actor = actor,
-                CommentId = DomainId.NewGuid(),
-                CommentsId = DomainId.Create(user.Id),
-                FromRule = true,
-                Text = (await FormatAsync(action.Text, @event))!
-            };
-
-            if (!string.IsNullOrWhiteSpace(action.Url))
-            {
-                var url = await FormatAsync(action.Url, @event);
+        if (!string.IsNullOrEmpty(action.Client))
+        {
+            actor = RefToken.Client(action.Client);
+        }
 
-                if (Uri.TryCreate(url, UriKind.RelativeOrAbsolute, out var uri))
-                {
-                    ruleJob.Url = uri;
-                }
-            }
+        var ruleJob = new CreateComment
+        {
+            AppId = CommentsCommand.NoApp,
+            Actor = actor,
+            CommentId = DomainId.NewGuid(),
+            CommentsId = DomainId.Create(user.Id),
+            FromRule = true,
+            Text = (await FormatAsync(action.Text, @event))!
+        };
+
+        if (!string.IsNullOrWhiteSpace(action.Url))
+        {
+            var url = await FormatAsync(action.Url, @event);
 
-            return (Description, ruleJob);
+            if (Uri.TryCreate(url, UriKind.RelativeOrAbsolute, out var uri))
+            {
+                ruleJob.Url = uri;
+            }
         }
 
-        return ("Ignore", new CreateComment());
+        return (Description, ruleJob);
     }
 
     protected override async Task<Result> ExecuteJobAsync(CreateComment job,

backend/extensions/Squidex.Extensions/Actions/Prerender/PrerenderActionHandler.cs

@@ -27,8 +27,8 @@ public sealed class PrerenderActionHandler : RuleActionHandler<PrerenderAction,
     {
         var url = await FormatAsync(action.Url, @event);
 
-        var request = new { prerenderToken = action.Token, url };
-        var requestBody = ToJson(request);
+        var requestObject = new { prerenderToken = action.Token, url };
+        var requestBody = ToJson(requestObject);
 
         return ($"Recache {url}", new PrerenderJob { RequestBody = requestBody });
     }

backend/src/Squidex.Domain.Apps.Core.Operations/IUrlGenerator.cs

@@ -22,6 +22,8 @@ public interface IUrlGenerator
 
     string AssetContent(NamedId<DomainId> appId, string idOrSlug);
 
+    string AssetContent(NamedId<DomainId> appId, string idOrSlug, long version);
+
     string AssetContentBase();
 
     string AssetContentBase(string appName);

backend/src/Squidex.Web/Services/UrlGenerator.cs

@@ -67,6 +67,11 @@ public sealed class UrlGenerator : IUrlGenerator
         return urlGenerator.BuildUrl($"api/assets/{appId.Name}/{idOrSlug}");
     }
 
+    public string AssetContent(NamedId<DomainId> appId, string idOrSlug, long version)
+    {
+        return urlGenerator.BuildUrl($"api/assets/{appId.Name}/{idOrSlug}?version={version}");
+    }
+
     public string? AssetSource(NamedId<DomainId> appId, DomainId assetId, long fileVersion)
     {
         return assetFileStore.GeneratePublicUrl(appId.Id, assetId, fileVersion, null);

backend/src/Squidex/Config/Authentication/IdentityServices.cs

@@ -17,7 +17,7 @@ public static class IdentityServices
         services.Configure<MyIdentityOptions>(config,
             "identity");
 
-        services.AddHttpClient("USers");
+        services.AddHttpClient("Users");
 
         services.AddSingletonAs<DefaultUserResolver>()
             .AsOptional<IUserResolver>();

backend/src/Squidex/appsettings.json

@@ -355,11 +355,11 @@
         "name": "Squidex",
 
         "stackdriver": {
-          // True, to enable stackdriver integration.
-          "enabled": false,
+            // True, to enable stackdriver integration.
+            "enabled": false,
 
-          // The ID of your Google Cloud project.
-          "projectId": ""
+            // The ID of your Google Cloud project.
+            "projectId": ""
         },
 
         "otlp": {
@@ -672,11 +672,16 @@
         "version": "squidex:version"
     },
 
-    // Kafka Producer configuration
+    // Kafka producer configuration
     "kafka": {
         "bootstrapServers": ""
     },
 
+    // Deepdetect configuration
+    "deepdetect": {
+        "url": ""
+    },
+
     // The client information for twitter.
     "twitter": {
         "clientId": "QZhb3HQcGCvE6G8yNNP9ksNet",

backend/tests/Squidex.Domain.Apps.Entities.Tests/Contents/TestData/FakeUrlGenerator.cs

@@ -30,6 +30,11 @@ public sealed class FakeUrlGenerator : IUrlGenerator
         return $"assets/{appId.Name}/{idOrSlug}";
     }
 
+    public string AssetContent(NamedId<DomainId> appId, string idOrSlug, long version)
+    {
+        return $"assets/{appId.Name}/{idOrSlug}?version={version}";
+    }
+
     public string ContentUI(NamedId<DomainId> appId, NamedId<DomainId> schemaId, DomainId contentId)
     {
         return $"contents/{schemaId.Name}/{contentId}";