// ========================================================================== // Squidex Headless CMS // ========================================================================== // Copyright (c) Squidex UG (haftungsbeschraenkt) // All rights reserved. Licensed under the MIT license. // ========================================================================== using System.ComponentModel.DataAnnotations; using System.Net.Http.Json; using System.Text.RegularExpressions; using Squidex.Domain.Apps.Core; using Squidex.Domain.Apps.Core.Assets; using Squidex.Domain.Apps.Core.HandleRules; using Squidex.Domain.Apps.Core.Rules.Deprecated; using Squidex.Domain.Apps.Core.Rules.EnrichedEvents; using Squidex.Domain.Apps.Entities; using Squidex.Domain.Apps.Entities.Assets; using Squidex.Domain.Apps.Entities.Assets.Commands; using Squidex.Flows; using Squidex.Infrastructure.Commands; using Squidex.Infrastructure.Reflection; using Squidex.Text; namespace Squidex.Extensions.Actions.DeepDetect; [FlowStep( Title = "DeepDetect", IconImage = "", IconColor = "#526a75", Display = "Annotate image", Description = "Annotate an image using deep detect.")] #pragma warning disable CS0618 // Type or member is obsolete public sealed partial record DeepDetectFlowStep : FlowStep, IConvertibleToAction #pragma warning restore CS0618 // Type or member is obsolete { [Display(Name = "Min Propability", Description = "The minimum probability for objects to be recognized (0 - 100).")] [Editor(FlowStepEditor.Number)] public long MinimumPropability { get; set; } [Display(Name = "Max Tags", Description = "The maximum number of tags to use.")] [Editor(FlowStepEditor.Number)] public long MaximumTags { get; set; } public override async ValueTask ExecuteAsync(FlowExecutionContext executionContext, CancellationToken ct) { var @event = ((FlowEventContext)executionContext.Context).Event; if (@event is not EnrichedAssetEvent assetEvent) { executionContext.LogSkipped("Invalid event."); return Next(); } if (assetEvent.AssetType != AssetType.Image) { executionContext.LogSkipped("Invalid event (not an image)."); return Next(); } if (executionContext.IsSimulation) { executionContext.LogSkipSimulation(); return Next(); } var urlToDownload = executionContext.Resolve() .AssetContent(assetEvent.AppId, assetEvent.Id.ToString(), assetEvent.FileVersion); var httpClient = executionContext.Resolve() .CreateClient("DeepDetect"); var response = await httpClient.PostAsJsonAsync("predict", new { service = "squidexdetector", output = new { best = MaximumTags, confidence_threshold = MinimumPropability / 100d, }, data = new[] { urlToDownload, }, }, ct); var responseBody = await response.Content.ReadAsStringAsync(ct); if (!response.IsSuccessStatusCode) { executionContext.Log($"Failed with status code {response.StatusCode}", responseBody); response.EnsureSuccessStatusCode(); } var jsonResponse = executionContext.DeserializeJson(responseBody); var tags = jsonResponse!.Body.Predictions.SelectMany(x => x.Classes); if (!tags.Any()) { executionContext.Log("Warning: No tags returned.", responseBody); return Next(); } var app = await executionContext.Resolve() .GetAppAsync(assetEvent.AppId.Id, true, ct); if (app == null) { executionContext.LogSkipped("App not found."); return Next(); } var context = Context.Admin(app); var asset = await executionContext.Resolve() .FindAsync(context, assetEvent.Id, ct: ct); if (asset == null) { executionContext.LogSkipped("Asset not found."); return Next(); } var command = new AnnotateAsset { Tags = asset.TagNames, AssetId = asset.Id, AppId = asset.AppId, Actor = assetEvent.Actor, FromRule = true, }; foreach (var tag in tags) { var tagParts = tag.Cat.Split(',')[0].Split(' ', StringSplitOptions.RemoveEmptyEntries); if (IdRegex().IsMatch(tagParts[0])) { tagParts = tagParts.Skip(1).ToArray(); } var tagName = string.Join('_', tagParts.Select(x => x.Slugify())); command.Tags.Add($"ai/{tagName}"); } await executionContext.Resolve() .PublishAsync(command, ct); executionContext.Log("Tags Added.", responseBody); return Next(); } #pragma warning disable CS0618 // Type or member is obsolete public RuleAction ToAction() { return SimpleMapper.Map(this, new DeepDetectAction()); } #pragma warning restore CS0618 // Type or member is obsolete private sealed class DetectResponse { public DetectBody Body { get; set; } } private sealed class DetectBody { public DetectPredications[] Predictions { get; set; } } private sealed class DetectPredications { public DetectClass[] Classes { get; set; } } private sealed class DetectClass { public double Prob { get; set; } public string Cat { get; set; } } [GeneratedRegex("^n[0-9]+$")] private static partial Regex IdRegex(); }