From a0213d5ec02cfc2c01159b98fb23537e9ba23f2a Mon Sep 17 00:00:00 2001 From: Sebastian Date: Tue, 17 Aug 2021 22:08:10 +0200 Subject: [PATCH] Azure metadata source. --- .../Actions/Algolia/AlgoliaActionHandler.cs | 2 +- .../Actions/Kafka/KafkaPlugin.cs | 6 +- .../Assets/Azure/AzureMetadataSource.cs | 100 ++++++++++++++++++ .../Azure/AzureMetadataSourceOptions.cs | 23 ++++ .../Assets/Azure/AzureMetadataSourcePlugin.cs | 30 ++++++ .../Squidex.Extensions.csproj | 1 + 6 files changed, 158 insertions(+), 4 deletions(-) create mode 100644 backend/extensions/Squidex.Extensions/Assets/Azure/AzureMetadataSource.cs create mode 100644 backend/extensions/Squidex.Extensions/Assets/Azure/AzureMetadataSourceOptions.cs create mode 100644 backend/extensions/Squidex.Extensions/Assets/Azure/AzureMetadataSourcePlugin.cs diff --git a/backend/extensions/Squidex.Extensions/Actions/Algolia/AlgoliaActionHandler.cs b/backend/extensions/Squidex.Extensions/Actions/Algolia/AlgoliaActionHandler.cs index 9ff262a78..8c05e4c0a 100644 --- a/backend/extensions/Squidex.Extensions/Actions/Algolia/AlgoliaActionHandler.cs +++ b/backend/extensions/Squidex.Extensions/Actions/Algolia/AlgoliaActionHandler.cs @@ -45,7 +45,7 @@ namespace Squidex.Extensions.Actions.Algolia var ruleDescription = string.Empty; var contentId = entityEvent.Id.ToString(); - var content = (JObject?)null; + var content = (JObject)null; if (delete) { diff --git a/backend/extensions/Squidex.Extensions/Actions/Kafka/KafkaPlugin.cs b/backend/extensions/Squidex.Extensions/Actions/Kafka/KafkaPlugin.cs index 400c0662f..908ad5784 100644 --- a/backend/extensions/Squidex.Extensions/Actions/Kafka/KafkaPlugin.cs +++ b/backend/extensions/Squidex.Extensions/Actions/Kafka/KafkaPlugin.cs @@ -16,14 +16,14 @@ namespace Squidex.Extensions.Actions.Kafka { public void ConfigureServices(IServiceCollection services, IConfiguration config) { - var kafkaOptions = config.GetSection("kafka").Get(); + var options = config.GetSection("kafka").Get(); - if (kafkaOptions.IsProducerConfigured()) + if (options.IsProducerConfigured()) { services.AddRuleAction(); services.AddSingleton(); - services.AddSingleton(Options.Create(kafkaOptions)); + services.AddSingleton(Options.Create(options)); } } } diff --git a/backend/extensions/Squidex.Extensions/Assets/Azure/AzureMetadataSource.cs b/backend/extensions/Squidex.Extensions/Assets/Azure/AzureMetadataSource.cs new file mode 100644 index 000000000..40c6ca9d9 --- /dev/null +++ b/backend/extensions/Squidex.Extensions/Assets/Azure/AzureMetadataSource.cs @@ -0,0 +1,100 @@ +// ========================================================================== +// Squidex Headless CMS +// ========================================================================== +// Copyright (c) Squidex UG (haftungsbeschraenkt) +// All rights reserved. Licensed under the MIT license. +// ========================================================================== + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; +using Microsoft.Azure.CognitiveServices.Vision.ComputerVision; +using Microsoft.Azure.CognitiveServices.Vision.ComputerVision.Models; +using Microsoft.Extensions.Options; +using Squidex.Domain.Apps.Core.Assets; +using Squidex.Domain.Apps.Entities.Assets; +using Squidex.Domain.Apps.Entities.Assets.Commands; +using Squidex.Infrastructure.Json.Objects; +using Squidex.Log; + +namespace Squidex.Extensions.Assets.Azure +{ + public sealed class AzureMetadataSource : IAssetMetadataSource + { + private const long MaxSize = 5 * 1025 * 1024; + private readonly ISemanticLog log; + private readonly ComputerVisionClient client; + private readonly char[] trimChars = + { + ' ', + '_', + '-' + }; + private readonly List features = new List + { + VisualFeatureTypes.Categories, + VisualFeatureTypes.Description, + VisualFeatureTypes.Color + }; + + public int Order => int.MaxValue; + + public AzureMetadataSource(IOptions options, ISemanticLog log) + { + client = new ComputerVisionClient(new ApiKeyServiceClientCredentials(options.Value.ApiKey)) + { + Endpoint = options.Value.Endpoint + }; + + this.log = log; + } + + public async Task EnhanceAsync(UploadAssetCommand command) + { + try + { + if (command.Type == AssetType.Image && command.File.FileSize <= MaxSize) + { + using (var stream = command.File.OpenRead()) + { + var result = await client.AnalyzeImageInStreamAsync(stream, features); + + command.Tags ??= new HashSet(); + + if (result.Color?.DominantColorForeground != null) + { + command.Tags.Add($"color/{result.Color.DominantColorForeground.Trim(trimChars).ToLowerInvariant()}"); + } + + if (result.Categories != null) + { + foreach (var category in result.Categories.OrderByDescending(x => x.Score).Take(3)) + { + command.Tags.Add($"category/{category.Name.Trim(trimChars).ToLowerInvariant()}"); + } + } + + var description = result.Description?.Captions?.OrderByDescending(x => x.Confidence)?.FirstOrDefault()?.Text; + + if (description != null) + { + command.Metadata["caption"] = JsonValue.Create(description); + } + } + } + } + catch (Exception ex) + { + log.LogError(ex, w => w + .WriteProperty("action", "EnrichWithAssure") + .WriteProperty("status", "Failed")); + } + } + + public IEnumerable Format(IAssetEntity asset) + { + yield break; + } + } +} diff --git a/backend/extensions/Squidex.Extensions/Assets/Azure/AzureMetadataSourceOptions.cs b/backend/extensions/Squidex.Extensions/Assets/Azure/AzureMetadataSourceOptions.cs new file mode 100644 index 000000000..2a22caa66 --- /dev/null +++ b/backend/extensions/Squidex.Extensions/Assets/Azure/AzureMetadataSourceOptions.cs @@ -0,0 +1,23 @@ +// ========================================================================== +// Squidex Headless CMS +// ========================================================================== +// Copyright (c) Squidex UG (haftungsbeschraenkt) +// All rights reserved. Licensed under the MIT license. +// ========================================================================== + +namespace Squidex.Extensions.Assets.Azure +{ + public sealed class AzureMetataSourceOptions + { + public string Endpoint { get; set; } + + public string ApiKey { get; set; } + + public bool IsConfigured() + { + return + !string.IsNullOrWhiteSpace(Endpoint) && + !string.IsNullOrWhiteSpace(ApiKey); + } + } +} diff --git a/backend/extensions/Squidex.Extensions/Assets/Azure/AzureMetadataSourcePlugin.cs b/backend/extensions/Squidex.Extensions/Assets/Azure/AzureMetadataSourcePlugin.cs new file mode 100644 index 000000000..b6abf7985 --- /dev/null +++ b/backend/extensions/Squidex.Extensions/Assets/Azure/AzureMetadataSourcePlugin.cs @@ -0,0 +1,30 @@ +// ========================================================================== +// Squidex Headless CMS +// ========================================================================== +// Copyright (c) Squidex UG (haftungsbeschraenkt) +// All rights reserved. Licensed under the MIT license. +// ========================================================================== + +using System; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Options; +using Squidex.Domain.Apps.Entities.Assets; +using Squidex.Infrastructure.Plugins; + +namespace Squidex.Extensions.Assets.Azure +{ + public sealed class AzureMetadataSourcePlugin : IPlugin + { + public void ConfigureServices(IServiceCollection services, IConfiguration config) + { + var options = config.GetSection("assets:azureCognitive").Get(); + + if (options.IsConfigured()) + { + services.AddSingleton(); + services.AddSingleton(Options.Create(options)); + } + } + } +} diff --git a/backend/extensions/Squidex.Extensions/Squidex.Extensions.csproj b/backend/extensions/Squidex.Extensions/Squidex.Extensions.csproj index 9550a15fb..580759a0b 100644 --- a/backend/extensions/Squidex.Extensions/Squidex.Extensions.csproj +++ b/backend/extensions/Squidex.Extensions/Squidex.Extensions.csproj @@ -16,6 +16,7 @@ +