Tags field normalizer with tests.

src/Squidex.Domain.Apps.Core.Model/Schemas/TagsFieldNormalization.cs

@@ -0,0 +1,15 @@
+// ==========================================================================
+//  Squidex Headless CMS
+// ==========================================================================
+//  Copyright (c) Squidex UG (haftungsbeschraenkt)
+//  All rights reserved. Licensed under the MIT license.
+// ==========================================================================
+
+namespace Squidex.Domain.Apps.Core.Schemas
+{
+    public enum TagsFieldNormalization
+    {
+        None,
+        Schema
+    }
+}

src/Squidex.Domain.Apps.Core.Model/Schemas/TagsFieldProperties.cs

@@ -16,7 +16,7 @@ namespace Squidex.Domain.Apps.Core.Schemas
 
         public int? MaxItems { get; set; }
 
-        public bool Normalize { get; set; }
+        public TagsFieldNormalization Normalization { get; set; }
 
         public override T Accept<T>(IFieldPropertiesVisitor<T> visitor)
         {

src/Squidex.Domain.Apps.Core.Operations/Tags/TagNormalizer.cs

@@ -11,34 +11,53 @@ using System.Threading.Tasks;
 using Newtonsoft.Json.Linq;
 using Squidex.Domain.Apps.Core.Contents;
 using Squidex.Domain.Apps.Core.Schemas;
+using Squidex.Infrastructure;
 
 namespace Squidex.Domain.Apps.Core.Tags
 {
     public static class TagNormalizer
     {
-        public static async Task NormalizeAsync(ITagService service, Guid appId, Guid schemaId, Schema schema, params NamedContentData[] datas)
+        public static async Task NormalizeAsync(this ITagService tagService, Guid appId, Guid schemaId, Schema schema, NamedContentData newData, NamedContentData oldData)
         {
-            var tagsValues = new HashSet<string>();
-            var tagsArrays = new List<JArray>();
+            Guard.NotNull(tagService, nameof(tagService));
+            Guard.NotNull(schema, nameof(schema));
+            Guard.NotNull(newData, nameof(newData));
 
-            GetValues(schema, tagsValues, tagsArrays, datas);
+            var newValues = new HashSet<string>();
+            var newArrays = new List<JArray>();
 
-            if (tagsValues.Count > 0)
+            var oldValues = new HashSet<string>();
+            var oldArrays = new List<JArray>();
+
+            GetValues(schema, newValues, newArrays, newData);
+
+            if (oldData != null)
             {
-                var normalized = await service.NormalizeTagsAsync(appId, $"Schemas_{schemaId}", tagsValues, null);
+                GetValues(schema, oldValues, oldArrays, oldData);
+            }
 
-                foreach (var array in tagsArrays)
+            if (newValues.Count > 0)
+            {
+                var normalized = await tagService.NormalizeTagsAsync(appId, TagGroups.Schemas(schemaId), newValues, oldValues);
+
+                foreach (var array in newArrays)
                 {
                     for (var i = 0; i < array.Count; i++)
                     {
-                        array[i] = normalized[array[i].ToString()];
+                        if (normalized.TryGetValue(array[i].ToString(), out var result))
+                        {
+                            array[i] = result;
+                        }
                     }
                 }
             }
         }
 
-        public static async Task DeNormalizeAsync(ITagService service, Guid appId, Guid schemaId, Schema schema, params NamedContentData[] datas)
+        public static async Task DenormalizeAsync(this ITagService tagService, Guid appId, Guid schemaId, Schema schema, params NamedContentData[] datas)
         {
+            Guard.NotNull(tagService, nameof(tagService));
+            Guard.NotNull(schema, nameof(schema));
+
             var tagsValues = new HashSet<string>();
             var tagsArrays = new List<JArray>();
 
@@ -46,13 +65,16 @@ namespace Squidex.Domain.Apps.Core.Tags
 
             if (tagsValues.Count > 0)
             {
-                var denormalized = await service.DenormalizeTagsAsync(appId, $"Schemas_{schemaId}", tagsValues);
+                var denormalized = await tagService.DenormalizeTagsAsync(appId, TagGroups.Schemas(schemaId), tagsValues);
 
                 foreach (var array in tagsArrays)
                 {
                     for (var i = 0; i < array.Count; i++)
                     {
-                        array[i] = denormalized[array[i].ToString()];
+                        if (denormalized.TryGetValue(array[i].ToString(), out var result))
+                        {
+                            array[i] = result;
+                        }
                     }
                 }
             }
@@ -62,7 +84,7 @@ namespace Squidex.Domain.Apps.Core.Tags
         {
             foreach (var field in schema.Fields)
             {
-                if (field.RawProperties is TagsFieldProperties tags && tags.Normalize)
+                if (field is IField<TagsFieldProperties> tags && tags.Properties.Normalization == TagsFieldNormalization.Schema)
                 {
                     foreach (var data in datas)
                     {
@@ -79,7 +101,7 @@ namespace Squidex.Domain.Apps.Core.Tags
                 {
                     foreach (var nestedField in arrayField.Fields)
                     {
-                        if (field.RawProperties is TagsFieldProperties nestedTags && nestedTags.Normalize)
+                        if (nestedField is IField<TagsFieldProperties> nestedTags && nestedTags.Properties.Normalization == TagsFieldNormalization.Schema)
                         {
                             foreach (var data in datas)
                             {
@@ -95,9 +117,9 @@ namespace Squidex.Domain.Apps.Core.Tags
                                                 {
                                                     var nestedObject = (JObject)value;
 
-                                                    if (nestedObject.TryGetValue(nestedField.Name, out _))
+                                                    if (nestedObject.TryGetValue(nestedField.Name, out var nestedValue))
                                                     {
-                                                        ExtractTags(partition.Value, values, arrays);
+                                                        ExtractTags(nestedValue, values, arrays);
                                                     }
                                                 }
                                             }

src/Squidex.Domain.Apps.Entities/Contents/ContentQueryService.cs

@@ -126,12 +126,12 @@ namespace Squidex.Domain.Apps.Entities.Contents
 
         private IContentEntity Transform(QueryContext context, ISchemaEntity schema, bool checkType, IContentEntity content)
         {
-            return Transform(context, schema, checkType, Enumerable.Repeat(content, 1)).FirstOrDefault();
+            return TansformCore(context, schema, checkType, Enumerable.Repeat(content, 1)).FirstOrDefault();
         }
 
         private IResultList<IContentEntity> Transform(QueryContext context, ISchemaEntity schema, bool checkType, IResultList<IContentEntity> contents)
         {
-            var transformed = Transform(context, schema, checkType, (IEnumerable<IContentEntity>)contents);
+            var transformed = TansformCore(context, schema, checkType, contents);
 
             return ResultList.Create(contents.Total, transformed);
         }
@@ -143,7 +143,7 @@ namespace Squidex.Domain.Apps.Entities.Contents
             return ResultList.Create(contents.Total, sorted);
         }
 
-        private IEnumerable<IContentEntity> Transform(QueryContext context, ISchemaEntity schema, bool checkType, IEnumerable<IContentEntity> contents)
+        private IEnumerable<IContentEntity> TansformCore(QueryContext context, ISchemaEntity schema, bool checkType, IEnumerable<IContentEntity> contents)
         {
             using (Profiler.TraceMethod<ContentQueryService>())
             {

src/Squidex.Domain.Apps.Entities/Contents/Queries/FilterTagTransformer.cs

@@ -59,7 +59,9 @@ namespace Squidex.Domain.Apps.Entities.Contents.Queries
 
         private bool IsTagField(IReadOnlyList<string> path)
         {
-            return schema.SchemaDef.FieldsByName.TryGetValue(path[1], out var field) && field is IField<TagsFieldProperties> tags && tags.Properties.Normalize;
+            return schema.SchemaDef.FieldsByName.TryGetValue(path[1], out var field) &&
+                field is IField<TagsFieldProperties> fieldTags &&
+                fieldTags.Properties.Normalization == TagsFieldNormalization.Schema;
         }
     }
 }

tests/Squidex.Domain.Apps.Core.Tests/Operations/Tags/TagNormalizerTests.cs

@@ -0,0 +1,127 @@
+// ==========================================================================
+//  Squidex Headless CMS
+// ==========================================================================
+//  Copyright (c) Squidex UG (haftungsbeschraenkt)
+//  All rights reserved. Licensed under the MIT license.
+// ==========================================================================
+
+using System;
+using System.Collections.Generic;
+using System.Threading.Tasks;
+using FakeItEasy;
+using Newtonsoft.Json.Linq;
+using Squidex.Domain.Apps.Core.Contents;
+using Squidex.Domain.Apps.Core.Schemas;
+using Squidex.Domain.Apps.Core.Tags;
+using Xunit;
+
+namespace Squidex.Domain.Apps.Core.Operations.Tags
+{
+    public class TagNormalizerTests
+    {
+        private static readonly JTokenEqualityComparer JTokenEqualityComparer = new JTokenEqualityComparer();
+        private readonly ITagService tagService = A.Fake<ITagService>();
+        private readonly Guid appId = Guid.NewGuid();
+        private readonly Guid schemaId = Guid.NewGuid();
+        private readonly Schema schema;
+
+        public TagNormalizerTests()
+        {
+            schema =
+                new Schema("my-schema")
+                    .AddTags(1, "tags1", Partitioning.Invariant)
+                    .AddTags(2, "tags2", Partitioning.Invariant, new TagsFieldProperties { Normalization = TagsFieldNormalization.Schema })
+                    .AddString(3, "string", Partitioning.Invariant)
+                    .AddArray(4, "array", Partitioning.Invariant, f => f
+                        .AddTags(401, "nestedTags1")
+                        .AddTags(402, "nestedTags2", new TagsFieldProperties { Normalization = TagsFieldNormalization.Schema })
+                        .AddString(403, "string"));
+        }
+
+        [Fact]
+        public async Task Should_normalize_tags_with_old_data()
+        {
+            var newData = GenerateData("n_raw");
+            var oldData = GenerateData("o_raw");
+
+            A.CallTo(() => tagService.NormalizeTagsAsync(appId, TagGroups.Schemas(schemaId),
+                A<HashSet<string>>.That.IsSameSequenceAs("n_raw2_1", "n_raw2_2", "n_raw4"),
+                A<HashSet<string>>.That.IsSameSequenceAs("o_raw2_1", "o_raw2_2", "o_raw4")))
+                .Returns(new Dictionary<string, string>
+                {
+                    ["n_raw2_2"] = "id2_2",
+                    ["n_raw2_1"] = "id2_1",
+                    ["n_raw4"] = "id4"
+                });
+
+            await tagService.NormalizeAsync(appId, schemaId, schema, newData, oldData);
+
+            Assert.Equal(new JArray("id2_1", "id2_2"), newData["tags2"]["iv"], JTokenEqualityComparer);
+            Assert.Equal(new JArray("id4"), newData["array"]["iv"][0]["nestedTags2"], JTokenEqualityComparer);
+        }
+
+        [Fact]
+        public async Task Should_normalize_tags_without_old_data()
+        {
+            var newData = GenerateData("name");
+
+            A.CallTo(() => tagService.NormalizeTagsAsync(appId, TagGroups.Schemas(schemaId),
+                A<HashSet<string>>.That.IsSameSequenceAs("name2_1", "name2_2", "name4"),
+                A<HashSet<string>>.That.IsEmpty()))
+                .Returns(new Dictionary<string, string>
+                {
+                    ["name2_2"] = "id2_2",
+                    ["name2_1"] = "id2_1",
+                    ["name4"] = "id4"
+                });
+
+            await tagService.NormalizeAsync(appId, schemaId, schema, newData, null);
+
+            Assert.Equal(new JArray("id2_1", "id2_2"), newData["tags2"]["iv"], JTokenEqualityComparer);
+            Assert.Equal(new JArray("id4"), newData["array"]["iv"][0]["nestedTags2"], JTokenEqualityComparer);
+        }
+
+        [Fact]
+        public async Task Should_denormalize_tags()
+        {
+            var newData = GenerateData("id");
+
+            A.CallTo(() => tagService.NormalizeTagsAsync(appId, TagGroups.Schemas(schemaId),
+                A<HashSet<string>>.That.IsSameSequenceAs("id2_1", "id2_2", "id4"),
+                A<HashSet<string>>.That.IsEmpty()))
+                .Returns(new Dictionary<string, string>
+                {
+                    ["id2_2"] = "name2_2",
+                    ["id2_1"] = "name2_1",
+                    ["id4"] = "name4"
+                });
+
+            await tagService.NormalizeAsync(appId, schemaId, schema, newData, null);
+
+            Assert.Equal(new JArray("name2_1", "name2_2"), newData["tags2"]["iv"], JTokenEqualityComparer);
+            Assert.Equal(new JArray("name4"), newData["array"]["iv"][0]["nestedTags2"], JTokenEqualityComparer);
+        }
+
+        private static NamedContentData GenerateData(string prefix)
+        {
+            return new NamedContentData()
+                .AddField("tags1",
+                    new ContentFieldData()
+                        .AddValue("iv", new JArray($"{prefix}1")))
+                .AddField("tags2",
+                    new ContentFieldData()
+                        .AddValue("iv", new JArray($"{prefix}2_1", $"{prefix}2_2")))
+                .AddField("string",
+                    new ContentFieldData()
+                        .AddValue("iv", $"{prefix}stringValue"))
+                .AddField("array",
+                    new ContentFieldData()
+                        .AddValue("iv",
+                            new JArray(
+                                new JObject(
+                                    new JProperty("nestedTags1", new JArray($"{prefix}3")),
+                                    new JProperty("nestedTags2", new JArray($"{prefix}4")),
+                                    new JProperty("string", $"{prefix}nestedStringValue")))));
+        }
+    }
+}

tests/Squidex.Domain.Apps.Entities.Tests/Assets/Queries/FilterTagTransformerTests.cs

@@ -22,25 +22,25 @@ namespace Squidex.Domain.Apps.Entities.Assets.Queries
         [Fact]
         public void Should_normalize_tags()
         {
-            A.CallTo(() => tagService.GetTagIdsAsync(appId, TagGroups.Assets, A<HashSet<string>>.That.Contains("tag1")))
-                .Returns(new Dictionary<string, string> { ["tag1"] = "normalized1" });
+            A.CallTo(() => tagService.GetTagIdsAsync(appId, TagGroups.Assets, A<HashSet<string>>.That.Contains("name1")))
+                .Returns(new Dictionary<string, string> { ["name1"] = "id1" });
 
-            var source = FilterBuilder.Eq("tags", "tag1");
+            var source = FilterBuilder.Eq("tags", "name1");
             var result = FilterTagTransformer.Transform(source, appId, tagService);
 
-            Assert.Equal("tags == normalized1", result.ToString());
+            Assert.Equal("tags == id1", result.ToString());
         }
 
         [Fact]
         public void Should_not_fail_when_tags_not_found()
         {
-            A.CallTo(() => tagService.GetTagIdsAsync(appId, TagGroups.Assets, A<HashSet<string>>.That.Contains("tag1")))
+            A.CallTo(() => tagService.GetTagIdsAsync(appId, TagGroups.Assets, A<HashSet<string>>.That.Contains("name1")))
                 .Returns(new Dictionary<string, string>());
 
-            var source = FilterBuilder.Eq("tags", "tag1");
+            var source = FilterBuilder.Eq("tags", "name1");
             var result = FilterTagTransformer.Transform(source, appId, tagService);
 
-            Assert.Equal("tags == tag1", result.ToString());
+            Assert.Equal("tags == name1", result.ToString());
         }
 
         [Fact]

tests/Squidex.Domain.Apps.Entities.Tests/Contents/Queries/FilterTagTransformerTests.cs

@@ -29,7 +29,7 @@ namespace Squidex.Domain.Apps.Entities.Contents.Queries
             var schemaDef =
                 new Schema("schema")
                     .AddTags(1, "tags1", Partitioning.Invariant)
-                    .AddTags(2, "tags2", Partitioning.Invariant, new TagsFieldProperties { Normalize = true })
+                    .AddTags(2, "tags2", Partitioning.Invariant, new TagsFieldProperties { Normalization = TagsFieldNormalization.Schema })
                     .AddString(3, "string", Partitioning.Invariant);
 
             A.CallTo(() => schema.Id).Returns(schemaId);
@@ -39,25 +39,25 @@ namespace Squidex.Domain.Apps.Entities.Contents.Queries
         [Fact]
         public void Should_normalize_tags()
         {
-            A.CallTo(() => tagService.GetTagIdsAsync(appId, TagGroups.Schemas(schemaId), A<HashSet<string>>.That.Contains("tag1")))
-                .Returns(new Dictionary<string, string> { ["tag1"] = "normalized1" });
+            A.CallTo(() => tagService.GetTagIdsAsync(appId, TagGroups.Schemas(schemaId), A<HashSet<string>>.That.Contains("name1")))
+                .Returns(new Dictionary<string, string> { ["name1"] = "id1" });
 
-            var source = FilterBuilder.Eq("data.tags2.iv", "tag1");
+            var source = FilterBuilder.Eq("data.tags2.iv", "name1");
             var result = FilterTagTransformer.Transform(source, appId, schema, tagService);
 
-            Assert.Equal("data.tags2.iv == normalized1", result.ToString());
+            Assert.Equal("data.tags2.iv == id1", result.ToString());
         }
 
         [Fact]
         public void Should_not_fail_when_tags_not_found()
         {
-            A.CallTo(() => tagService.GetTagIdsAsync(appId, TagGroups.Assets, A<HashSet<string>>.That.Contains("tag1")))
+            A.CallTo(() => tagService.GetTagIdsAsync(appId, TagGroups.Assets, A<HashSet<string>>.That.Contains("name1")))
                 .Returns(new Dictionary<string, string>());
 
-            var source = FilterBuilder.Eq("data.tags2.iv", "tag1");
+            var source = FilterBuilder.Eq("data.tags2.iv", "name1");
             var result = FilterTagTransformer.Transform(source, appId, schema, tagService);
 
-            Assert.Equal("data.tags2.iv == tag1", result.ToString());
+            Assert.Equal("data.tags2.iv == name1", result.ToString());
         }
 
         [Fact]

tests/Squidex.Domain.Apps.Entities.Tests/Tags/TagGrainTests.cs

@@ -34,8 +34,8 @@ namespace Squidex.Domain.Apps.Entities.Tags
         [Fact]
         public async Task Should_delete_and_reset_state_when_cleaning()
         {
-            await sut.NormalizeTagsAsync(HashSet.Of("tag1", "tag2"), null);
-            await sut.NormalizeTagsAsync(HashSet.Of("tag2", "tag3"), null);
+            await sut.NormalizeTagsAsync(HashSet.Of("name1", "name2"), null);
+            await sut.NormalizeTagsAsync(HashSet.Of("name2", "name3"), null);
             await sut.ClearAsync();
 
             var allTags = await sut.GetTagsAsync();
@@ -51,9 +51,9 @@ namespace Squidex.Domain.Apps.Entities.Tags
         {
             var tags = new TagSet
             {
-                ["1"] = new Tag { Name = "tag1", Count = 1 },
-                ["2"] = new Tag { Name = "tag2", Count = 2 },
-                ["3"] = new Tag { Name = "tag3", Count = 6 }
+                ["id1"] = new Tag { Name = "name1", Count = 1 },
+                ["id2"] = new Tag { Name = "name2", Count = 2 },
+                ["id3"] = new Tag { Name = "name3", Count = 6 }
             };
 
             await sut.RebuildAsync(tags);
@@ -62,9 +62,9 @@ namespace Squidex.Domain.Apps.Entities.Tags
 
             Assert.Equal(new Dictionary<string, int>
             {
-                ["tag1"] = 1,
-                ["tag2"] = 2,
-                ["tag3"] = 6
+                ["name1"] = 1,
+                ["name2"] = 2,
+                ["name3"] = 6
             }, allTags);
 
             Assert.Same(tags, await sut.GetExportableTagsAsync());
@@ -73,40 +73,40 @@ namespace Squidex.Domain.Apps.Entities.Tags
         [Fact]
         public async Task Should_add_tags_to_grain()
         {
-            await sut.NormalizeTagsAsync(HashSet.Of("tag1", "tag2"), null);
-            await sut.NormalizeTagsAsync(HashSet.Of("tag2", "tag3"), null);
+            await sut.NormalizeTagsAsync(HashSet.Of("name1", "name2"), null);
+            await sut.NormalizeTagsAsync(HashSet.Of("name2", "name3"), null);
 
             var allTags = await sut.GetTagsAsync();
 
             Assert.Equal(new Dictionary<string, int>
             {
-                ["tag1"] = 1,
-                ["tag2"] = 2,
-                ["tag3"] = 1
+                ["name1"] = 1,
+                ["name2"] = 2,
+                ["name3"] = 1
             }, allTags);
         }
 
         [Fact]
         public async Task Should_not_add_tags_if_already_added()
         {
-            var result1 = await sut.NormalizeTagsAsync(HashSet.Of("tag1", "tag2"), null);
-            var result2 = await sut.NormalizeTagsAsync(HashSet.Of("tag1", "tag2", "tag3"), new HashSet<string>(result1.Values));
+            var result1 = await sut.NormalizeTagsAsync(HashSet.Of("name1", "name2"), null);
+            var result2 = await sut.NormalizeTagsAsync(HashSet.Of("name1", "name2", "name3"), new HashSet<string>(result1.Values));
 
             var allTags = await sut.GetTagsAsync();
 
             Assert.Equal(new Dictionary<string, int>
             {
-                ["tag1"] = 1,
-                ["tag2"] = 1,
-                ["tag3"] = 1
+                ["name1"] = 1,
+                ["name2"] = 1,
+                ["name3"] = 1
             }, allTags);
         }
 
         [Fact]
         public async Task Should_remove_tags_from_grain()
         {
-            var result1 = await sut.NormalizeTagsAsync(HashSet.Of("tag1", "tag2"), null);
-            var result2 = await sut.NormalizeTagsAsync(HashSet.Of("tag2", "tag3"), null);
+            var result1 = await sut.NormalizeTagsAsync(HashSet.Of("name1", "name2"), null);
+            var result2 = await sut.NormalizeTagsAsync(HashSet.Of("name2", "name3"), null);
 
             await sut.NormalizeTagsAsync(null, new HashSet<string>(result1.Values));
 
@@ -114,16 +114,16 @@ namespace Squidex.Domain.Apps.Entities.Tags
 
             Assert.Equal(new Dictionary<string, int>
             {
-                ["tag2"] = 1,
-                ["tag3"] = 1
+                ["name2"] = 1,
+                ["name3"] = 1
             }, allTags);
         }
 
         [Fact]
         public async Task Should_resolve_tag_names()
         {
-            var tagIds = await sut.NormalizeTagsAsync(HashSet.Of("tag1", "tag2"), null);
-            var tagNames = await sut.GetTagIdsAsync(HashSet.Of("tag1", "tag2", "invalid1"));
+            var tagIds = await sut.NormalizeTagsAsync(HashSet.Of("name1", "name2"), null);
+            var tagNames = await sut.GetTagIdsAsync(HashSet.Of("name1", "name2", "invalid1"));
 
             Assert.Equal(tagIds, tagNames);
         }