diff --git a/backend/extensions/Squidex.Extensions/Text/Azure/AzureIndexDefinition.cs b/backend/extensions/Squidex.Extensions/Text/Azure/AzureIndexDefinition.cs index 4ea7ddc2d..d7bfde33d 100644 --- a/backend/extensions/Squidex.Extensions/Text/Azure/AzureIndexDefinition.cs +++ b/backend/extensions/Squidex.Extensions/Text/Azure/AzureIndexDefinition.cs @@ -15,7 +15,7 @@ namespace Squidex.Extensions.Text.Azure { public static class AzureIndexDefinition { - private static readonly Dictionary AllowedLanguages = new Dictionary(StringComparer.OrdinalIgnoreCase) + private static readonly Dictionary FieldAnalyzers = new (StringComparer.OrdinalIgnoreCase) { ["iv"] = ("iv", LexicalAnalyzerName.StandardLucene.ToString()), ["zh"] = ("zh", LexicalAnalyzerName.ZhHansLucene.ToString()) @@ -49,25 +49,30 @@ namespace Squidex.Extensions.Text.Azure { var fieldName = language.Replace('-', '_'); - AllowedLanguages[language] = (fieldName, analyzer); + FieldAnalyzers[language] = (fieldName, analyzer); } } } } - public static string GetTextField(string key) + public static string GetFieldName(string key) { - if (AllowedLanguages.TryGetValue(key, out var field)) + if (FieldAnalyzers.TryGetValue(key, out var analyzer)) { - return field.Field; + return analyzer.Field; } - if (key.Length > 2 && AllowedLanguages.TryGetValue(key[2..], out field)) + if (key.Length > 0) { - return field.Field; + var language = key[2..]; + + if (FieldAnalyzers.TryGetValue(language, out analyzer)) + { + return analyzer.Field; + } } - return AllowedLanguages["iv"].Field; + return "iv"; } public static SearchIndex Create(string indexName) @@ -103,12 +108,20 @@ namespace Squidex.Extensions.Text.Azure IsFilterable = true }, new SimpleField("servePublished", SearchFieldDataType.Boolean) + { + IsFilterable = true + }, + new SimpleField("geoObject", SearchFieldDataType.GeographyPoint) + { + IsFilterable = true + }, + new SimpleField("geoField", SearchFieldDataType.String) { IsFilterable = true } }; - foreach (var (field, analyzer) in AllowedLanguages.Values) + foreach (var (field, analyzer) in FieldAnalyzers.Values) { fields.Add( new SearchableField(field) diff --git a/backend/extensions/Squidex.Extensions/Text/Azure/AzureTextIndex.cs b/backend/extensions/Squidex.Extensions/Text/Azure/AzureTextIndex.cs index bc745ecff..120f5ec70 100644 --- a/backend/extensions/Squidex.Extensions/Text/Azure/AzureTextIndex.cs +++ b/backend/extensions/Squidex.Extensions/Text/Azure/AzureTextIndex.cs @@ -26,19 +26,16 @@ namespace Squidex.Extensions.Text.Azure { private readonly SearchIndexClient indexClient; private readonly SearchClient searchClient; - private readonly int waitAfterUpdate; + private readonly QueryParser queryParser = new QueryParser(AzureIndexDefinition.GetFieldName); public AzureTextIndex( string serviceEndpoint, string serviceApiKey, - string indexName, - int waitAfterUpdate = 0) + string indexName) { indexClient = new SearchIndexClient(new Uri(serviceEndpoint), new AzureKeyCredential(serviceApiKey)); searchClient = indexClient.GetSearchClient(indexName); - - this.waitAfterUpdate = waitAfterUpdate; } public async Task InitializeAsync( @@ -76,20 +73,19 @@ namespace Squidex.Extensions.Text.Azure } await searchClient.IndexDocumentsAsync(batch, cancellationToken: ct); - - if (waitAfterUpdate > 0) - { - await Task.Delay(waitAfterUpdate, ct); - } } - public Task> SearchAsync(IAppEntity app, GeoQuery query, SearchScope scope, + public async Task> SearchAsync(IAppEntity app, GeoQuery query, SearchScope scope, CancellationToken ct = default) { Guard.NotNull(app, nameof(app)); Guard.NotNull(query, nameof(query)); - return Task.FromResult>(null); + var result = new List<(DomainId Id, double Score)>(); + + await SearchAsync(result, "*", BuildGeoQuery(query, scope), query.Take, 1, ct); + + return result.OrderByDescending(x => x.Score).Select(x => x.Id).Distinct().ToList(); } public async Task> SearchAsync(IAppEntity app, TextQuery query, SearchScope scope, @@ -98,57 +94,57 @@ namespace Squidex.Extensions.Text.Azure Guard.NotNull(app, nameof(app)); Guard.NotNull(query, nameof(query)); - if (string.IsNullOrWhiteSpace(query.Text)) + var parsed = queryParser.Parse(query.Text); + + if (parsed == null) { return null; } - List<(DomainId, double)> documents; + var result = new List<(DomainId Id, double Score)>(); if (query.RequiredSchemaIds?.Count > 0) { - documents = await SearchBySchemaAsync(query.Text, query.RequiredSchemaIds, scope, query.Take, 1, ct); + await SearchBySchemaAsync(result, parsed.Text, query.RequiredSchemaIds, scope, query.Take, 1, ct); } else if (query.PreferredSchemaId == null) { - documents = await SearchByAppAsync(query.Text, app, scope, query.Take, 1, ct); + await SearchByAppAsync(result, parsed.Text, app, scope, query.Take, 1, ct); } else { - var halfBucket = query.Take / 2; + var halfTake = query.Take / 2; var schemaIds = Enumerable.Repeat(query.PreferredSchemaId.Value, 1); - documents = await SearchBySchemaAsync( - query.Text, - schemaIds, - scope, - halfBucket, 1, - ct); - - documents.AddRange(await SearchByAppAsync(query.Text, app, scope, halfBucket, 1, ct)); + await SearchBySchemaAsync(result, parsed.Text, schemaIds, scope, halfTake, 1.1, ct); + await SearchByAppAsync(result, parsed.Text, app, scope, halfTake, 1, ct); } - return documents.OrderByDescending(x => x.Item2).Select(x => x.Item1).Distinct().ToList(); + return result.OrderByDescending(x => x.Score).Select(x => x.Id).Distinct().ToList(); } - private Task> SearchBySchemaAsync(string search, IEnumerable schemaIds, SearchScope scope, int limit, double factor, + private Task SearchBySchemaAsync(List<(DomainId, double)> result, string text, IEnumerable schemaIds, SearchScope scope, int take, double factor, CancellationToken ct = default) { - var filter = $"{string.Join(" or ", schemaIds.Select(x => $"schemaId eq '{x}'"))} and {GetServeField(scope)} eq true"; + var searchField = GetServeField(scope); - return SearchAsync(search, filter, limit, factor, ct); + var filter = $"{string.Join(" or ", schemaIds.Select(x => $"schemaId eq '{x}'"))} and {searchField} eq true"; + + return SearchAsync(result, text, filter, take, factor, ct); } - private Task> SearchByAppAsync(string search, IAppEntity app, SearchScope scope, int limit, double factor, + private Task SearchByAppAsync(List<(DomainId, double)> result, string text, IAppEntity app, SearchScope scope, int take, double factor, CancellationToken ct = default) { - var filter = $"appId eq '{app.Id}' and {GetServeField(scope)} eq true"; + var searchField = GetServeField(scope); + + var filter = $"appId eq '{app.Id}' and {searchField} eq true"; - return SearchAsync(search, filter, limit, factor, ct); + return SearchAsync(result, text, filter, take, factor, ct); } - private async Task> SearchAsync(string search, string filter, int size, double factor, + private async Task SearchAsync(List<(DomainId, double)> result, string text, string filter, int take, double factor, CancellationToken ct = default) { var searchOptions = new SearchOptions @@ -157,22 +153,30 @@ namespace Squidex.Extensions.Text.Azure }; searchOptions.Select.Add("contentId"); - searchOptions.Size = size; + searchOptions.Size = take; searchOptions.QueryType = SearchQueryType.Full; - var results = await searchClient.SearchAsync(search, searchOptions, ct); - - var ids = new List<(DomainId, double)>(); + var results = await searchClient.SearchAsync(text, searchOptions, ct); await foreach (var item in results.Value.GetResultsAsync().WithCancellation(ct)) { if (item != null) { - ids.Add((DomainId.Create(item.Document["contentId"].ToString()), factor * item.Score ?? 0)); + var id = DomainId.Create(item.Document["contentId"].ToString()); + + result.Add((id, factor * item.Score ?? 0)); } } + } + + private static string BuildGeoQuery(GeoQuery query, SearchScope scope) + { + var (schema, field, lat, lng, radius, _) = query; + + var searchField = GetServeField(scope); + var searchDistance = radius / 1000; - return ids; + return $"schemaId eq '{schema}' and geoField eq '{field}' and geo.distance(geoObject, geography'POINT({lng} {lat})') lt {searchDistance} and {searchField} eq true"; } private static string GetServeField(SearchScope scope) diff --git a/backend/extensions/Squidex.Extensions/Text/Azure/CommandFactory.cs b/backend/extensions/Squidex.Extensions/Text/Azure/CommandFactory.cs index c833772f5..97b0a30d8 100644 --- a/backend/extensions/Squidex.Extensions/Text/Azure/CommandFactory.cs +++ b/backend/extensions/Squidex.Extensions/Text/Azure/CommandFactory.cs @@ -9,6 +9,7 @@ using System; using System.Collections.Generic; using System.Text; using Azure.Search.Documents.Models; +using GeoJSON.Net.Geometry; using Squidex.Domain.Apps.Entities.Contents.Text; namespace Squidex.Extensions.Text.Azure @@ -20,57 +21,92 @@ namespace Squidex.Extensions.Text.Azure switch (command) { case UpsertIndexEntry upsert: - batch.Add(UpsertEntry(upsert)); + UpsertTextEntry(upsert, batch); break; case UpdateIndexEntry update: - batch.Add(UpdateEntry(update)); + UpdateEntry(update, batch); break; case DeleteIndexEntry delete: - batch.Add(DeleteEntry(delete)); + DeleteEntry(delete, batch); break; } } - private static IndexDocumentsAction UpsertEntry(UpsertIndexEntry upsert) + private static void UpsertTextEntry(UpsertIndexEntry upsert, IList> batch) { - var searchDocument = new SearchDocument + var geoField = string.Empty; + var geoObject = (object)null; + + if (upsert.GeoObjects != null) { - ["docId"] = upsert.DocId.ToBase64(), - ["appId"] = upsert.AppId.Id.ToString(), - ["appName"] = upsert.AppId.Name, - ["contentId"] = upsert.ContentId.ToString(), - ["schemaId"] = upsert.SchemaId.Id.ToString(), - ["schemaName"] = upsert.SchemaId.Name, - ["serveAll"] = upsert.ServeAll, - ["servePublished"] = upsert.ServePublished - }; + foreach (var (key, value) in upsert.GeoObjects) + { + if (value is Point point) + { + geoField = key; + geoObject = new + { + type = "Point", + coordinates = new[] + { + point.Coordinates.Longitude, + point.Coordinates.Latitude + } + }; + break; + } + } + } - if (upsert.Texts != null) + if (upsert.Texts != null || geoObject != null) { + var document = new SearchDocument + { + ["docId"] = upsert.DocId.ToBase64(), + ["appId"] = upsert.AppId.Id.ToString(), + ["appName"] = upsert.AppId.Name, + ["contentId"] = upsert.ContentId.ToString(), + ["schemaId"] = upsert.SchemaId.Id.ToString(), + ["schemaName"] = upsert.SchemaId.Name, + ["serveAll"] = upsert.ServeAll, + ["servePublished"] = upsert.ServePublished, + ["geoField"] = geoField, + ["geoObject"] = geoObject + }; + foreach (var (key, value) in upsert.Texts) { - searchDocument[AzureIndexDefinition.GetTextField(key)] = value; + var text = value; + + var languageCode = AzureIndexDefinition.GetFieldName(key); + + if (document.TryGetValue(languageCode, out var existing)) + { + text = $"{existing} {value}"; + } + + document[languageCode] = text; } - } - return IndexDocumentsAction.MergeOrUpload(searchDocument); + batch.Add(IndexDocumentsAction.MergeOrUpload(document)); + } } - private static IndexDocumentsAction UpdateEntry(UpdateIndexEntry update) + private static void UpdateEntry(UpdateIndexEntry update, IList> batch) { - var searchDocument = new SearchDocument + var document = new SearchDocument { ["docId"] = update.DocId.ToBase64(), ["serveAll"] = update.ServeAll, ["servePublished"] = update.ServePublished, }; - return IndexDocumentsAction.MergeOrUpload(searchDocument); + batch.Add(IndexDocumentsAction.MergeOrUpload(document)); } - private static IndexDocumentsAction DeleteEntry(DeleteIndexEntry delete) + private static void DeleteEntry(DeleteIndexEntry delete, IList> batch) { - return IndexDocumentsAction.Delete("docId", delete.DocId.ToBase64()); + batch.Add(IndexDocumentsAction.Delete("docId", delete.DocId.ToBase64())); } private static string ToBase64(this string value) diff --git a/backend/extensions/Squidex.Extensions/Text/ElasticSearch/CommandFactory.cs b/backend/extensions/Squidex.Extensions/Text/ElasticSearch/CommandFactory.cs index 1a1b2fedc..a4271c375 100644 --- a/backend/extensions/Squidex.Extensions/Text/ElasticSearch/CommandFactory.cs +++ b/backend/extensions/Squidex.Extensions/Text/ElasticSearch/CommandFactory.cs @@ -6,6 +6,7 @@ // ========================================================================== using System.Collections.Generic; +using GeoJSON.Net.Geometry; using Squidex.Domain.Apps.Entities.Contents.Text; namespace Squidex.Extensions.Text.ElasticSearch @@ -30,26 +31,67 @@ namespace Squidex.Extensions.Text.ElasticSearch private static void UpsertEntry(UpsertIndexEntry upsert, List args, string indexName) { - args.Add(new + var geoField = string.Empty; + var geoObject = (object)null; + + if (upsert.GeoObjects != null) { - index = new + foreach (var (key, value) in upsert.GeoObjects) { - _id = upsert.DocId, - _index = indexName + if (value is Point point) + { + geoField = key; + geoObject = new + { + lat = point.Coordinates.Latitude, + lon = point.Coordinates.Longitude + }; + break; + } } - }); + } - args.Add(new + if (upsert.Texts != null || geoObject != null) { - appId = upsert.AppId.Id.ToString(), - appName = upsert.AppId.Name, - contentId = upsert.ContentId.ToString(), - schemaId = upsert.SchemaId.Id.ToString(), - schemaName = upsert.SchemaId.Name, - serveAll = upsert.ServeAll, - servePublished = upsert.ServePublished, - texts = upsert.Texts - }); + args.Add(new + { + index = new + { + _id = upsert.DocId, + _index = indexName + } + }); + + var texts = new Dictionary(); + + foreach (var (key, value) in upsert.Texts) + { + var text = value; + + var languageCode = ElasticSearchIndexDefinition.GetFieldName(key); + + if (texts.TryGetValue(languageCode, out var existing)) + { + text = $"{existing} {value}"; + } + + texts[languageCode] = text; + } + + args.Add(new + { + appId = upsert.AppId.Id.ToString(), + appName = upsert.AppId.Name, + contentId = upsert.ContentId.ToString(), + schemaId = upsert.SchemaId.Id.ToString(), + schemaName = upsert.SchemaId.Name, + serveAll = upsert.ServeAll, + servePublished = upsert.ServePublished, + texts, + geoField, + geoObject + }); + } } private static void UpdateEntry(UpdateIndexEntry update, List args, string indexName) diff --git a/backend/extensions/Squidex.Extensions/Text/ElasticSearch/ElasticSearchIndexDefinition.cs b/backend/extensions/Squidex.Extensions/Text/ElasticSearch/ElasticSearchIndexDefinition.cs new file mode 100644 index 000000000..53bac5839 --- /dev/null +++ b/backend/extensions/Squidex.Extensions/Text/ElasticSearch/ElasticSearchIndexDefinition.cs @@ -0,0 +1,140 @@ +// ========================================================================== +// Squidex Headless CMS +// ========================================================================== +// Copyright (c) Squidex UG (haftungsbeschraenkt) +// All rights reserved. Licensed under the MIT license. +// ========================================================================== + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using Elasticsearch.Net; + +namespace Squidex.Extensions.Text.ElasticSearch +{ + public static class ElasticSearchIndexDefinition + { + private static readonly Dictionary FieldPaths; + private static readonly Dictionary FieldAnalyzers = new Dictionary + { + ["ar"] = "arabic", + ["hy"] = "armenian", + ["eu"] = "basque", + ["bn"] = "bengali", + ["br"] = "brazilian", + ["bg"] = "bulgarian", + ["ca"] = "catalan", + ["zh"] = "cjk", + ["ja"] = "cjk", + ["ko"] = "cjk", + ["cs"] = "czech", + ["da"] = "danish", + ["nl"] = "dutch", + ["en"] = "english", + ["fi"] = "finnish", + ["fr"] = "french", + ["gl"] = "galician", + ["de"] = "german", + ["el"] = "greek", + ["hi"] = "hindi", + ["hu"] = "hungarian", + ["id"] = "indonesian", + ["ga"] = "irish", + ["it"] = "italian", + ["lv"] = "latvian", + ["lt"] = "lithuanian", + ["no"] = "norwegian", + ["pt"] = "portuguese", + ["ro"] = "romanian", + ["ru"] = "russian", + ["ku"] = "sorani", + ["es"] = "spanish", + ["sv"] = "swedish", + ["tr"] = "turkish", + ["th"] = "thai" + }; + + static ElasticSearchIndexDefinition() + { + FieldPaths = FieldAnalyzers.ToDictionary(x => x.Key, x => $"texts.{x.Key}"); + } + + public static string GetFieldName(string key) + { + if (FieldAnalyzers.ContainsKey(key)) + { + return key; + } + + if (key.Length > 0) + { + var language = key[2..]; + + if (FieldAnalyzers.ContainsKey(language)) + { + return language; + } + } + + return "iv"; + } + + public static string GetFieldPath(string key) + { + if (FieldPaths.TryGetValue(key, out var path)) + { + return path; + } + + if (key.Length > 0) + { + var language = key[2..]; + + if (FieldPaths.TryGetValue(language, out path)) + { + return path; + } + } + + return "texts.iv"; + } + + public static async Task ApplyAsync(IElasticLowLevelClient elastic, string indexName, + CancellationToken ct = default) + { + var query = new + { + properties = new Dictionary + { + ["geoObject"] = new + { + type ="geo_point" + } + } + }; + + foreach (var (key, analyzer) in FieldAnalyzers) + { + query.properties[GetFieldPath(key)] = new + { + type = "text", + analyzer + }; + } + + var result = await elastic.Indices.PutMappingAsync(indexName, CreatePost(query), ctx: ct); + + if (!result.Success) + { + throw new InvalidOperationException($"Failed with ${result.Body}", result.OriginalException); + } + } + + private static PostData CreatePost(T data) + { + return new SerializableData(data); + } + } +} diff --git a/backend/extensions/Squidex.Extensions/Text/ElasticSearch/ElasticSearchMapping.cs b/backend/extensions/Squidex.Extensions/Text/ElasticSearch/ElasticSearchMapping.cs deleted file mode 100644 index 84eccb230..000000000 --- a/backend/extensions/Squidex.Extensions/Text/ElasticSearch/ElasticSearchMapping.cs +++ /dev/null @@ -1,226 +0,0 @@ -// ========================================================================== -// Squidex Headless CMS -// ========================================================================== -// Copyright (c) Squidex UG (haftungsbeschraenkt) -// All rights reserved. Licensed under the MIT license. -// ========================================================================== - -using System; -using System.Collections.Generic; -using System.Threading; -using System.Threading.Tasks; -using Elasticsearch.Net; - -namespace Squidex.Extensions.Text.ElasticSearch -{ - public static class ElasticSearchMapping - { - public static async Task ApplyAsync(IElasticLowLevelClient elastic, string indexName, - CancellationToken ct = default) - { - var query = new - { - properties = new Dictionary - { - ["texts.ar"] = new - { - type = "text", - analyzer = "arabic" - }, - ["texts.hy"] = new - { - type = "text", - analyzer = "armenian" - }, - ["texts.eu"] = new - { - type = "text", - analyzer = "basque" - }, - ["texts.bn"] = new - { - type = "text", - analyzer = "bengali" - }, - ["texts.br"] = new - { - type = "text", - analyzer = "brazilian" - }, - ["texts.bg"] = new - { - type = "text", - analyzer = "bulgarian" - }, - ["texts.ca"] = new - { - type = "text", - analyzer = "catalan" - }, - ["texts.zh"] = new - { - type = "text", - analyzer = "cjk" - }, - ["texts.ja"] = new - { - type = "text", - analyzer = "cjk" - }, - ["texts.ko"] = new - { - type = "text", - analyzer = "cjk" - }, - ["texts.cs"] = new - { - type = "text", - analyzer = "czech" - }, - ["texts.da"] = new - { - type = "text", - analyzer = "danish" - }, - ["texts.nl"] = new - { - type = "text", - analyzer = "dutch" - }, - ["texts.en"] = new - { - type = "text", - analyzer = "english" - }, - ["texts.fi"] = new - { - type = "text", - analyzer = "finnish" - }, - ["texts.fr"] = new - { - type = "text", - analyzer = "french" - }, - ["texts.gl"] = new - { - type = "text", - analyzer = "galician" - }, - ["texts.de"] = new - { - type = "text", - analyzer = "german" - }, - ["texts.el"] = new - { - type = "text", - analyzer = "greek" - }, - ["texts.hi"] = new - { - type = "text", - analyzer = "hindi" - }, - ["texts.hu"] = new - { - type = "text", - analyzer = "hungarian" - }, - ["texts.id"] = new - { - type = "text", - analyzer = "indonesian" - }, - ["texts.ga"] = new - { - type = "text", - analyzer = "irish" - }, - ["texts.it"] = new - { - type = "text", - analyzer = "italian" - }, - ["texts.lv"] = new - { - type = "text", - analyzer = "latvian" - }, - ["texts.lt"] = new - { - type = "text", - analyzer = "lithuanian" - }, - ["texts.nb"] = new - { - type = "text", - analyzer = "norwegian" - }, - ["texts.nn"] = new - { - type = "text", - analyzer = "norwegian" - }, - ["texts.no"] = new - { - type = "text", - analyzer = "norwegian" - }, - ["texts.pt"] = new - { - type = "text", - analyzer = "portuguese" - }, - ["texts.ro"] = new - { - type = "text", - analyzer = "romanian" - }, - ["texts.ru"] = new - { - type = "text", - analyzer = "russian" - }, - ["texts.ku"] = new - { - type = "text", - analyzer = "sorani" - }, - ["texts.es"] = new - { - type = "text", - analyzer = "spanish" - }, - ["texts.sv"] = new - { - type = "text", - analyzer = "swedish" - }, - ["texts.tr"] = new - { - type = "text", - analyzer = "turkish" - }, - ["texts.th"] = new - { - type = "text", - analyzer = "thai" - } - } - }; - - var result = await elastic.Indices.PutMappingAsync(indexName, CreatePost(query), ctx: ct); - - if (!result.Success) - { - throw new InvalidOperationException($"Failed with ${result.Body}", result.OriginalException); - } - } - - private static PostData CreatePost(T data) - { - return new SerializableData(data); - } - } -} diff --git a/backend/extensions/Squidex.Extensions/Text/ElasticSearch/ElasticSearchTextIndex.cs b/backend/extensions/Squidex.Extensions/Text/ElasticSearch/ElasticSearchTextIndex.cs index d03d02df3..1299748c0 100644 --- a/backend/extensions/Squidex.Extensions/Text/ElasticSearch/ElasticSearchTextIndex.cs +++ b/backend/extensions/Squidex.Extensions/Text/ElasticSearch/ElasticSearchTextIndex.cs @@ -8,9 +8,11 @@ using System; using System.Collections.Generic; using System.Linq; +using System.Text.RegularExpressions; using System.Threading; using System.Threading.Tasks; using Elasticsearch.Net; +using Newtonsoft.Json; using Squidex.Domain.Apps.Entities.Apps; using Squidex.Domain.Apps.Entities.Contents; using Squidex.Domain.Apps.Entities.Contents.Text; @@ -21,25 +23,25 @@ namespace Squidex.Extensions.Text.ElasticSearch { public sealed class ElasticSearchTextIndex : ITextIndex, IInitializable { + private static readonly Regex LanguageRegex = new Regex(@"[^\w]+([a-z\-_]{2,}):", RegexOptions.Compiled | RegexOptions.ExplicitCapture); + private static readonly Regex LanguageRegexStart = new Regex(@"$^([a-z\-_]{2,}):", RegexOptions.Compiled | RegexOptions.ExplicitCapture); private readonly ElasticLowLevelClient client; + private readonly QueryParser queryParser = new QueryParser(ElasticSearchIndexDefinition.GetFieldPath); private readonly string indexName; - private readonly int waitAfterUpdate; - public ElasticSearchTextIndex(string configurationString, string indexName, int waitAfterUpdate = 0) + public ElasticSearchTextIndex(string configurationString, string indexName) { var config = new ConnectionConfiguration(new Uri(configurationString)); client = new ElasticLowLevelClient(config); this.indexName = indexName; - - this.waitAfterUpdate = waitAfterUpdate; } public Task InitializeAsync( CancellationToken ct) { - return ElasticSearchMapping.ApplyAsync(client, indexName, ct); + return ElasticSearchIndexDefinition.ApplyAsync(client, indexName, ct); } public Task ClearAsync( @@ -69,17 +71,68 @@ namespace Squidex.Extensions.Text.ElasticSearch { throw new InvalidOperationException($"Failed with ${result.Body}", result.OriginalException); } - - if (waitAfterUpdate > 0) - { - await Task.Delay(waitAfterUpdate, ct); - } } - public Task> SearchAsync(IAppEntity app, GeoQuery query, SearchScope scope, + public async Task> SearchAsync(IAppEntity app, GeoQuery query, SearchScope scope, CancellationToken ct = default) { - return Task.FromResult>(null); + Guard.NotNull(app, nameof(app)); + Guard.NotNull(query, nameof(query)); + + var serveField = GetServeField(scope); + + var elasticQuery = new + { + query = new + { + @bool = new + { + filter = new object[] + { + new + { + term = new Dictionary + { + ["schemaId.keyword"] = query.SchemaId.ToString() + } + }, + new + { + term = new Dictionary + { + ["geoField.keyword"] = query.Field + } + }, + new + { + term = new Dictionary + { + [serveField] = "true" + } + }, + new + { + geo_distance = new + { + geoObject = new + { + lat = query.Latitude, + lon = query.Longitude + }, + distance = $"{query.Radius}m" + } + } + }, + } + }, + _source = new[] + { + "contentId" + }, + size = query.Take + }; + + return await SearchAsync(elasticQuery, ct); } public async Task> SearchAsync(IAppEntity app, TextQuery query, SearchScope scope, @@ -88,34 +141,13 @@ namespace Squidex.Extensions.Text.ElasticSearch Guard.NotNull(app, nameof(app)); Guard.NotNull(query, nameof(query)); - var queryText = query.Text; + var parsed = queryParser.Parse(query.Text); - if (string.IsNullOrWhiteSpace(queryText)) + if (parsed == null) { return null; } - var isFuzzy = queryText.EndsWith("~", StringComparison.OrdinalIgnoreCase); - - if (isFuzzy) - { - queryText = queryText[..^1]; - } - - var field = "texts.*"; - - if (queryText.Length >= 4 && queryText.IndexOf(":", StringComparison.OrdinalIgnoreCase) == 2) - { - var candidateLanguage = queryText.Substring(0, 2); - - if (Language.IsValidLanguage(candidateLanguage)) - { - field = $"texts.{candidateLanguage}"; - - queryText = queryText[3..]; - } - } - var serveField = GetServeField(scope); var elasticQuery = new @@ -124,7 +156,7 @@ namespace Squidex.Extensions.Text.ElasticSearch { @bool = new { - must = new List + filter = new List { new { @@ -139,18 +171,13 @@ namespace Squidex.Extensions.Text.ElasticSearch { [serveField] = "true" } - }, - new + } + }, + must = new + { + query_string = new { - multi_match = new - { - fuzziness = isFuzzy ? (object)"AUTO" : 0, - fields = new[] - { - field - }, - query = query.Text - } + query = parsed.Text } }, should = new List() @@ -160,7 +187,7 @@ namespace Squidex.Extensions.Text.ElasticSearch { "contentId" }, - size = 2000 + size = query.Take }; if (query.RequiredSchemaIds?.Count > 0) @@ -173,7 +200,7 @@ namespace Squidex.Extensions.Text.ElasticSearch } }; - elasticQuery.query.@bool.must.Add(bySchema); + elasticQuery.query.@bool.filter.Add(bySchema); } else if (query.PreferredSchemaId.HasValue) { @@ -188,7 +215,15 @@ namespace Squidex.Extensions.Text.ElasticSearch elasticQuery.query.@bool.should.Add(bySchema); } - var result = await client.SearchAsync(indexName, CreatePost(elasticQuery), ctx: ct); + var json = JsonConvert.SerializeObject(elasticQuery, Formatting.Indented); + + return await SearchAsync(elasticQuery, ct); + } + + private async Task> SearchAsync(object query, + CancellationToken ct) + { + var result = await client.SearchAsync(indexName, CreatePost(query), ctx: ct); if (!result.Success) { diff --git a/backend/src/Squidex.Domain.Apps.Entities.MongoDb/Squidex.Domain.Apps.Entities.MongoDb.csproj b/backend/src/Squidex.Domain.Apps.Entities.MongoDb/Squidex.Domain.Apps.Entities.MongoDb.csproj index 03e3a4eea..a7651a3cc 100644 --- a/backend/src/Squidex.Domain.Apps.Entities.MongoDb/Squidex.Domain.Apps.Entities.MongoDb.csproj +++ b/backend/src/Squidex.Domain.Apps.Entities.MongoDb/Squidex.Domain.Apps.Entities.MongoDb.csproj @@ -17,12 +17,13 @@ + all runtime; build; native; contentfiles; analyzers; buildtransitive - + diff --git a/backend/src/Squidex.Domain.Apps.Entities.MongoDb/Text/AtlasIndexDefinition.cs b/backend/src/Squidex.Domain.Apps.Entities.MongoDb/Text/AtlasIndexDefinition.cs new file mode 100644 index 000000000..55863de29 --- /dev/null +++ b/backend/src/Squidex.Domain.Apps.Entities.MongoDb/Text/AtlasIndexDefinition.cs @@ -0,0 +1,210 @@ +// ========================================================================== +// Squidex Headless CMS +// ========================================================================== +// Copyright (c) Squidex UG (haftungsbeschraenkt) +// All rights reserved. Licensed under the MIT license. +// ========================================================================== + +using System.Collections.Generic; +using System.Linq; +using System.Net; +using System.Net.Http; +using System.Net.Http.Json; +using System.Threading; +using System.Threading.Tasks; +using Squidex.Hosting.Configuration; + +namespace Squidex.Domain.Apps.Entities.MongoDb.Text +{ + public static class AtlasIndexDefinition + { + private static readonly Dictionary FieldPaths = new Dictionary(); + private static readonly Dictionary FieldAnalyzers = new Dictionary + { + ["iv"] = "lucene.standard", + ["ar"] = "lucene.arabic", + ["hy"] = "lucene.armenian", + ["eu"] = "lucene.basque", + ["bn"] = "lucene.bengali", + ["br"] = "lucene.brazilian", + ["bg"] = "lucene.bulgarian", + ["ca"] = "lucene.catalan", + ["ko"] = "lucene.cjk", + ["da"] = "lucene.danish", + ["nl"] = "lucene.dutch", + ["en"] = "lucene.english", + ["fi"] = "lucene.finnish", + ["fr"] = "lucene.french", + ["gl"] = "lucene.galician", + ["de"] = "lucene.german", + ["el"] = "lucene.greek", + ["hi"] = "lucene.hindi", + ["hu"] = "lucene.hungarian", + ["id"] = "lucene.indonesian", + ["ga"] = "lucene.irish", + ["it"] = "lucene.italian", + ["jp"] = "lucene.japanese", + ["lv"] = "lucene.latvian", + ["no"] = "lucene.norwegian", + ["fa"] = "lucene.persian", + ["pt"] = "lucene.portuguese", + ["ro"] = "lucene.romanian", + ["ru"] = "lucene.russian", + ["zh"] = "lucene.smartcn", + ["es"] = "lucene.spanish", + ["sv"] = "lucene.swedish", + ["th"] = "lucene.thai", + ["tr"] = "lucene.turkish", + ["uk"] = "lucene.ukrainian" + }; + + public sealed class ErrorResponse + { + public string Detail { get; set; } + + public string ErrorCode { get; set; } + } + + static AtlasIndexDefinition() + { + FieldPaths = FieldAnalyzers.ToDictionary(x => x.Key, x => $"t.{x.Key}"); + } + + public static string GetFieldName(string key) + { + if (FieldAnalyzers.ContainsKey(key)) + { + return key; + } + + if (key.Length > 0) + { + var language = key[2..]; + + if (FieldAnalyzers.ContainsKey(language)) + { + return language; + } + } + + return "iv"; + } + + public static string GetFieldPath(string key) + { + if (FieldPaths.TryGetValue(key, out var path)) + { + return path; + } + + if (key.Length > 0) + { + var language = key[2..]; + + if (FieldPaths.TryGetValue(language, out path)) + { + return path; + } + } + + return "t.iv"; + } + + public static async Task CreateIndexAsync(AtlasOptions options, + string database, + string collectionName, + CancellationToken ct) + { + var (index, name) = Create(database, collectionName); + + using (var httpClient = new HttpClient(new HttpClientHandler + { + Credentials = new NetworkCredential(options.PublicKey, options.PrivateKey, "cloud.mongodb.com") + })) + { + var url = $"https://cloud.mongodb.com/api/atlas/v1.0/groups/{options.GroupId}/clusters/{options.ClusterName}/fts/indexes"; + + var result = await httpClient.PostAsJsonAsync(url, index, ct); + + if (result.IsSuccessStatusCode) + { + return name; + } + + var error = await result.Content.ReadFromJsonAsync(cancellationToken: ct); + + if (error?.ErrorCode != "ATLAS_FTS_DUPLICATE_INDEX") + { + var message = new ConfigurationError($"Creating index failed with {result.StatusCode}: {error?.Detail}"); + + throw new ConfigurationException(message); + } + } + + return name; + } + + public static (object, string) Create(string database, string collectionName) + { + var name = $"{database}_{collectionName}_text".ToLowerInvariant(); + + var texts = new + { + type = "document", + fields = new Dictionary(), + dynamic = false + }; + + var index = new + { + collectionName, + database, + name, + mappings = new + { + dynamic = false, + fields = new Dictionary + { + ["_ai"] = new + { + type = "string", + analyzer = "lucene.keyword" + }, + ["_si"] = new + { + type = "string", + analyzer = "lucene.keyword" + }, + ["_ci"] = new + { + type = "string", + analyzer = "lucene.keyword" + }, + ["fa"] = new + { + type = "boolean" + }, + ["fp"] = new + { + type = "boolean" + }, + ["t"] = texts + } + } + }; + + foreach (var (field, analyzer) in FieldAnalyzers) + { + texts.fields[field] = new + { + type = "string", + analyzer, + searchAnalyzer = analyzer, + store = false + }; + } + + return (index, name); + } + } +} diff --git a/backend/src/Squidex.Domain.Apps.Entities.MongoDb/Text/AtlasOptions.cs b/backend/src/Squidex.Domain.Apps.Entities.MongoDb/Text/AtlasOptions.cs new file mode 100644 index 000000000..e71812463 --- /dev/null +++ b/backend/src/Squidex.Domain.Apps.Entities.MongoDb/Text/AtlasOptions.cs @@ -0,0 +1,31 @@ +// ========================================================================== +// Squidex Headless CMS +// ========================================================================== +// Copyright (c) Squidex UG (haftungsbeschraenkt) +// All rights reserved. Licensed under the MIT license. +// ========================================================================== + +namespace Squidex.Domain.Apps.Entities.MongoDb.Text +{ + public sealed class AtlasOptions + { + public string GroupId { get; set; } + + public string ClusterName { get; set; } + + public string PublicKey { get; set; } + + public string PrivateKey { get; set; } + + public bool FullTextEnabled { get; set; } + + public bool IsConfigured() + { + return + !string.IsNullOrWhiteSpace(GroupId) && + !string.IsNullOrWhiteSpace(ClusterName) && + !string.IsNullOrWhiteSpace(PublicKey) && + !string.IsNullOrWhiteSpace(PrivateKey); + } + } +} diff --git a/backend/src/Squidex.Domain.Apps.Entities.MongoDb/Text/AtlasTextIndex.cs b/backend/src/Squidex.Domain.Apps.Entities.MongoDb/Text/AtlasTextIndex.cs new file mode 100644 index 000000000..2ee252e02 --- /dev/null +++ b/backend/src/Squidex.Domain.Apps.Entities.MongoDb/Text/AtlasTextIndex.cs @@ -0,0 +1,164 @@ +// ========================================================================== +// Squidex Headless CMS +// ========================================================================== +// Copyright (c) Squidex UG (haftungsbeschraenkt) +// All rights reserved. Licensed under the MIT license. +// ========================================================================== + +using System.Collections.Generic; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using Lucene.Net.Analysis.Standard; +using Lucene.Net.Analysis.Util; +using Lucene.Net.Util; +using Microsoft.Extensions.Options; +using MongoDB.Bson; +using MongoDB.Driver; +using Squidex.Domain.Apps.Entities.Apps; +using Squidex.Domain.Apps.Entities.Contents; +using Squidex.Domain.Apps.Entities.Contents.Text; +using Squidex.Infrastructure; +using LuceneQueryAnalyzer = Lucene.Net.QueryParsers.Classic.QueryParser; + +namespace Squidex.Domain.Apps.Entities.MongoDb.Text +{ + public sealed class AtlasTextIndex : MongoTextIndexBase> + { + private static readonly LuceneQueryVisitor QueryVisitor = new LuceneQueryVisitor(AtlasIndexDefinition.GetFieldPath); + private static readonly LuceneQueryAnalyzer QueryParser = + new LuceneQueryAnalyzer(LuceneVersion.LUCENE_48, "*", + new StandardAnalyzer(LuceneVersion.LUCENE_48, CharArraySet.EMPTY_SET)); + private readonly AtlasOptions options; + private string index; + + public AtlasTextIndex(IMongoDatabase database, IOptions options, bool setup = false) + : base(database, setup) + { + this.options = options.Value; + } + + protected override async Task SetupCollectionAsync(IMongoCollection>> collection, + CancellationToken ct) + { + await base.SetupCollectionAsync(collection, ct); + + index = await AtlasIndexDefinition.CreateIndexAsync(options, + Database.DatabaseNamespace.DatabaseName, CollectionName(), ct); + } + + protected override Dictionary BuildTexts(Dictionary source) + { + var texts = new Dictionary(); + + foreach (var (key, value) in source) + { + var text = value; + + var languageCode = AtlasIndexDefinition.GetFieldName(key); + + if (texts.TryGetValue(languageCode, out var existing)) + { + text = $"{existing} {value}"; + } + + texts[languageCode] = text; + } + + return texts; + } + + public override async Task?> SearchAsync(IAppEntity app, TextQuery query, SearchScope scope, + CancellationToken ct = default) + { + Guard.NotNull(app, nameof(app)); + Guard.NotNull(query, nameof(query)); + + var (search, take) = query; + + if (string.IsNullOrWhiteSpace(search)) + { + return null; + } + + var luceneQuery = QueryParser.Parse(search); + + var serveField = scope == SearchScope.All ? "fa" : "fp"; + + var compound = new BsonDocument + { + ["must"] = new BsonArray + { + QueryVisitor.Visit(luceneQuery) + }, + ["filter"] = new BsonArray + { + new BsonDocument + { + ["text"] = new BsonDocument + { + ["path"] = "_ai", + ["query"] = app.Id.ToString() + } + }, + new BsonDocument + { + ["equals"] = new BsonDocument + { + ["path"] = serveField, + ["value"] = true + } + } + } + }; + + if (query.PreferredSchemaId != null) + { + compound["should"] = new BsonArray + { + new BsonDocument + { + ["text"] = new BsonDocument + { + ["path"] = "_si", + ["query"] = query.PreferredSchemaId.Value.ToString() + } + } + }; + } + else if (query.RequiredSchemaIds?.Count > 0) + { + compound["should"] = new BsonArray(query.RequiredSchemaIds.Select(x => + new BsonDocument + { + ["text"] = new BsonDocument + { + ["path"] = "_si", + ["query"] = x.ToString() + } + })); + + compound["minimumShouldMatch"] = 1; + } + + var searchQuery = new BsonDocument + { + ["compound"] = compound, + }; + + if (index != null) + { + searchQuery["index"] = index; + } + + var results = + await Collection.Aggregate().Search(searchQuery).Limit(take) + .Project( + Projection.Include(x => x.ContentId) + ) + .ToListAsync(ct); + + return results.Select(x => x.ContentId).ToList(); + } + } +} diff --git a/backend/src/Squidex.Domain.Apps.Entities.MongoDb/FullText/CommandFactory.cs b/backend/src/Squidex.Domain.Apps.Entities.MongoDb/Text/CommandFactory.cs similarity index 70% rename from backend/src/Squidex.Domain.Apps.Entities.MongoDb/FullText/CommandFactory.cs rename to backend/src/Squidex.Domain.Apps.Entities.MongoDb/Text/CommandFactory.cs index b0f2f8415..0eed3d4ed 100644 --- a/backend/src/Squidex.Domain.Apps.Entities.MongoDb/FullText/CommandFactory.cs +++ b/backend/src/Squidex.Domain.Apps.Entities.MongoDb/Text/CommandFactory.cs @@ -11,14 +11,23 @@ using System.Linq; using MongoDB.Driver; using Squidex.Domain.Apps.Entities.Contents.Text; -namespace Squidex.Domain.Apps.Entities.MongoDb.FullText +namespace Squidex.Domain.Apps.Entities.MongoDb.Text { - public static class CommandFactory + public sealed class CommandFactory where T : class { - private static readonly FilterDefinitionBuilder Filter = Builders.Filter; - private static readonly UpdateDefinitionBuilder Update = Builders.Update; +#pragma warning disable RECS0108 // Warns about static fields in generic types + private static readonly FilterDefinitionBuilder> Filter = Builders>.Filter; + private static readonly UpdateDefinitionBuilder> Update = Builders>.Update; +#pragma warning restore RECS0108 // Warns about static fields in generic types - public static void CreateCommands(IndexCommand command, List> writes) + private readonly Func, T> textBuilder; + + public CommandFactory(Func, T> textBuilder) + { + this.textBuilder = textBuilder; + } + + public void CreateCommands(IndexCommand command, List>> writes) { switch (command) { @@ -34,10 +43,10 @@ namespace Squidex.Domain.Apps.Entities.MongoDb.FullText } } - private static void UpsertEntry(UpsertIndexEntry upsert, List> writes) + private void UpsertEntry(UpsertIndexEntry upsert, List>> writes) { writes.Add( - new UpdateOneModel( + new UpdateOneModel>( Filter.And( Filter.Eq(x => x.DocId, upsert.DocId), Filter.Exists(x => x.GeoField, false), @@ -45,7 +54,7 @@ namespace Squidex.Domain.Apps.Entities.MongoDb.FullText Update .Set(x => x.ServeAll, upsert.ServeAll) .Set(x => x.ServePublished, upsert.ServePublished) - .Set(x => x.Texts, upsert.Texts?.Values.Select(MongoTextIndexEntityText.FromText).ToList()) + .Set(x => x.Texts, BuildTexts(upsert)) .SetOnInsert(x => x.Id, Guid.NewGuid().ToString()) .SetOnInsert(x => x.DocId, upsert.DocId) .SetOnInsert(x => x.AppId, upsert.AppId.Id) @@ -60,7 +69,7 @@ namespace Squidex.Domain.Apps.Entities.MongoDb.FullText if (!upsert.IsNew) { writes.Add( - new DeleteOneModel( + new DeleteOneModel>( Filter.And( Filter.Eq(x => x.DocId, upsert.DocId), Filter.Exists(x => x.GeoField), @@ -70,8 +79,8 @@ namespace Squidex.Domain.Apps.Entities.MongoDb.FullText foreach (var (field, geoObject) in upsert.GeoObjects) { writes.Add( - new InsertOneModel( - new MongoTextIndexEntity + new InsertOneModel>( + new MongoTextIndexEntity { Id = Guid.NewGuid().ToString(), AppId = upsert.AppId.Id, @@ -87,20 +96,25 @@ namespace Squidex.Domain.Apps.Entities.MongoDb.FullText } } - private static void UpdateEntry(UpdateIndexEntry update, List> writes) + private T? BuildTexts(UpsertIndexEntry upsert) + { + return upsert.Texts == null ? null : textBuilder(upsert.Texts); + } + + private static void UpdateEntry(UpdateIndexEntry update, List>> writes) { writes.Add( - new UpdateOneModel( + new UpdateOneModel>( Filter.Eq(x => x.DocId, update.DocId), Update .Set(x => x.ServeAll, update.ServeAll) .Set(x => x.ServePublished, update.ServePublished))); } - private static void DeleteEntry(DeleteIndexEntry delete, List> writes) + private static void DeleteEntry(DeleteIndexEntry delete, List>> writes) { writes.Add( - new DeleteOneModel( + new DeleteOneModel>( Filter.Eq(x => x.DocId, delete.DocId))); } } diff --git a/backend/src/Squidex.Domain.Apps.Entities.MongoDb/Text/LuceneQueryVisitor.cs b/backend/src/Squidex.Domain.Apps.Entities.MongoDb/Text/LuceneQueryVisitor.cs new file mode 100644 index 000000000..d7892e07a --- /dev/null +++ b/backend/src/Squidex.Domain.Apps.Entities.MongoDb/Text/LuceneQueryVisitor.cs @@ -0,0 +1,329 @@ +// ========================================================================== +// Squidex Headless CMS +// ========================================================================== +// Copyright (c) Squidex UG (haftungsbeschraenkt) +// All rights reserved. Licensed under the MIT license. +// ========================================================================== + +using System; +using System.Globalization; +using System.Linq; +using System.Text; +using Lucene.Net.Index; +using Lucene.Net.Search; +using Lucene.Net.Util; +using MongoDB.Bson; + +namespace Squidex.Domain.Apps.Entities.MongoDb.Text +{ + public sealed class LuceneQueryVisitor + { + private readonly Func? fieldConverter; + + public LuceneQueryVisitor(Func? fieldConverter = null) + { + this.fieldConverter = fieldConverter; + } + + public BsonDocument Visit(Query query) + { + switch (query) + { + case BooleanQuery booleanQuery: + return VisitBoolean(booleanQuery); + case TermQuery termQuery: + return VisitTerm(termQuery); + case PhraseQuery phraseQuery: + return VisitPhrase(phraseQuery); + case WildcardQuery wildcardQuery: + return VisitWilcard(wildcardQuery); + case PrefixQuery prefixQuery: + return VisitPrefix(prefixQuery); + case FuzzyQuery fuzzyQuery: + return VisitFuzzy(fuzzyQuery); + case NumericRangeQuery rangeQuery: + return VisitNumericRange(rangeQuery); + case NumericRangeQuery rangeQuery: + return VisitNumericRange(rangeQuery); + case NumericRangeQuery rangeQuery: + return VisitNumericRange(rangeQuery); + case NumericRangeQuery rangeQuery: + return VisitNumericRange(rangeQuery); + case TermRangeQuery termRangeQuery: + return VisitTermRange(termRangeQuery); + default: + throw new NotSupportedException(); + } + } + + private BsonDocument VisitTermRange(TermRangeQuery termRangeQuery) + { + if (!TryParseValue(termRangeQuery.LowerTerm, out var min) || + !TryParseValue(termRangeQuery.UpperTerm, out var max)) + { + throw new NotSupportedException(); + } + + var minField = termRangeQuery.IncludesLower ? "gte" : "gt"; + var maxField = termRangeQuery.IncludesUpper ? "lte" : "lt"; + + var doc = new BsonDocument + { + ["path"] = GetPath(termRangeQuery.Field), + [minField] = BsonValue.Create(min), + [maxField] = BsonValue.Create(max) + }; + + ApplyBoost(termRangeQuery, doc); + + return new BsonDocument + { + ["range"] = doc + }; + } + + private BsonDocument VisitNumericRange(NumericRangeQuery numericRangeQuery) where T : struct, IComparable + { + var minField = numericRangeQuery.IncludesMin ? "gte" : "gt"; + var maxField = numericRangeQuery.IncludesMin ? "lte" : "lt"; + + var doc = new BsonDocument + { + ["path"] = GetPath(numericRangeQuery.Field), + [minField] = BsonValue.Create(numericRangeQuery.Min), + [maxField] = BsonValue.Create(numericRangeQuery.Max) + }; + + ApplyBoost(numericRangeQuery, doc); + + return new BsonDocument + { + ["range"] = doc + }; + } + + private BsonDocument VisitFuzzy(FuzzyQuery fuzzyQuery) + { + var doc = CreateDefaultDoc(fuzzyQuery, fuzzyQuery.Term); + + if (fuzzyQuery.MaxEdits > 0) + { + var fuzzy = new BsonDocument + { + ["maxEdits"] = fuzzyQuery.MaxEdits, + }; + + if (fuzzyQuery.PrefixLength > 0) + { + fuzzy["prefixLength"] = fuzzyQuery.PrefixLength; + } + + doc["fuzzy"] = fuzzy; + } + + return new BsonDocument + { + ["text"] = doc + }; + } + + private BsonDocument VisitPrefix(PrefixQuery prefixQuery) + { + var doc = CreateDefaultDoc(prefixQuery, new Term(prefixQuery.Prefix.Field, prefixQuery.Prefix.Text + "*")); + + return new BsonDocument + { + ["wildcard"] = doc + }; + } + + private BsonDocument VisitWilcard(WildcardQuery wildcardQuery) + { + var doc = CreateDefaultDoc(wildcardQuery, wildcardQuery.Term); + + return new BsonDocument + { + ["wildcard"] = doc + }; + } + + private BsonDocument VisitPhrase(PhraseQuery phraseQuery) + { + var terms = phraseQuery.GetTerms(); + + var doc = new BsonDocument + { + ["path"] = GetPath(terms[0].Field), + }; + + if (terms.Length == 1) + { + doc["query"] = terms[0].Text; + } + else + { + doc["query"] = new BsonArray(terms.Select(x => x.Text)); + } + + if (phraseQuery.Slop != 0) + { + doc["slop"] = phraseQuery.Slop; + } + + ApplyBoost(phraseQuery, doc); + + return new BsonDocument + { + ["phrase"] = doc + }; + } + + private BsonDocument VisitTerm(TermQuery termQuery) + { + var doc = CreateDefaultDoc(termQuery, termQuery.Term); + + return new BsonDocument + { + ["text"] = doc + }; + } + + private BsonDocument VisitBoolean(BooleanQuery booleanQuery) + { + var doc = new BsonDocument(); + + BsonArray? musts = null; + BsonArray? mustNots = null; + BsonArray? shoulds = null; + + foreach (var clause in booleanQuery.Clauses) + { + var converted = Visit(clause.Query); + + switch (clause.Occur) + { + case Occur.MUST: + musts ??= new BsonArray(); + musts.Add(converted); + break; + case Occur.SHOULD: + shoulds ??= new BsonArray(); + shoulds.Add(converted); + break; + case Occur.MUST_NOT: + mustNots ??= new BsonArray(); + mustNots.Add(converted); + break; + } + } + + if (musts != null) + { + doc.Add("must", musts); + } + + if (mustNots != null) + { + doc.Add("mustNot", mustNots); + } + + if (shoulds != null) + { + doc.Add("should", shoulds); + } + + if (booleanQuery.MinimumNumberShouldMatch > 0) + { + doc["minimumShouldMatch"] = booleanQuery.MinimumNumberShouldMatch; + } + + return new BsonDocument + { + ["compound"] = doc + }; + } + + private BsonDocument CreateDefaultDoc(Query query, Term term) + { + var doc = new BsonDocument + { + ["path"] = GetPath(term.Field), + ["query"] = term.Text + }; + + ApplyBoost(query, doc); + + return doc; + } + + private BsonValue GetPath(string field) + { + if (field != "*" && fieldConverter != null) + { + field = fieldConverter(field); + } + + if (field.Contains('*', StringComparison.Ordinal)) + { + return new BsonDocument + { + ["wildcard"] = field + }; + } + + return field; + } + +#pragma warning disable RECS0018 // Comparison of floating point numbers with equality operator + private static void ApplyBoost(Query query, BsonDocument doc) + { + if (query.Boost != 1) + { + doc["score"] = new BsonDocument + { + ["boost"] = query.Boost + }; + } + } + + private static bool TryParseValue(BytesRef bytes, out object result) + { + result = null!; + + try + { + var text = Encoding.ASCII.GetString(bytes.Bytes, bytes.Offset, bytes.Length); + + if (!double.TryParse(text, NumberStyles.Any, CultureInfo.InvariantCulture, out var number)) + { + return false; + } + + var integer = (long)number; + + if (number == integer) + { + if (integer <= int.MaxValue && integer >= int.MinValue) + { + result = (int)integer; + } + else + { + result = integer; + } + } + else + { + result = number; + } + + return true; + } + catch (Exception) + { + return false; + } + } +#pragma warning restore RECS0018 // Comparison of floating point numbers with equality operator + } +} diff --git a/backend/src/Squidex.Domain.Apps.Entities.MongoDb/Text/LuceneSearchDefinitionExtensions.cs b/backend/src/Squidex.Domain.Apps.Entities.MongoDb/Text/LuceneSearchDefinitionExtensions.cs new file mode 100644 index 000000000..4776fb556 --- /dev/null +++ b/backend/src/Squidex.Domain.Apps.Entities.MongoDb/Text/LuceneSearchDefinitionExtensions.cs @@ -0,0 +1,57 @@ +// ========================================================================== +// Squidex Headless CMS +// ========================================================================== +// Copyright (c) Squidex UG (haftungsbeschraenkt) +// All rights reserved. Licensed under the MIT license. +// ========================================================================== + +using System; +using MongoDB.Bson; +using MongoDB.Bson.Serialization; +using MongoDB.Driver; + +namespace Squidex.Domain.Apps.Entities.MongoDb.Text +{ + public static class LuceneSearchDefinitionExtensions + { + public static IAggregateFluent Search( + this IAggregateFluent aggregate, + BsonDocument search) + { + const string OperatorName = "$search"; + + var stage = new DelegatedPipelineStageDefinition( + OperatorName, + serializer => + { + var document = new BsonDocument(OperatorName, search); + + return new RenderedPipelineStageDefinition(OperatorName, document, serializer); + }); + + return aggregate.AppendStage(stage); + } + + private sealed class DelegatedPipelineStageDefinition : PipelineStageDefinition + { + private readonly Func, RenderedPipelineStageDefinition> renderer; + + public override string OperatorName { get; } + + public DelegatedPipelineStageDefinition(string operatorName, + Func, RenderedPipelineStageDefinition> renderer) + { + this.renderer = renderer; + + OperatorName = operatorName; + } + + public override RenderedPipelineStageDefinition Render( + IBsonSerializer inputSerializer, + IBsonSerializerRegistry serializerRegistry) + { + return renderer(inputSerializer); + } + } + } +} diff --git a/backend/src/Squidex.Domain.Apps.Entities.MongoDb/Text/MongoTextIndex.cs b/backend/src/Squidex.Domain.Apps.Entities.MongoDb/Text/MongoTextIndex.cs new file mode 100644 index 000000000..5cf2c0204 --- /dev/null +++ b/backend/src/Squidex.Domain.Apps.Entities.MongoDb/Text/MongoTextIndex.cs @@ -0,0 +1,44 @@ +// ========================================================================== +// Squidex Headless CMS +// ========================================================================== +// Copyright (c) Squidex UG (haftungsbeschraenkt) +// All rights reserved. Licensed under the MIT license. +// ========================================================================== + +using System.Collections.Generic; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using MongoDB.Driver; + +namespace Squidex.Domain.Apps.Entities.MongoDb.Text +{ + public sealed class MongoTextIndex : MongoTextIndexBase> + { + public MongoTextIndex(IMongoDatabase database, bool setup = false) + : base(database, setup) + { + } + + protected override async Task SetupCollectionAsync(IMongoCollection>> collection, + CancellationToken ct) + { + await base.SetupCollectionAsync(collection, ct); + + await collection.Indexes.CreateOneAsync( + new CreateIndexModel>>( + Index + .Text("t.t") + .Ascending(x => x.AppId) + .Ascending(x => x.ServeAll) + .Ascending(x => x.ServePublished) + .Ascending(x => x.SchemaId)), + cancellationToken: ct); + } + + protected override List BuildTexts(Dictionary source) + { + return source.Select(x => new MongoTextIndexEntityText { Text = x.Value }).ToList(); + } + } +} diff --git a/backend/src/Squidex.Domain.Apps.Entities.MongoDb/FullText/MongoTextIndex.cs b/backend/src/Squidex.Domain.Apps.Entities.MongoDb/Text/MongoTextIndexBase.cs similarity index 63% rename from backend/src/Squidex.Domain.Apps.Entities.MongoDb/FullText/MongoTextIndex.cs rename to backend/src/Squidex.Domain.Apps.Entities.MongoDb/Text/MongoTextIndexBase.cs index 28cbcea2c..757ac7946 100644 --- a/backend/src/Squidex.Domain.Apps.Entities.MongoDb/FullText/MongoTextIndex.cs +++ b/backend/src/Squidex.Domain.Apps.Entities.MongoDb/Text/MongoTextIndexBase.cs @@ -18,14 +18,15 @@ using Squidex.Domain.Apps.Entities.Contents.Text; using Squidex.Infrastructure; using Squidex.Infrastructure.MongoDb; -namespace Squidex.Domain.Apps.Entities.MongoDb.FullText +namespace Squidex.Domain.Apps.Entities.MongoDb.Text { - public sealed class MongoTextIndex : MongoRepositoryBase, ITextIndex, IDeleter + public abstract class MongoTextIndexBase : MongoRepositoryBase>, ITextIndex, IDeleter where T : class { - private readonly ProjectionDefinition searchTextProjection; - private readonly ProjectionDefinition searchGeoProjection; + private readonly ProjectionDefinition> searchTextProjection; + private readonly ProjectionDefinition> searchGeoProjection; + private readonly CommandFactory commandFactory; - private sealed class MongoTextResult + protected sealed class MongoTextResult { [BsonId] [BsonElement] @@ -42,30 +43,26 @@ namespace Squidex.Domain.Apps.Entities.MongoDb.FullText public double Score { get; set; } } - public MongoTextIndex(IMongoDatabase database, bool setup = false) + protected MongoTextIndexBase(IMongoDatabase database, bool setup = false) : base(database, setup) { searchGeoProjection = Projection.Include(x => x.ContentId); searchTextProjection = Projection.Include(x => x.ContentId).MetaTextScore("score"); + +#pragma warning disable MA0056 // Do not call overridable members in constructor + commandFactory = new CommandFactory(BuildTexts); +#pragma warning restore MA0056 // Do not call overridable members in constructor } - protected override Task SetupCollectionAsync(IMongoCollection collection, + protected override Task SetupCollectionAsync(IMongoCollection> collection, CancellationToken ct) { return collection.Indexes.CreateManyAsync(new[] { - new CreateIndexModel( + new CreateIndexModel>( Index.Ascending(x => x.DocId)), - new CreateIndexModel( - Index - .Text("t.t") - .Ascending(x => x.AppId) - .Ascending(x => x.ServeAll) - .Ascending(x => x.ServePublished) - .Ascending(x => x.SchemaId)), - - new CreateIndexModel( + new CreateIndexModel>( Index .Ascending(x => x.AppId) .Ascending(x => x.ServeAll) @@ -81,20 +78,22 @@ namespace Squidex.Domain.Apps.Entities.MongoDb.FullText return "TextIndex"; } + protected abstract T BuildTexts(Dictionary source); + async Task IDeleter.DeleteAppAsync(IAppEntity app, CancellationToken ct) { await Collection.DeleteManyAsync(Filter.Eq(x => x.AppId, app.Id), ct); } - public Task ExecuteAsync(IndexCommand[] commands, + public virtual Task ExecuteAsync(IndexCommand[] commands, CancellationToken ct = default) { - var writes = new List>(commands.Length); + var writes = new List>>(commands.Length); foreach (var command in commands) { - CommandFactory.CreateCommands(command, writes); + commandFactory.CreateCommands(command, writes); } if (writes.Count == 0) @@ -105,9 +104,12 @@ namespace Squidex.Domain.Apps.Entities.MongoDb.FullText return Collection.BulkWriteAsync(writes, BulkUnordered, ct); } - public async Task?> SearchAsync(IAppEntity app, GeoQuery query, SearchScope scope, + public virtual async Task?> SearchAsync(IAppEntity app, GeoQuery query, SearchScope scope, CancellationToken ct = default) { + Guard.NotNull(app, nameof(app)); + Guard.NotNull(query, nameof(query)); + var findFilter = Filter.And( Filter.Eq(x => x.AppId, app.Id), @@ -122,45 +124,43 @@ namespace Squidex.Domain.Apps.Entities.MongoDb.FullText return byGeo.Select(x => x.ContentId).ToList(); } - public async Task?> SearchAsync(IAppEntity app, TextQuery query, SearchScope scope, + public virtual async Task?> SearchAsync(IAppEntity app, TextQuery query, SearchScope scope, CancellationToken ct = default) { - if (string.IsNullOrWhiteSpace(query.Text)) + Guard.NotNull(app, nameof(app)); + Guard.NotNull(query, nameof(query)); + + var (search, take) = query; + + if (string.IsNullOrWhiteSpace(search)) { return null; } - List<(DomainId, double)> documents; + var result = new List<(DomainId Id, double Score)>(); if (query.RequiredSchemaIds?.Count > 0) { - documents = await SearchBySchemaAsync(query.Text, app, query.RequiredSchemaIds, scope, query.Take, 1, ct); + await SearchBySchemaAsync(result, search, app, query.RequiredSchemaIds, scope, take, 1, ct); } else if (query.PreferredSchemaId == null) { - documents = await SearchByAppAsync(query.Text, app, scope, query.Take, 1, ct); + await SearchByAppAsync(result, search, app, scope, take, 1, ct); } else { - var halfBucket = query.Take / 2; + var halfBucket = take / 2; var schemaIds = Enumerable.Repeat(query.PreferredSchemaId.Value, 1); - documents = await SearchBySchemaAsync( - query.Text, - app, - schemaIds, - scope, - halfBucket, 1, - ct); - - documents.AddRange(await SearchByAppAsync(query.Text, app, scope, halfBucket, 1, ct)); + await SearchBySchemaAsync(result, search, app, schemaIds, scope, halfBucket, 1.1, ct); + await SearchByAppAsync(result, search, app, scope, halfBucket, 1, ct); } - return documents.OrderByDescending(x => x.Item2).Select(x => x.Item1).Distinct().ToList(); + return result.OrderByDescending(x => x.Score).Select(x => x.Id).Distinct().ToList(); } - private Task> SearchBySchemaAsync(string text, IAppEntity app, IEnumerable schemaIds, SearchScope scope, int limit, double factor, + private Task SearchBySchemaAsync(List<(DomainId, double)> result, string text, IAppEntity app, IEnumerable schemaIds, SearchScope scope, int take, double factor, CancellationToken ct = default) { var filter = @@ -170,10 +170,10 @@ namespace Squidex.Domain.Apps.Entities.MongoDb.FullText Filter_ByScope(scope), Filter.Text(text, "none")); - return SearchAsync(filter, scope, limit, factor, ct); + return SearchAsync(result, filter, scope, take, factor, ct); } - private Task> SearchByAppAsync(string text, IAppEntity app, SearchScope scope, int limit, double factor, + private Task SearchByAppAsync(List<(DomainId, double)> result, string text, IAppEntity app, SearchScope scope, int take, double factor, CancellationToken ct = default) { var filter = @@ -183,24 +183,24 @@ namespace Squidex.Domain.Apps.Entities.MongoDb.FullText Filter_ByScope(scope), Filter.Text(text, "none")); - return SearchAsync(filter, scope, limit, factor, ct); + return SearchAsync(result, filter, scope, take, factor, ct); } - private async Task> SearchAsync(FilterDefinition filter, SearchScope scope, int limit, double factor, + private async Task SearchAsync(List<(DomainId, double)> result, FilterDefinition> filter, SearchScope scope, int take, double factor, CancellationToken ct = default) { var collection = GetCollection(scope); var find = - collection.Find(filter).Limit(limit) + collection.Find(filter).Limit(take) .Project(searchTextProjection).Sort(Sort.MetaTextScore("score")); var documents = await find.ToListAsync(ct); - return documents.Select(x => (x.ContentId, x.Score * factor)).ToList(); + result.AddRange(documents.Select(x => (x.ContentId, x.Score * factor))); } - private static FilterDefinition Filter_ByScope(SearchScope scope) + private static FilterDefinition> Filter_ByScope(SearchScope scope) { if (scope == SearchScope.All) { @@ -212,7 +212,7 @@ namespace Squidex.Domain.Apps.Entities.MongoDb.FullText } } - private IMongoCollection GetCollection(SearchScope scope) + private IMongoCollection> GetCollection(SearchScope scope) { if (scope == SearchScope.All) { diff --git a/backend/src/Squidex.Domain.Apps.Entities.MongoDb/FullText/MongoTextIndexEntity.cs b/backend/src/Squidex.Domain.Apps.Entities.MongoDb/Text/MongoTextIndexEntity.cs similarity index 89% rename from backend/src/Squidex.Domain.Apps.Entities.MongoDb/FullText/MongoTextIndexEntity.cs rename to backend/src/Squidex.Domain.Apps.Entities.MongoDb/Text/MongoTextIndexEntity.cs index ed3e7aa93..6a9a07fef 100644 --- a/backend/src/Squidex.Domain.Apps.Entities.MongoDb/FullText/MongoTextIndexEntity.cs +++ b/backend/src/Squidex.Domain.Apps.Entities.MongoDb/Text/MongoTextIndexEntity.cs @@ -5,16 +5,15 @@ // All rights reserved. Licensed under the MIT license. // ========================================================================== -using System.Collections.Generic; using GeoJSON.Net; using MongoDB.Bson; using MongoDB.Bson.Serialization.Attributes; using Squidex.Infrastructure; using Squidex.Infrastructure.MongoDb; -namespace Squidex.Domain.Apps.Entities.MongoDb.FullText +namespace Squidex.Domain.Apps.Entities.MongoDb.Text { - public sealed class MongoTextIndexEntity + public sealed class MongoTextIndexEntity { [BsonId] [BsonElement] @@ -50,7 +49,7 @@ namespace Squidex.Domain.Apps.Entities.MongoDb.FullText [BsonIgnoreIfNull] [BsonElement("t")] - public List Texts { get; set; } + public T Texts { get; set; } [BsonIgnoreIfNull] [BsonElement("gf")] diff --git a/backend/src/Squidex.Domain.Apps.Entities.MongoDb/FullText/MongoTextIndexEntityText.cs b/backend/src/Squidex.Domain.Apps.Entities.MongoDb/Text/MongoTextIndexEntityText.cs similarity index 93% rename from backend/src/Squidex.Domain.Apps.Entities.MongoDb/FullText/MongoTextIndexEntityText.cs rename to backend/src/Squidex.Domain.Apps.Entities.MongoDb/Text/MongoTextIndexEntityText.cs index 5a61bc54a..04c8f28c9 100644 --- a/backend/src/Squidex.Domain.Apps.Entities.MongoDb/FullText/MongoTextIndexEntityText.cs +++ b/backend/src/Squidex.Domain.Apps.Entities.MongoDb/Text/MongoTextIndexEntityText.cs @@ -7,7 +7,7 @@ using MongoDB.Bson.Serialization.Attributes; -namespace Squidex.Domain.Apps.Entities.MongoDb.FullText +namespace Squidex.Domain.Apps.Entities.MongoDb.Text { public sealed class MongoTextIndexEntityText { diff --git a/backend/src/Squidex.Domain.Apps.Entities.MongoDb/FullText/MongoTextIndexerState.cs b/backend/src/Squidex.Domain.Apps.Entities.MongoDb/Text/MongoTextIndexerState.cs similarity index 98% rename from backend/src/Squidex.Domain.Apps.Entities.MongoDb/FullText/MongoTextIndexerState.cs rename to backend/src/Squidex.Domain.Apps.Entities.MongoDb/Text/MongoTextIndexerState.cs index 68ab9cbbb..68a3ee302 100644 --- a/backend/src/Squidex.Domain.Apps.Entities.MongoDb/FullText/MongoTextIndexerState.cs +++ b/backend/src/Squidex.Domain.Apps.Entities.MongoDb/Text/MongoTextIndexerState.cs @@ -16,7 +16,7 @@ using Squidex.Domain.Apps.Entities.Contents.Text.State; using Squidex.Infrastructure; using Squidex.Infrastructure.MongoDb; -namespace Squidex.Domain.Apps.Entities.MongoDb.FullText +namespace Squidex.Domain.Apps.Entities.MongoDb.Text { public sealed class MongoTextIndexerState : MongoRepositoryBase, ITextIndexerState, IDeleter { diff --git a/backend/src/Squidex.Domain.Apps.Entities/Contents/Text/Query.cs b/backend/src/Squidex.Domain.Apps.Entities/Contents/Text/Query.cs new file mode 100644 index 000000000..26f4985c6 --- /dev/null +++ b/backend/src/Squidex.Domain.Apps.Entities/Contents/Text/Query.cs @@ -0,0 +1,14 @@ +// ========================================================================== +// Squidex Headless CMS +// ========================================================================== +// Copyright (c) Squidex UG (haftungsbeschraenkt) +// All rights reserved. Licensed under the MIT license. +// ========================================================================== + +namespace Squidex.Domain.Apps.Entities.Contents.Text +{ + public sealed class Query + { + public string Text { get; init; } + } +} diff --git a/backend/src/Squidex.Domain.Apps.Entities/Contents/Text/QueryParser.cs b/backend/src/Squidex.Domain.Apps.Entities/Contents/Text/QueryParser.cs new file mode 100644 index 000000000..22771d766 --- /dev/null +++ b/backend/src/Squidex.Domain.Apps.Entities/Contents/Text/QueryParser.cs @@ -0,0 +1,82 @@ +// ========================================================================== +// Squidex Headless CMS +// ========================================================================== +// Copyright (c) Squidex UG (haftungsbeschraenkt) +// All rights reserved. Licensed under the MIT license. +// ========================================================================== + +using System; +using System.Text; + +namespace Squidex.Domain.Apps.Entities.Contents.Text +{ + public sealed class QueryParser + { + private readonly Func fieldProvider; + + public QueryParser(Func fieldProvider) + { + this.fieldProvider = fieldProvider; + } + + public Query? Parse(string text) + { + if (string.IsNullOrWhiteSpace(text)) + { + return null; + } + + text = text.Trim(); + text = ConvertFieldNames(text); + + return new Query + { + Text = text + }; + } + + private string ConvertFieldNames(string query) + { + var indexOfColon = query.IndexOf(':', StringComparison.Ordinal); + + if (indexOfColon < 0) + { + return query; + } + + var sb = new StringBuilder(); + + int position = 0, lastIndexOfColon = 0; + + while (indexOfColon >= 0) + { + lastIndexOfColon = indexOfColon; + + var i = 0; + + for (i = indexOfColon - 1; i >= position; i--) + { + var c = query[i]; + + if (!char.IsLetter(c) && c != '-' && c != '_') + { + break; + } + } + + i++; + + sb.Append(query[position..i]); + sb.Append(fieldProvider(query[i..indexOfColon])); + + position = indexOfColon + 1; + + indexOfColon = query.IndexOf(':', position); + } + + sb.Append(query[lastIndexOfColon..]); + + return sb.ToString(); + } + } +} diff --git a/backend/src/Squidex/Config/Domain/StoreServices.cs b/backend/src/Squidex/Config/Domain/StoreServices.cs index e19e7ef21..d3e1eac66 100644 --- a/backend/src/Squidex/Config/Domain/StoreServices.cs +++ b/backend/src/Squidex/Config/Domain/StoreServices.cs @@ -24,10 +24,10 @@ using Squidex.Domain.Apps.Entities.History.Repositories; using Squidex.Domain.Apps.Entities.MongoDb.Apps; using Squidex.Domain.Apps.Entities.MongoDb.Assets; using Squidex.Domain.Apps.Entities.MongoDb.Contents; -using Squidex.Domain.Apps.Entities.MongoDb.FullText; using Squidex.Domain.Apps.Entities.MongoDb.History; using Squidex.Domain.Apps.Entities.MongoDb.Rules; using Squidex.Domain.Apps.Entities.MongoDb.Schemas; +using Squidex.Domain.Apps.Entities.MongoDb.Text; using Squidex.Domain.Apps.Entities.Rules.DomainObject; using Squidex.Domain.Apps.Entities.Rules.Repositories; using Squidex.Domain.Apps.Entities.Schemas; @@ -135,9 +135,6 @@ namespace Squidex.Config.Domain services.AddSingletonAs() .AsOptional().As().As(); - services.AddSingletonAs() - .AsOptional().As(); - services.AddSingletonAs() .As().As(); @@ -151,6 +148,21 @@ namespace Squidex.Config.Domain builder.SetDefaultScopeEntity(); builder.SetDefaultApplicationEntity(); }); + + var atlasOptions = config.GetSection("store:mongoDb:atlas").Get() ?? new (); + + if (atlasOptions.IsConfigured() && atlasOptions.FullTextEnabled) + { + services.Configure(config.GetSection("store:mongoDb:atlas")); + + services.AddSingletonAs() + .AsOptional().As(); + } + else + { + services.AddSingletonAs() + .AsOptional().As(); + } } }); diff --git a/backend/src/Squidex/Squidex.csproj b/backend/src/Squidex/Squidex.csproj index 3d80284c0..a197f4291 100644 --- a/backend/src/Squidex/Squidex.csproj +++ b/backend/src/Squidex/Squidex.csproj @@ -161,4 +161,6 @@ $(NoWarn);CS1591;1591;1573;1572;NU1605;IDE0060 + + \ No newline at end of file diff --git a/backend/src/Squidex/appsettings.json b/backend/src/Squidex/appsettings.json index afeff303f..62af22d7b 100644 --- a/backend/src/Squidex/appsettings.json +++ b/backend/src/Squidex/appsettings.json @@ -437,7 +437,22 @@ "contentDatabase": "SquidexContent", // The database for all your other read collections. - "database": "Squidex" + "database": "Squidex", + + "atlas": { + // The organization id. + "groupId": "", + + // The name of the cluster. + "clusterName": "", + + // Credentials to your account. + "publicKey": "", + "privateKey": "", + + // True, if you want to enable mongo atlas for full text search instead of MongoDB. + "fullTextEnabled": false + } } }, diff --git a/backend/tests/Squidex.Domain.Apps.Entities.Tests/Assets/MongoDb/AssetQueryTests.cs b/backend/tests/Squidex.Domain.Apps.Entities.Tests/Assets/MongoDb/AssetQueryTests.cs index ab4443f1b..ea28aed97 100644 --- a/backend/tests/Squidex.Domain.Apps.Entities.Tests/Assets/MongoDb/AssetQueryTests.cs +++ b/backend/tests/Squidex.Domain.Apps.Entities.Tests/Assets/MongoDb/AssetQueryTests.cs @@ -28,8 +28,6 @@ namespace Squidex.Domain.Apps.Entities.Assets.MongoDb { public class AssetQueryTests { - private static readonly IBsonSerializerRegistry Registry = BsonSerializer.SerializerRegistry; - private static readonly IBsonSerializer Serializer = BsonSerializer.SerializerRegistry.GetSerializer(); private readonly DomainId appId = DomainId.NewGuid(); static AssetQueryTests() @@ -212,13 +210,15 @@ namespace Squidex.Domain.Apps.Entities.Assets.MongoDb private void AssertQuery(string expected, ClrQuery query, object? arg = null) { - var rendered = - query.AdjustToModel(appId).BuildFilter(false).Filter! - .Render(Serializer, Registry).ToString(); + var filter = query.AdjustToModel(appId).BuildFilter(false).Filter!; - var expectation = Cleanup(expected, arg); + var rendered = + filter.Render( + BsonSerializer.SerializerRegistry.GetSerializer(), + BsonSerializer.SerializerRegistry) + .ToString(); - Assert.Equal(expectation, rendered); + Assert.Equal(Cleanup(expected, arg), rendered); } private void AssertSorting(string expected, params SortNode[] sort) @@ -230,14 +230,16 @@ namespace Squidex.Domain.Apps.Entities.Assets.MongoDb A.CallTo(() => cursor.Sort(A>._)) .Invokes((SortDefinition sortDefinition) => { - rendered = sortDefinition.Render(Serializer, Registry).ToString(); + rendered = + sortDefinition.Render( + BsonSerializer.SerializerRegistry.GetSerializer(), + BsonSerializer.SerializerRegistry) + .ToString(); }); cursor.QuerySort(new ClrQuery { Sort = sort.ToList() }.AdjustToModel(appId)); - var expectation = Cleanup(expected); - - Assert.Equal(expectation, rendered); + Assert.Equal(Cleanup(expected), rendered); } private static string Cleanup(string filter, object? arg = null) diff --git a/backend/tests/Squidex.Domain.Apps.Entities.Tests/Assets/MongoDb/AssetsQueryFixture.cs b/backend/tests/Squidex.Domain.Apps.Entities.Tests/Assets/MongoDb/AssetsQueryFixture.cs index ff85e8da2..2f355d2d7 100644 --- a/backend/tests/Squidex.Domain.Apps.Entities.Tests/Assets/MongoDb/AssetsQueryFixture.cs +++ b/backend/tests/Squidex.Domain.Apps.Entities.Tests/Assets/MongoDb/AssetsQueryFixture.cs @@ -15,6 +15,7 @@ using Newtonsoft.Json; using Squidex.Domain.Apps.Core.Assets; using Squidex.Domain.Apps.Core.TestHelpers; using Squidex.Domain.Apps.Entities.MongoDb.Assets; +using Squidex.Domain.Apps.Entities.TestHelpers; using Squidex.Infrastructure; using Squidex.Infrastructure.Json.Objects; using Squidex.Infrastructure.MongoDb; @@ -25,7 +26,7 @@ namespace Squidex.Domain.Apps.Entities.Assets.MongoDb { private readonly Random random = new Random(); private readonly int numValues = 250; - private readonly IMongoClient mongoClient = new MongoClient("mongodb://localhost"); + private readonly IMongoClient mongoClient; private readonly IMongoDatabase mongoDatabase; public MongoAssetRepository AssetRepository { get; } @@ -38,7 +39,8 @@ namespace Squidex.Domain.Apps.Entities.Assets.MongoDb public AssetsQueryFixture() { - mongoDatabase = mongoClient.GetDatabase("Squidex_Testing"); + mongoClient = new MongoClient(TestConfig.Configuration["mongodb:configuration"]); + mongoDatabase = mongoClient.GetDatabase(TestConfig.Configuration["mongodb:database"]); SetupJson(); diff --git a/backend/tests/Squidex.Domain.Apps.Entities.Tests/Contents/MongoDb/ContentQueryTests.cs b/backend/tests/Squidex.Domain.Apps.Entities.Tests/Contents/MongoDb/ContentQueryTests.cs index 501346ee9..4e52a36de 100644 --- a/backend/tests/Squidex.Domain.Apps.Entities.Tests/Contents/MongoDb/ContentQueryTests.cs +++ b/backend/tests/Squidex.Domain.Apps.Entities.Tests/Contents/MongoDb/ContentQueryTests.cs @@ -32,8 +32,6 @@ namespace Squidex.Domain.Apps.Entities.Contents.MongoDb { public class ContentQueryTests { - private static readonly IBsonSerializerRegistry Registry = BsonSerializer.SerializerRegistry; - private static readonly IBsonSerializer Serializer = BsonSerializer.SerializerRegistry.GetSerializer(); private readonly DomainId appId = DomainId.NewGuid(); private readonly Schema schemaDef; private readonly LanguagesConfig languagesConfig = LanguagesConfig.English.Set(Language.DE); @@ -233,13 +231,15 @@ namespace Squidex.Domain.Apps.Entities.Contents.MongoDb private void AssertQuery(ClrQuery query, string expected, object? arg = null) { - var rendered = - query.AdjustToModel(appId).BuildFilter().Filter! - .Render(Serializer, Registry).ToString(); + var filter = query.AdjustToModel(appId).BuildFilter(false).Filter!; - var expectation = Cleanup(expected, arg); + var rendered = + filter.Render( + BsonSerializer.SerializerRegistry.GetSerializer(), + BsonSerializer.SerializerRegistry) + .ToString(); - Assert.Equal(expectation, rendered); + Assert.Equal(Cleanup(expected, arg), rendered); } private void AssertSorting(string expected, params SortNode[] sort) @@ -251,14 +251,16 @@ namespace Squidex.Domain.Apps.Entities.Contents.MongoDb A.CallTo(() => cursor.Sort(A>._)) .Invokes((SortDefinition sortDefinition) => { - rendered = sortDefinition.Render(Serializer, Registry).ToString(); + rendered = + sortDefinition.Render( + BsonSerializer.SerializerRegistry.GetSerializer(), + BsonSerializer.SerializerRegistry) + .ToString(); }); cursor.QuerySort(new ClrQuery { Sort = sort.ToList() }.AdjustToModel(appId)); - var expectation = Cleanup(expected); - - Assert.Equal(expectation, rendered); + Assert.Equal(Cleanup(expected), rendered); } private static string Cleanup(string filter, object? arg = null) diff --git a/backend/tests/Squidex.Domain.Apps.Entities.Tests/Contents/MongoDb/ContentsQueryFixture.cs b/backend/tests/Squidex.Domain.Apps.Entities.Tests/Contents/MongoDb/ContentsQueryFixture.cs index 3fa9de265..4881caafd 100644 --- a/backend/tests/Squidex.Domain.Apps.Entities.Tests/Contents/MongoDb/ContentsQueryFixture.cs +++ b/backend/tests/Squidex.Domain.Apps.Entities.Tests/Contents/MongoDb/ContentsQueryFixture.cs @@ -33,7 +33,7 @@ namespace Squidex.Domain.Apps.Entities.Contents.MongoDb { private readonly Random random = new Random(); private readonly int numValues = 10000; - private readonly IMongoClient mongoClient = new MongoClient("mongodb://localhost"); + private readonly IMongoClient mongoClient; private readonly IMongoDatabase mongoDatabase; public MongoContentRepository ContentRepository { get; } @@ -55,7 +55,8 @@ namespace Squidex.Domain.Apps.Entities.Contents.MongoDb public ContentsQueryFixture() { - mongoDatabase = mongoClient.GetDatabase("Squidex_Testing"); + mongoClient = new MongoClient(TestConfig.Configuration["mongodb:configuration"]); + mongoDatabase = mongoClient.GetDatabase(TestConfig.Configuration["mongodb:database"]); SetupJson(); diff --git a/backend/tests/Squidex.Domain.Apps.Entities.Tests/Contents/Text/AtlasParsingTests.cs b/backend/tests/Squidex.Domain.Apps.Entities.Tests/Contents/Text/AtlasParsingTests.cs new file mode 100644 index 000000000..91facff96 --- /dev/null +++ b/backend/tests/Squidex.Domain.Apps.Entities.Tests/Contents/Text/AtlasParsingTests.cs @@ -0,0 +1,423 @@ +// ========================================================================== +// Squidex Headless CMS +// ========================================================================== +// Copyright (c) Squidex UG (haftungsbeschraenkt) +// All rights reserved. Licensed under the MIT license. +// ========================================================================== + +using System.IO; +using System.Text; +using System.Text.Json; +using Lucene.Net.Analysis.Standard; +using Lucene.Net.Analysis.Util; +using Lucene.Net.Util; +using MongoDB.Bson; +using Squidex.Domain.Apps.Entities.MongoDb.Text; +using Xunit; +using LuceneQueryAnalyzer = Lucene.Net.QueryParsers.Classic.QueryParser; + +namespace Squidex.Domain.Apps.Entities.Contents.Text +{ + public class AtlasParsingTests + { + private static readonly LuceneQueryVisitor QueryVisitor = new LuceneQueryVisitor(); + private static readonly LuceneQueryAnalyzer QueryParser = + new LuceneQueryAnalyzer(LuceneVersion.LUCENE_48, "*", + new StandardAnalyzer(LuceneVersion.LUCENE_48, CharArraySet.EMPTY_SET)); + private static readonly JsonSerializerOptions JsonSerializerOptions = new JsonSerializerOptions + { + WriteIndented = true + }; + + [Fact] + public void Should_parse_term_query() + { + var actual = ParseQuery("hello"); + + var expected = CreateQuery(new + { + text = new + { + path = new + { + wildcard = "*" + }, + query = "hello" + } + }); + + Assert.Equal(expected, actual); + } + + [Fact] + public void Should_parse_phrase_query() + { + var actual = ParseQuery("\"hello dolly\""); + + var expected = CreateQuery(new + { + phrase = new + { + path = new + { + wildcard = "*" + }, + query = new[] { "hello", "dolly" } + } + }); + + Assert.Equal(expected, actual); + } + + [Fact] + public void Should_parse_compound_phrase_query() + { + var actual = ParseQuery("title:\"The Right Way\" AND text:go"); + + var expected = CreateQuery(new + { + compound = new + { + must = new object[] + { + new + { + phrase = new + { + path = "title", + query = new[] + { + "the", + "right", + "way" + } + } + }, + new + { + text = new + { + path = "text", + query = "go" + } + } + } + } + }); + + Assert.Equal(expected, actual); + } + + [Fact] + public void Should_parse_compound_phrase_query_with_widldcard() + { + var actual = ParseQuery("title:\"Do it right\" AND right"); + + var expected = CreateQuery(new + { + compound = new + { + must = new object[] + { + new + { + phrase = new + { + path = "title", + query = new[] + { + "do", + "it", + "right" + } + } + }, + new + { + text = new + { + path = new + { + wildcard = "*" + }, + query = "right" + } + } + } + } + }); + + Assert.Equal(expected, actual); + } + + [Fact] + public void Should_parse_wildcard_query() + { + var actual = ParseQuery("te?t"); + + var expected = CreateQuery(new + { + wildcard = new + { + path = new + { + wildcard = "*" + }, + query = "te?t" + } + }); + + Assert.Equal(expected, actual); + } + + [Fact] + public void Should_parse_prefix_query() + { + var actual = ParseQuery("test*"); + + var expected = CreateQuery(new + { + wildcard = new + { + path = new + { + wildcard = "*" + }, + query = "test*" + } + }); + + Assert.Equal(expected, actual); + } + + [Fact] + public void Should_parse_fuzzy_query() + { + var actual = ParseQuery("roam~"); + + var expected = CreateQuery(new + { + text = new + { + path = new + { + wildcard = "*" + }, + query = "roam", + fuzzy = new + { + maxEdits = 2 + } + } + }); + + Assert.Equal(expected, actual); + } + + [Fact] + public void Should_parse_fuzzy_query_with_max_edits() + { + var actual = ParseQuery("roam~1"); + + var expected = CreateQuery(new + { + text = new + { + path = new + { + wildcard = "*" + }, + query = "roam", + fuzzy = new + { + maxEdits = 1 + } + } + }); + + Assert.Equal(expected, actual); + } + + [Fact] + public void Should_parse_fuzzy_phrase_query_with_slop() + { + var actual = ParseQuery("\"jakarta apache\"~10"); + + var expected = CreateQuery(new + { + phrase = new + { + path = new + { + wildcard = "*" + }, + query = new[] + { + "jakarta", + "apache" + }, + slop = 10 + } + }); + + Assert.Equal(expected, actual); + } + + [Fact] + public void Should_parse_compound_query_with_brackets() + { + var actual = ParseQuery("(jakarta OR apache) AND website"); + + var expected = CreateQuery(new + { + compound = new + { + must = new object[] + { + new + { + compound = new + { + should = new object[] + { + new + { + text = new + { + path = new + { + wildcard = "*" + }, + query = "jakarta" + } + }, + new + { + text = new + { + path = new + { + wildcard = "*" + }, + query = "apache" + } + } + } + } + }, + new + { + text = new + { + path = new + { + wildcard = "*" + }, + query = "website" + } + } + } + } + }); + + Assert.Equal(expected, actual); + } + + [Fact] + public void Should_parse_compound_query_and_optimize() + { + var actual = ParseQuery("title:(+return +\"pink panther\")"); + + var expected = CreateQuery(new + { + compound = new + { + must = new object[] + { + new + { + text = new + { + path = "title", + query = "return" + } + }, + new + { + phrase = new + { + path = "title", + query = new[] + { + "pink", + "panther" + } + } + } + } + } + }); + + Assert.Equal(expected, actual); + } + + [Fact] + public void Should_parse_range_query() + { + var actual = ParseQuery("mod_date:[20020101 TO 20030101]"); + + var expected = CreateQuery(new + { + range = new + { + path = "mod_date", + gte = 20020101, + lte = 20030101 + } + }); + + Assert.Equal(expected, actual); + } + + [Fact] + public void Should_parse_open_range_query() + { + var actual = ParseQuery("mod_date:{20020101 TO 20030101}"); + + var expected = CreateQuery(new + { + range = new + { + path = "mod_date", + gt = 20020101, + lt = 20030101 + } + }); + + Assert.Equal(expected, actual); + } + + private static object CreateQuery(object query) + { + return JsonSerializer.Serialize(query, JsonSerializerOptions); + } + + private static object ParseQuery(string query) + { + var luceneQuery = QueryParser.Parse(query); + + var rendered = QueryVisitor.Visit(luceneQuery); + + var jsonStream = new MemoryStream(); + var jsonDocument = JsonDocument.Parse(rendered.ToJson()); + + var jsonWriter = new Utf8JsonWriter(jsonStream, new JsonWriterOptions { Indented = true }); + + jsonDocument.WriteTo(jsonWriter); + + jsonWriter.Flush(); + + return Encoding.UTF8.GetString(jsonStream.ToArray()); + } + } +} diff --git a/backend/tests/Squidex.Domain.Apps.Entities.Tests/Contents/Text/AtlasTextIndexFixture.cs b/backend/tests/Squidex.Domain.Apps.Entities.Tests/Contents/Text/AtlasTextIndexFixture.cs new file mode 100644 index 000000000..5b7b66507 --- /dev/null +++ b/backend/tests/Squidex.Domain.Apps.Entities.Tests/Contents/Text/AtlasTextIndexFixture.cs @@ -0,0 +1,38 @@ +// ========================================================================== +// Squidex Headless CMS +// ========================================================================== +// Copyright (c) Squidex UG (haftungsbeschraenkt) +// All rights reserved. Licensed under the MIT license. +// ========================================================================== + +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.Options; +using MongoDB.Driver; +using Newtonsoft.Json; +using Squidex.Domain.Apps.Core.TestHelpers; +using Squidex.Domain.Apps.Entities.MongoDb.Text; +using Squidex.Domain.Apps.Entities.TestHelpers; +using Squidex.Infrastructure.MongoDb; + +namespace Squidex.Domain.Apps.Entities.Contents.Text +{ + public sealed class AtlasTextIndexFixture + { + public AtlasTextIndex Index { get; } + + public AtlasTextIndexFixture() + { + BsonJsonConvention.Register(JsonSerializer.Create(TestUtils.CreateSerializerSettings())); + + DomainIdSerializer.Register(); + + var mongoClient = new MongoClient(TestConfig.Configuration["atlas:configuration"]); + var mongoDatabase = mongoClient.GetDatabase(TestConfig.Configuration["atlas:database"]); + + var options = TestConfig.Configuration.GetSection("atlas").Get(); + + Index = new AtlasTextIndex(mongoDatabase, Options.Create(options), false); + Index.InitializeAsync(default).Wait(); + } + } +} diff --git a/backend/tests/Squidex.Domain.Apps.Entities.Tests/Contents/Text/AtlasTextIndexTests.cs b/backend/tests/Squidex.Domain.Apps.Entities.Tests/Contents/Text/AtlasTextIndexTests.cs new file mode 100644 index 000000000..21eb50abd --- /dev/null +++ b/backend/tests/Squidex.Domain.Apps.Entities.Tests/Contents/Text/AtlasTextIndexTests.cs @@ -0,0 +1,62 @@ +// ========================================================================== +// Squidex Headless CMS +// ========================================================================== +// Copyright (c) Squidex UG (haftungsbeschraenkt) +// All rights reserved. Licensed under the MIT license. +// ========================================================================== + +using System.Threading.Tasks; +using Xunit; + +#pragma warning disable SA1300 // Element should begin with upper-case letter + +namespace Squidex.Domain.Apps.Entities.Contents.Text +{ + [Trait("Category", "Dependencies")] + public class AtlasTextIndexTests : TextIndexerTestsBase, IClassFixture + { + public override bool SupportsQuerySyntax => true; + + public override bool SupportsGeo => true; + + public override int WaitAfterUpdate => 2000; + + public AtlasTextIndexFixture _ { get; } + + public AtlasTextIndexTests(AtlasTextIndexFixture fixture) + { + _ = fixture; + } + + public override ITextIndex CreateIndex() + { + return _.Index; + } + + [Fact] + public async Task Should_retrieve_english_stopword_only_for_german_query() + { + await CreateTextAsync(ids1[0], "de", "and und"); + await CreateTextAsync(ids2[0], "en", "and und"); + + await SearchText(expected: ids2, text: "und"); + } + + [Fact] + public async Task Should_retrieve_german_stopword_only_for_english_query() + { + await CreateTextAsync(ids1[0], "de", "and und"); + await CreateTextAsync(ids2[0], "en", "and und"); + + await SearchText(expected: ids1, text: "and"); + } + + [Fact] + public async Task Should_index_cjk_content_and_retrieve() + { + await CreateTextAsync(ids1[0], "zh", "東京大学"); + + await SearchText(expected: ids1, text: "東京"); + } + } +} diff --git a/backend/tests/Squidex.Domain.Apps.Entities.Tests/Contents/Text/AzureTextIndexFixture.cs b/backend/tests/Squidex.Domain.Apps.Entities.Tests/Contents/Text/AzureTextIndexFixture.cs new file mode 100644 index 000000000..02476e645 --- /dev/null +++ b/backend/tests/Squidex.Domain.Apps.Entities.Tests/Contents/Text/AzureTextIndexFixture.cs @@ -0,0 +1,26 @@ +// ========================================================================== +// Squidex Headless CMS +// ========================================================================== +// Copyright (c) Squidex UG (haftungsbeschraenkt) +// All rights reserved. Licensed under the MIT license. +// ========================================================================== + +using Squidex.Domain.Apps.Entities.TestHelpers; +using Squidex.Extensions.Text.Azure; + +namespace Squidex.Domain.Apps.Entities.Contents.Text +{ + public sealed class AzureTextIndexFixture + { + public AzureTextIndex Index { get; } + + public AzureTextIndexFixture() + { + Index = new AzureTextIndex( + TestConfig.Configuration["azureText:serviceEndpoint"], + TestConfig.Configuration["azureText:apiKey"], + TestConfig.Configuration["azureText:indexName"]); + Index.InitializeAsync(default).Wait(); + } + } +} diff --git a/backend/tests/Squidex.Domain.Apps.Entities.Tests/Contents/Text/TextIndexerTests_Elastic.cs b/backend/tests/Squidex.Domain.Apps.Entities.Tests/Contents/Text/AzureTextIndexTests.cs similarity index 75% rename from backend/tests/Squidex.Domain.Apps.Entities.Tests/Contents/Text/TextIndexerTests_Elastic.cs rename to backend/tests/Squidex.Domain.Apps.Entities.Tests/Contents/Text/AzureTextIndexTests.cs index d45db4694..9551ccaf3 100644 --- a/backend/tests/Squidex.Domain.Apps.Entities.Tests/Contents/Text/TextIndexerTests_Elastic.cs +++ b/backend/tests/Squidex.Domain.Apps.Entities.Tests/Contents/Text/AzureTextIndexTests.cs @@ -6,21 +6,29 @@ // ========================================================================== using System.Threading.Tasks; -using Squidex.Extensions.Text.ElasticSearch; using Xunit; +#pragma warning disable SA1300 // Element should begin with upper-case letter + namespace Squidex.Domain.Apps.Entities.Contents.Text { [Trait("Category", "Dependencies")] - public class TextIndexerTests_Elastic : TextIndexerTestsBase + public class AzureTextIndexTests : TextIndexerTestsBase, IClassFixture { - public override ITextIndex CreateIndex() - { - var index = new ElasticSearchTextIndex("http://localhost:9200", "squidex", 1000); + public override bool SupportsGeo => true; - index.InitializeAsync(default).Wait(); + public override int WaitAfterUpdate => 2000; - return index; + public AzureTextIndexFixture _ { get; } + + public AzureTextIndexTests(AzureTextIndexFixture fixture) + { + _ = fixture; + } + + public override ITextIndex CreateIndex() + { + return _.Index; } [Fact] diff --git a/backend/tests/Squidex.Domain.Apps.Entities.Tests/Contents/Text/ElasticTextIndexFixture.cs b/backend/tests/Squidex.Domain.Apps.Entities.Tests/Contents/Text/ElasticTextIndexFixture.cs new file mode 100644 index 000000000..725393341 --- /dev/null +++ b/backend/tests/Squidex.Domain.Apps.Entities.Tests/Contents/Text/ElasticTextIndexFixture.cs @@ -0,0 +1,25 @@ +// ========================================================================== +// Squidex Headless CMS +// ========================================================================== +// Copyright (c) Squidex UG (haftungsbeschraenkt) +// All rights reserved. Licensed under the MIT license. +// ========================================================================== + +using Squidex.Domain.Apps.Entities.TestHelpers; +using Squidex.Extensions.Text.ElasticSearch; + +namespace Squidex.Domain.Apps.Entities.Contents.Text +{ + public sealed class ElasticTextIndexFixture + { + public ElasticSearchTextIndex Index { get; } + + public ElasticTextIndexFixture() + { + Index = new ElasticSearchTextIndex( + TestConfig.Configuration["elastic:configuration"], + TestConfig.Configuration["elastic:indexName"]); + Index.InitializeAsync(default).Wait(); + } + } +} diff --git a/backend/tests/Squidex.Domain.Apps.Entities.Tests/Contents/Text/TextIndexerTests_Azure.cs b/backend/tests/Squidex.Domain.Apps.Entities.Tests/Contents/Text/ElasticTextIndexTests.cs similarity index 74% rename from backend/tests/Squidex.Domain.Apps.Entities.Tests/Contents/Text/TextIndexerTests_Azure.cs rename to backend/tests/Squidex.Domain.Apps.Entities.Tests/Contents/Text/ElasticTextIndexTests.cs index edfcbdb57..a3001ddef 100644 --- a/backend/tests/Squidex.Domain.Apps.Entities.Tests/Contents/Text/TextIndexerTests_Azure.cs +++ b/backend/tests/Squidex.Domain.Apps.Entities.Tests/Contents/Text/ElasticTextIndexTests.cs @@ -6,21 +6,29 @@ // ========================================================================== using System.Threading.Tasks; -using Squidex.Extensions.Text.Azure; using Xunit; +#pragma warning disable SA1300 // Element should begin with upper-case letter + namespace Squidex.Domain.Apps.Entities.Contents.Text { [Trait("Category", "Dependencies")] - public class TextIndexerTests_Azure : TextIndexerTestsBase + public class ElasticTextIndexTests : TextIndexerTestsBase, IClassFixture { - public override ITextIndex CreateIndex() - { - var index = new AzureTextIndex("https://squidex.search.windows.net", "API_KEY", "test", 2000); + public override bool SupportsGeo => true; - index.InitializeAsync(default).Wait(); + public override int WaitAfterUpdate => 2000; - return index; + public ElasticTextIndexFixture _ { get; } + + public ElasticTextIndexTests(ElasticTextIndexFixture fixture) + { + _ = fixture; + } + + public override ITextIndex CreateIndex() + { + return _.Index; } [Fact] diff --git a/backend/tests/Squidex.Domain.Apps.Entities.Tests/Contents/Text/MongoTextIndexFixture.cs b/backend/tests/Squidex.Domain.Apps.Entities.Tests/Contents/Text/MongoTextIndexFixture.cs new file mode 100644 index 000000000..e20ed859a --- /dev/null +++ b/backend/tests/Squidex.Domain.Apps.Entities.Tests/Contents/Text/MongoTextIndexFixture.cs @@ -0,0 +1,34 @@ +// ========================================================================== +// Squidex Headless CMS +// ========================================================================== +// Copyright (c) Squidex UG (haftungsbeschraenkt) +// All rights reserved. Licensed under the MIT license. +// ========================================================================== + +using MongoDB.Driver; +using Newtonsoft.Json; +using Squidex.Domain.Apps.Core.TestHelpers; +using Squidex.Domain.Apps.Entities.MongoDb.Text; +using Squidex.Domain.Apps.Entities.TestHelpers; +using Squidex.Infrastructure.MongoDb; + +namespace Squidex.Domain.Apps.Entities.Contents.Text +{ + public sealed class MongoTextIndexFixture + { + public MongoTextIndex Index { get; } + + public MongoTextIndexFixture() + { + BsonJsonConvention.Register(JsonSerializer.Create(TestUtils.CreateSerializerSettings())); + + DomainIdSerializer.Register(); + + var mongoClient = new MongoClient(TestConfig.Configuration["mongodb:configuration"]); + var mongoDatabase = mongoClient.GetDatabase(TestConfig.Configuration["mongodb:database"]); + + Index = new MongoTextIndex(mongoDatabase, false); + Index.InitializeAsync(default).Wait(); + } + } +} diff --git a/backend/tests/Squidex.Domain.Apps.Entities.Tests/Contents/Text/TextIndexerTests_Mongo.cs b/backend/tests/Squidex.Domain.Apps.Entities.Tests/Contents/Text/MongoTextIndexTests.cs similarity index 67% rename from backend/tests/Squidex.Domain.Apps.Entities.Tests/Contents/Text/TextIndexerTests_Mongo.cs rename to backend/tests/Squidex.Domain.Apps.Entities.Tests/Contents/Text/MongoTextIndexTests.cs index 93f81a8a3..f71ded8d1 100644 --- a/backend/tests/Squidex.Domain.Apps.Entities.Tests/Contents/Text/TextIndexerTests_Mongo.cs +++ b/backend/tests/Squidex.Domain.Apps.Entities.Tests/Contents/Text/MongoTextIndexTests.cs @@ -8,38 +8,29 @@ using System.Linq; using System.Threading.Tasks; using MongoDB.Driver; -using Newtonsoft.Json; -using Squidex.Domain.Apps.Core.TestHelpers; -using Squidex.Domain.Apps.Entities.MongoDb.FullText; -using Squidex.Infrastructure.MongoDb; using Xunit; +#pragma warning disable SA1300 // Element should begin with upper-case letter + namespace Squidex.Domain.Apps.Entities.Contents.Text { [Trait("Category", "Dependencies")] - public class TextIndexerTests_Mongo : TextIndexerTestsBase + public class MongoTextIndexTests : TextIndexerTestsBase, IClassFixture { public override bool SupportsQuerySyntax => false; public override bool SupportsGeo => true; - static TextIndexerTests_Mongo() - { - BsonJsonConvention.Register(JsonSerializer.Create(TestUtils.CreateSerializerSettings())); + public MongoTextIndexFixture _ { get; } - DomainIdSerializer.Register(); + public MongoTextIndexTests(MongoTextIndexFixture fixture) + { + _ = fixture; } public override ITextIndex CreateIndex() { - var mongoClient = new MongoClient("mongodb://localhost"); - var mongoDatabase = mongoClient.GetDatabase("Squidex_Testing"); - - var index = new MongoTextIndex(mongoDatabase, false); - - index.InitializeAsync(default).Wait(); - - return index; + return _.Index; } [Fact] diff --git a/backend/tests/Squidex.Domain.Apps.Entities.Tests/Contents/Text/QueryParserTests.cs b/backend/tests/Squidex.Domain.Apps.Entities.Tests/Contents/Text/QueryParserTests.cs new file mode 100644 index 000000000..0ca27ecbd --- /dev/null +++ b/backend/tests/Squidex.Domain.Apps.Entities.Tests/Contents/Text/QueryParserTests.cs @@ -0,0 +1,48 @@ +// ========================================================================== +// Squidex Headless CMS +// ========================================================================== +// Copyright (c) Squidex UG (haftungsbeschraenkt) +// All rights reserved. Licensed under the MIT license. +// ========================================================================== + +using Xunit; + +namespace Squidex.Domain.Apps.Entities.Contents.Text +{ + public class QueryParserTests + { + private readonly QueryParser sut = new QueryParser(x => $"texts.{x}"); + + [Fact] + public void Should_prefix_field_query() + { + var source = "en:Hello"; + + Assert.Equal("texts.en:Hello", sut.Parse(source)?.Text); + } + + [Fact] + public void Should_prefix_field_with_complex_language() + { + var source = "en-EN:Hello"; + + Assert.Equal("texts.en-EN:Hello", sut.Parse(source)?.Text); + } + + [Fact] + public void Should_prefix_field_query_within_query() + { + var source = "Hello en:World"; + + Assert.Equal("Hello texts.en:World", sut.Parse(source)?.Text); + } + + [Fact] + public void Should_prefix_field_query_within_complex_query() + { + var source = "Hallo OR (Hello en:World)"; + + Assert.Equal("Hallo OR (Hello texts.en:World)", sut.Parse(source)?.Text); + } + } +} diff --git a/backend/tests/Squidex.Domain.Apps.Entities.Tests/Contents/Text/TextIndexerTestsBase.cs b/backend/tests/Squidex.Domain.Apps.Entities.Tests/Contents/Text/TextIndexerTestsBase.cs index e6290d1f7..7664a095a 100644 --- a/backend/tests/Squidex.Domain.Apps.Entities.Tests/Contents/Text/TextIndexerTestsBase.cs +++ b/backend/tests/Squidex.Domain.Apps.Entities.Tests/Contents/Text/TextIndexerTestsBase.cs @@ -5,6 +5,7 @@ // All rights reserved. Licensed under the MIT license. // ========================================================================== +using System; using System.Collections.Generic; using System.Linq; using System.Threading.Tasks; @@ -32,12 +33,19 @@ namespace Squidex.Domain.Apps.Entities.Contents.Text private readonly NamedId appId = NamedId.Of(DomainId.NewGuid(), "my-app"); private readonly NamedId schemaId = NamedId.Of(DomainId.NewGuid(), "my-schema"); private readonly IAppEntity app; - private readonly TextIndexingProcess sut; + private readonly Lazy sut; + + protected TextIndexingProcess Sut + { + get { return sut.Value; } + } public virtual bool SupportsQuerySyntax => true; public virtual bool SupportsGeo => false; + public virtual int WaitAfterUpdate => 0; + protected TextIndexerTestsBase() { app = @@ -45,58 +53,65 @@ namespace Squidex.Domain.Apps.Entities.Contents.Text Language.DE, Language.EN); -#pragma warning disable MA0056 // Do not call overridable members in constructor - sut = new TextIndexingProcess(TestUtils.DefaultSerializer, CreateIndex(), new InMemoryTextIndexerState()); -#pragma warning restore MA0056 // Do not call overridable members in constructor + sut = new Lazy(CreateSut); } - public abstract ITextIndex CreateIndex(); - - [SkippableFact] - public async Task Should_index_invariant_content_and_retrieve_with_fuzzy() + private TextIndexingProcess CreateSut() { - Skip.IfNot(SupportsQuerySyntax); + var index = CreateIndex(); - await CreateTextAsync(ids1[0], "iv", "Hello"); - - await SearchText(expected: ids1, text: "helo~"); + return new TextIndexingProcess(TestUtils.DefaultSerializer, index, new InMemoryTextIndexerState()); } - [SkippableFact] - public async Task Should_index_invariant_content_and_retrieve_with_fuzzy_with_full_scope() + public abstract ITextIndex CreateIndex(); + + [Fact] + public async Task Should_search_with_fuzzy() { - Skip.IfNot(SupportsQuerySyntax); + if (!SupportsQuerySyntax) + { + return; + } - await CreateTextAsync(ids2[0], "iv", "World"); + await CreateTextAsync(ids1[0], "iv", "Hello"); - await SearchText(expected: ids2, text: "wold~", SearchScope.All); + await SearchText(expected: ids1, text: "helo~"); } - [SkippableFact] + [Fact] public async Task Should_search_by_field() { - Skip.IfNot(SupportsQuerySyntax); + if (!SupportsQuerySyntax) + { + return; + } await CreateTextAsync(ids1[0], "en", "City"); await SearchText(expected: ids1, text: "en:city"); } - [SkippableFact] + [Fact] public async Task Should_search_by_geo() { - Skip.IfNot(SupportsGeo); + if (!SupportsGeo) + { + return; + } + + var field = Guid.NewGuid().ToString(); - // Within radius - await CreateGeoAsync(ids1[0], "geo", 51.343391192211506, 12.401476788622826); + // Within search radius + await CreateGeoAsync(ids1[0], field, 51.343391192211506, 12.401476788622826); - // Not in radius - await CreateGeoAsync(ids2[0], "geo", 51.30765141427311, 12.379631713912486); + // Outside of search radius + await CreateGeoAsync(ids2[0], field, 51.30765141427311, 12.379631713912486); - await SearchGeo(expected: ids1, "geo.iv", 51.34641682574934, 12.401965298137707); + // Within search radius and correct field. + await SearchGeo(expected: ids1, $"{field}.iv", 51.34641682574934, 12.401965298137707); - // Wrong field - await SearchGeo(expected: null, "abc.iv", 51.48596429889613, 12.102629469505713); + // Within search radius but incorrect field. + await SearchGeo(expected: null, "other.iv", 51.48596429889613, 12.102629469505713); } [Fact] @@ -346,13 +361,15 @@ namespace Squidex.Domain.Apps.Entities.Contents.Text return UpdateAsync(id, new ContentDeleted()); } - private Task UpdateAsync(DomainId id, ContentEvent contentEvent) + private async Task UpdateAsync(DomainId id, ContentEvent contentEvent) { contentEvent.ContentId = id; contentEvent.AppId = appId; contentEvent.SchemaId = schemaId; - return sut.On(Enumerable.Repeat(Envelope.Create(contentEvent), 1)); + await Sut.On(Enumerable.Repeat(Envelope.Create(contentEvent), 1)); + + await Task.Delay(WaitAfterUpdate); } private static ContentData TextData(string language, string text) @@ -375,7 +392,7 @@ namespace Squidex.Domain.Apps.Entities.Contents.Text { var query = new GeoQuery(schemaId.Id, field, latitude, longitude, 1000, 1000); - var result = await sut.TextIndex.SearchAsync(app, query, target); + var result = await Sut.TextIndex.SearchAsync(app, query, target); if (expected != null) { @@ -394,7 +411,7 @@ namespace Squidex.Domain.Apps.Entities.Contents.Text RequiredSchemaIds = new List { schemaId.Id } }; - var result = await sut.TextIndex.SearchAsync(app, query, target); + var result = await Sut.TextIndex.SearchAsync(app, query, target); if (expected != null) { diff --git a/backend/tests/Squidex.Domain.Apps.Entities.Tests/Schemas/MongoDb/SchemasHashFixture.cs b/backend/tests/Squidex.Domain.Apps.Entities.Tests/Schemas/MongoDb/SchemasHashFixture.cs index 258d15a42..4c313b5fb 100644 --- a/backend/tests/Squidex.Domain.Apps.Entities.Tests/Schemas/MongoDb/SchemasHashFixture.cs +++ b/backend/tests/Squidex.Domain.Apps.Entities.Tests/Schemas/MongoDb/SchemasHashFixture.cs @@ -8,22 +8,21 @@ using System.Threading.Tasks; using MongoDB.Driver; using Squidex.Domain.Apps.Entities.MongoDb.Schemas; +using Squidex.Domain.Apps.Entities.TestHelpers; using Squidex.Infrastructure.MongoDb; namespace Squidex.Domain.Apps.Entities.Schemas.MongoDb { public sealed class SchemasHashFixture { - private readonly IMongoClient mongoClient = new MongoClient("mongodb://localhost"); - private readonly IMongoDatabase mongoDatabase; - public MongoSchemasHash SchemasHash { get; } public SchemasHashFixture() { InstantSerializer.Register(); - mongoDatabase = mongoClient.GetDatabase("Squidex_Testing"); + var mongoClient = new MongoClient(TestConfig.Configuration["mongodb:configuration"]); + var mongoDatabase = mongoClient.GetDatabase(TestConfig.Configuration["mongodb:database"]); var schemasHash = new MongoSchemasHash(mongoDatabase); diff --git a/backend/tests/Squidex.Domain.Apps.Entities.Tests/Squidex.Domain.Apps.Entities.Tests.csproj b/backend/tests/Squidex.Domain.Apps.Entities.Tests/Squidex.Domain.Apps.Entities.Tests.csproj index 53c45dd95..3049532c3 100644 --- a/backend/tests/Squidex.Domain.Apps.Entities.Tests/Squidex.Domain.Apps.Entities.Tests.csproj +++ b/backend/tests/Squidex.Domain.Apps.Entities.Tests/Squidex.Domain.Apps.Entities.Tests.csproj @@ -41,7 +41,6 @@ all runtime; build; native; contentfiles; analyzers - diff --git a/backend/tests/Squidex.Domain.Apps.Entities.Tests/TestHelpers/TestConfig.cs b/backend/tests/Squidex.Domain.Apps.Entities.Tests/TestHelpers/TestConfig.cs new file mode 100644 index 000000000..3b9ecbfd8 --- /dev/null +++ b/backend/tests/Squidex.Domain.Apps.Entities.Tests/TestHelpers/TestConfig.cs @@ -0,0 +1,29 @@ +// ========================================================================== +// Squidex Headless CMS +// ========================================================================== +// Copyright (c) Squidex UG (haftungsbeschraenkt) +// All rights reserved. Licensed under the MIT license. +// ========================================================================== + +using System.IO; +using Microsoft.Extensions.Configuration; + +namespace Squidex.Domain.Apps.Entities.TestHelpers +{ + public static class TestConfig + { + public static IConfiguration Configuration { get; } + + static TestConfig() + { + var basePath = Path.GetFullPath("../../../"); + + Configuration = new ConfigurationBuilder() + .SetBasePath(basePath) + .AddJsonFile("appsettings.json", true) + .AddJsonFile("appsettings.Development.json", true) + .AddEnvironmentVariables() + .Build(); + } + } +} diff --git a/backend/tests/Squidex.Domain.Apps.Entities.Tests/appSettings.json b/backend/tests/Squidex.Domain.Apps.Entities.Tests/appSettings.json new file mode 100644 index 000000000..33e6f5785 --- /dev/null +++ b/backend/tests/Squidex.Domain.Apps.Entities.Tests/appSettings.json @@ -0,0 +1,53 @@ +{ + "mongoDb": { + // The connection string to your Mongo Server. + // + // Read More: https://docs.mongodb.com/manual/reference/connection-string/ + "configuration": "mongodb://localhost", + + // The name of the event store database. + "database": "Squidex_Testing" + }, + "atlas": { + // The connection string to your Mongo Server. + // + // Read More: https://docs.mongodb.com/manual/reference/connection-string/ + "configuration": "mongodb://localhost", + + // The name of the event store database. + "database": "Squidex_Testing", + + // The organization id. + "groupId": "", + + // The name of the cluster. + "clusterName": "", + + // Credentials to your account. + "publicKey": "", + "privateKey": "", + + "fullTextEnabled": true + }, + "elastic": { + // The configuration to your elastic search cluster. + // + // Read More: https://www.elastic.co/guide/en/elasticsearch/client/javascript-api/current/client-configuration.html + "configuration": "http://localhost:9200", + + // The name of the test index. + "indexName": "test" + }, + "azureText": { + // The URL to your azure search instance. + // + // Read More: https://docs.microsoft.com/en-us/azure/search/search-create-service-portal#get-a-key-and-url-endpoint + "serviceEndpoint": "https://.search.windows.net", + + // The api key. See link above. + "apiKey": "", + + // The name of the index. + "indexName": "test" + } +} \ No newline at end of file diff --git a/backend/tests/Squidex.Infrastructure.Tests/Email/SmtpEmailSenderTests.cs b/backend/tests/Squidex.Infrastructure.Tests/Email/SmtpEmailSenderTests.cs index 9fbba778f..52131b37a 100644 --- a/backend/tests/Squidex.Infrastructure.Tests/Email/SmtpEmailSenderTests.cs +++ b/backend/tests/Squidex.Infrastructure.Tests/Email/SmtpEmailSenderTests.cs @@ -5,8 +5,11 @@ // All rights reserved. Licensed under the MIT license. // ========================================================================== +using System.Threading; using System.Threading.Tasks; +using Microsoft.Extensions.Configuration; using Microsoft.Extensions.Options; +using Squidex.Infrastructure.TestHelpers; using Xunit; namespace Squidex.Infrastructure.Email @@ -17,18 +20,19 @@ namespace Squidex.Infrastructure.Email [Fact] public async Task Should_handle_timeout_properly() { - var sut = new SmtpEmailSender(Options.Create(new SmtpOptions - { - Sender = "sebastian@squidex.io", - Server = "invalid", - Timeout = 1000 - })); + var options = TestConfig.Configuration.GetSection("email:smtp").Get(); + + var recipient = TestConfig.Configuration["email:smtp:recipient"]; - var timer = Task.Delay(5000); + var testSubject = TestConfig.Configuration["email:smtp:testSubject"]; + var testBody = TestConfig.Configuration["email:smtp:testBody"]; - var result = await Task.WhenAny(timer, sut.SendAsync("hello@squidex.io", "TEST", "TEST")); + var sut = new SmtpEmailSender(Options.Create(options)); - Assert.NotSame(timer, result); + using (var cts = new CancellationTokenSource(5000)) + { + await sut.SendAsync(recipient, testSubject, testBody, cts.Token); + } } } } diff --git a/backend/tests/Squidex.Infrastructure.Tests/EventSourcing/GetEventStoreFixture.cs b/backend/tests/Squidex.Infrastructure.Tests/EventSourcing/GetEventStoreFixture.cs index a092dcd88..fc15212bb 100644 --- a/backend/tests/Squidex.Infrastructure.Tests/EventSourcing/GetEventStoreFixture.cs +++ b/backend/tests/Squidex.Infrastructure.Tests/EventSourcing/GetEventStoreFixture.cs @@ -22,7 +22,7 @@ namespace Squidex.Infrastructure.EventSourcing { AppContext.SetSwitch("System.Net.Http.SocketsHttpHandler.Http2UnencryptedSupport", true); - settings = EventStoreClientSettings.Create("esdb://admin:changeit@127.0.0.1:2113?tls=false"); + settings = EventStoreClientSettings.Create(TestConfig.Configuration["eventStore:configuration"]); EventStore = new GetEventStore(settings, TestUtils.DefaultSerializer); EventStore.InitializeAsync(default).Wait(); diff --git a/backend/tests/Squidex.Infrastructure.Tests/EventSourcing/MongoEventStoreFixture.cs b/backend/tests/Squidex.Infrastructure.Tests/EventSourcing/MongoEventStoreFixture.cs index 88b972279..b252984ba 100644 --- a/backend/tests/Squidex.Infrastructure.Tests/EventSourcing/MongoEventStoreFixture.cs +++ b/backend/tests/Squidex.Infrastructure.Tests/EventSourcing/MongoEventStoreFixture.cs @@ -25,7 +25,7 @@ namespace Squidex.Infrastructure.EventSourcing protected MongoEventStoreFixture(string connectionString) { mongoClient = new MongoClient(connectionString); - mongoDatabase = mongoClient.GetDatabase("Squidex_Testing"); + mongoDatabase = mongoClient.GetDatabase(TestConfig.Configuration["mongodb:database"]); BsonJsonConvention.Register(JsonSerializer.Create(TestUtils.DefaultSettings())); @@ -49,7 +49,7 @@ namespace Squidex.Infrastructure.EventSourcing public sealed class MongoEventStoreDirectFixture : MongoEventStoreFixture { public MongoEventStoreDirectFixture() - : base("mongodb://localhost:27019") + : base(TestConfig.Configuration["mongodb:configuration"]) { } } @@ -57,7 +57,7 @@ namespace Squidex.Infrastructure.EventSourcing public sealed class MongoEventStoreReplicaSetFixture : MongoEventStoreFixture { public MongoEventStoreReplicaSetFixture() - : base("mongodb://localhost:27017") + : base(TestConfig.Configuration["mongodb:configurationReplica"]) { } } diff --git a/backend/tests/Squidex.Infrastructure.Tests/MongoDb/MongoQueryTests.cs b/backend/tests/Squidex.Infrastructure.Tests/MongoDb/MongoQueryTests.cs index 15d14c962..aa060708a 100644 --- a/backend/tests/Squidex.Infrastructure.Tests/MongoDb/MongoQueryTests.cs +++ b/backend/tests/Squidex.Infrastructure.Tests/MongoDb/MongoQueryTests.cs @@ -23,9 +23,6 @@ namespace Squidex.Infrastructure.MongoDb { public class MongoQueryTests { - private readonly IBsonSerializerRegistry registry = BsonSerializer.SerializerRegistry; - private readonly IBsonSerializer serializer = BsonSerializer.SerializerRegistry.GetSerializer(); - public class TestEntity { public DomainId Id { get; set; } @@ -322,18 +319,20 @@ namespace Squidex.Infrastructure.MongoDb AssertQuery(new ClrQuery { Filter = filter }, expected, arg); } - private void AssertQuery(ClrQuery query, string expected, object? arg = null) + private static void AssertQuery(ClrQuery query, string expected, object? arg = null) { - var rendered = - query.BuildFilter().Filter! - .Render(serializer, registry).ToString(); + var filter = query.BuildFilter().Filter!; - var expectation = Cleanup(expected, arg); + var rendered = + filter.Render( + BsonSerializer.SerializerRegistry.GetSerializer(), + BsonSerializer.SerializerRegistry) + .ToString(); - Assert.Equal(expectation, rendered); + Assert.Equal(Cleanup(expected, arg), rendered); } - private void AssertSorting(string expected, params SortNode[] sort) + private static void AssertSorting(string expected, params SortNode[] sort) { var cursor = A.Fake>(); @@ -342,14 +341,16 @@ namespace Squidex.Infrastructure.MongoDb A.CallTo(() => cursor.Sort(A>._)) .Invokes((SortDefinition sortDefinition) => { - rendered = sortDefinition.Render(serializer, registry).ToString(); + rendered = + sortDefinition.Render( + BsonSerializer.SerializerRegistry.GetSerializer(), + BsonSerializer.SerializerRegistry) + .ToString(); }); cursor.QuerySort(new ClrQuery { Sort = sort.ToList() }); - var expectation = Cleanup(expected); - - Assert.Equal(expectation, rendered); + Assert.Equal(Cleanup(expected), rendered); } private static string Cleanup(string filter, object? arg = null) diff --git a/backend/tests/Squidex.Infrastructure.Tests/TestHelpers/TestConfig.cs b/backend/tests/Squidex.Infrastructure.Tests/TestHelpers/TestConfig.cs new file mode 100644 index 000000000..082cf86ca --- /dev/null +++ b/backend/tests/Squidex.Infrastructure.Tests/TestHelpers/TestConfig.cs @@ -0,0 +1,29 @@ +// ========================================================================== +// Squidex Headless CMS +// ========================================================================== +// Copyright (c) Squidex UG (haftungsbeschraenkt) +// All rights reserved. Licensed under the MIT license. +// ========================================================================== + +using System.IO; +using Microsoft.Extensions.Configuration; + +namespace Squidex.Infrastructure.TestHelpers +{ + public static class TestConfig + { + public static IConfiguration Configuration { get; } + + static TestConfig() + { + var basePath = Path.GetFullPath("../../../"); + + Configuration = new ConfigurationBuilder() + .SetBasePath(basePath) + .AddJsonFile("appsettings.json", true) + .AddJsonFile("appsettings.Development.json", true) + .AddEnvironmentVariables() + .Build(); + } + } +} diff --git a/backend/tests/Squidex.Infrastructure.Tests/appSettings.json b/backend/tests/Squidex.Infrastructure.Tests/appSettings.json new file mode 100644 index 000000000..40ea78339 --- /dev/null +++ b/backend/tests/Squidex.Infrastructure.Tests/appSettings.json @@ -0,0 +1,15 @@ +{ + "mongoDb": { + // The connection string to your Mongo Server. + // + // Read More: https://docs.mongodb.com/manual/reference/connection-string/ + "configuration": "mongodb://localhost", + "configurationReplica": "mongodb://localhost", + + // The name of the event store database. + "database": "Squidex_Testing" + }, + "eventStore": { + "configuration": "esdb://admin:changeit@127.0.0.1:2113?tls=false" + } +} \ No newline at end of file