Browse Source

Feature/better search (#792)

* Azure cognitive search.

* Azure search finalized.

* More fixes.

* Remove api key.

* Document configuration options.

* Reverts the schema-id field.

* Geosearch tests.

* Elastic geo search support.

* Delete old file.

* Better support for query syntax in elastic.

* Temp

* More fixes.

* Code simplified.

* Add settings to json file.

* Fix tests
pull/793/head
Sebastian Stehle 4 years ago
committed by GitHub
parent
commit
a6d00c3c7e
No known key found for this signature in database GPG Key ID: 4AEE18F83AFDEB23
  1. 31
      backend/extensions/Squidex.Extensions/Text/Azure/AzureIndexDefinition.cs
  2. 82
      backend/extensions/Squidex.Extensions/Text/Azure/AzureTextIndex.cs
  3. 82
      backend/extensions/Squidex.Extensions/Text/Azure/CommandFactory.cs
  4. 72
      backend/extensions/Squidex.Extensions/Text/ElasticSearch/CommandFactory.cs
  5. 140
      backend/extensions/Squidex.Extensions/Text/ElasticSearch/ElasticSearchIndexDefinition.cs
  6. 226
      backend/extensions/Squidex.Extensions/Text/ElasticSearch/ElasticSearchMapping.cs
  7. 135
      backend/extensions/Squidex.Extensions/Text/ElasticSearch/ElasticSearchTextIndex.cs
  8. 3
      backend/src/Squidex.Domain.Apps.Entities.MongoDb/Squidex.Domain.Apps.Entities.MongoDb.csproj
  9. 210
      backend/src/Squidex.Domain.Apps.Entities.MongoDb/Text/AtlasIndexDefinition.cs
  10. 31
      backend/src/Squidex.Domain.Apps.Entities.MongoDb/Text/AtlasOptions.cs
  11. 164
      backend/src/Squidex.Domain.Apps.Entities.MongoDb/Text/AtlasTextIndex.cs
  12. 44
      backend/src/Squidex.Domain.Apps.Entities.MongoDb/Text/CommandFactory.cs
  13. 329
      backend/src/Squidex.Domain.Apps.Entities.MongoDb/Text/LuceneQueryVisitor.cs
  14. 57
      backend/src/Squidex.Domain.Apps.Entities.MongoDb/Text/LuceneSearchDefinitionExtensions.cs
  15. 44
      backend/src/Squidex.Domain.Apps.Entities.MongoDb/Text/MongoTextIndex.cs
  16. 92
      backend/src/Squidex.Domain.Apps.Entities.MongoDb/Text/MongoTextIndexBase.cs
  17. 7
      backend/src/Squidex.Domain.Apps.Entities.MongoDb/Text/MongoTextIndexEntity.cs
  18. 2
      backend/src/Squidex.Domain.Apps.Entities.MongoDb/Text/MongoTextIndexEntityText.cs
  19. 2
      backend/src/Squidex.Domain.Apps.Entities.MongoDb/Text/MongoTextIndexerState.cs
  20. 14
      backend/src/Squidex.Domain.Apps.Entities/Contents/Text/Query.cs
  21. 82
      backend/src/Squidex.Domain.Apps.Entities/Contents/Text/QueryParser.cs
  22. 20
      backend/src/Squidex/Config/Domain/StoreServices.cs
  23. 2
      backend/src/Squidex/Squidex.csproj
  24. 17
      backend/src/Squidex/appsettings.json
  25. 24
      backend/tests/Squidex.Domain.Apps.Entities.Tests/Assets/MongoDb/AssetQueryTests.cs
  26. 6
      backend/tests/Squidex.Domain.Apps.Entities.Tests/Assets/MongoDb/AssetsQueryFixture.cs
  27. 24
      backend/tests/Squidex.Domain.Apps.Entities.Tests/Contents/MongoDb/ContentQueryTests.cs
  28. 5
      backend/tests/Squidex.Domain.Apps.Entities.Tests/Contents/MongoDb/ContentsQueryFixture.cs
  29. 423
      backend/tests/Squidex.Domain.Apps.Entities.Tests/Contents/Text/AtlasParsingTests.cs
  30. 38
      backend/tests/Squidex.Domain.Apps.Entities.Tests/Contents/Text/AtlasTextIndexFixture.cs
  31. 62
      backend/tests/Squidex.Domain.Apps.Entities.Tests/Contents/Text/AtlasTextIndexTests.cs
  32. 26
      backend/tests/Squidex.Domain.Apps.Entities.Tests/Contents/Text/AzureTextIndexFixture.cs
  33. 22
      backend/tests/Squidex.Domain.Apps.Entities.Tests/Contents/Text/AzureTextIndexTests.cs
  34. 25
      backend/tests/Squidex.Domain.Apps.Entities.Tests/Contents/Text/ElasticTextIndexFixture.cs
  35. 22
      backend/tests/Squidex.Domain.Apps.Entities.Tests/Contents/Text/ElasticTextIndexTests.cs
  36. 34
      backend/tests/Squidex.Domain.Apps.Entities.Tests/Contents/Text/MongoTextIndexFixture.cs
  37. 25
      backend/tests/Squidex.Domain.Apps.Entities.Tests/Contents/Text/MongoTextIndexTests.cs
  38. 48
      backend/tests/Squidex.Domain.Apps.Entities.Tests/Contents/Text/QueryParserTests.cs
  39. 81
      backend/tests/Squidex.Domain.Apps.Entities.Tests/Contents/Text/TextIndexerTestsBase.cs
  40. 7
      backend/tests/Squidex.Domain.Apps.Entities.Tests/Schemas/MongoDb/SchemasHashFixture.cs
  41. 1
      backend/tests/Squidex.Domain.Apps.Entities.Tests/Squidex.Domain.Apps.Entities.Tests.csproj
  42. 29
      backend/tests/Squidex.Domain.Apps.Entities.Tests/TestHelpers/TestConfig.cs
  43. 53
      backend/tests/Squidex.Domain.Apps.Entities.Tests/appSettings.json
  44. 22
      backend/tests/Squidex.Infrastructure.Tests/Email/SmtpEmailSenderTests.cs
  45. 2
      backend/tests/Squidex.Infrastructure.Tests/EventSourcing/GetEventStoreFixture.cs
  46. 6
      backend/tests/Squidex.Infrastructure.Tests/EventSourcing/MongoEventStoreFixture.cs
  47. 29
      backend/tests/Squidex.Infrastructure.Tests/MongoDb/MongoQueryTests.cs
  48. 29
      backend/tests/Squidex.Infrastructure.Tests/TestHelpers/TestConfig.cs
  49. 15
      backend/tests/Squidex.Infrastructure.Tests/appSettings.json

31
backend/extensions/Squidex.Extensions/Text/Azure/AzureIndexDefinition.cs

@ -15,7 +15,7 @@ namespace Squidex.Extensions.Text.Azure
{ {
public static class AzureIndexDefinition public static class AzureIndexDefinition
{ {
private static readonly Dictionary<string, (string Field, string Analyzer)> AllowedLanguages = new Dictionary<string, (string Field, string Analyzer)>(StringComparer.OrdinalIgnoreCase) private static readonly Dictionary<string, (string Field, string Analyzer)> FieldAnalyzers = new (StringComparer.OrdinalIgnoreCase)
{ {
["iv"] = ("iv", LexicalAnalyzerName.StandardLucene.ToString()), ["iv"] = ("iv", LexicalAnalyzerName.StandardLucene.ToString()),
["zh"] = ("zh", LexicalAnalyzerName.ZhHansLucene.ToString()) ["zh"] = ("zh", LexicalAnalyzerName.ZhHansLucene.ToString())
@ -49,25 +49,30 @@ namespace Squidex.Extensions.Text.Azure
{ {
var fieldName = language.Replace('-', '_'); var fieldName = language.Replace('-', '_');
AllowedLanguages[language] = (fieldName, analyzer); FieldAnalyzers[language] = (fieldName, analyzer);
} }
} }
} }
} }
public static string GetTextField(string key) public static string GetFieldName(string key)
{ {
if (AllowedLanguages.TryGetValue(key, out var field)) if (FieldAnalyzers.TryGetValue(key, out var analyzer))
{ {
return field.Field; return analyzer.Field;
} }
if (key.Length > 2 && AllowedLanguages.TryGetValue(key[2..], out field)) if (key.Length > 0)
{ {
return field.Field; var language = key[2..];
if (FieldAnalyzers.TryGetValue(language, out analyzer))
{
return analyzer.Field;
}
} }
return AllowedLanguages["iv"].Field; return "iv";
} }
public static SearchIndex Create(string indexName) public static SearchIndex Create(string indexName)
@ -103,12 +108,20 @@ namespace Squidex.Extensions.Text.Azure
IsFilterable = true IsFilterable = true
}, },
new SimpleField("servePublished", SearchFieldDataType.Boolean) new SimpleField("servePublished", SearchFieldDataType.Boolean)
{
IsFilterable = true
},
new SimpleField("geoObject", SearchFieldDataType.GeographyPoint)
{
IsFilterable = true
},
new SimpleField("geoField", SearchFieldDataType.String)
{ {
IsFilterable = true IsFilterable = true
} }
}; };
foreach (var (field, analyzer) in AllowedLanguages.Values) foreach (var (field, analyzer) in FieldAnalyzers.Values)
{ {
fields.Add( fields.Add(
new SearchableField(field) new SearchableField(field)

82
backend/extensions/Squidex.Extensions/Text/Azure/AzureTextIndex.cs

@ -26,19 +26,16 @@ namespace Squidex.Extensions.Text.Azure
{ {
private readonly SearchIndexClient indexClient; private readonly SearchIndexClient indexClient;
private readonly SearchClient searchClient; private readonly SearchClient searchClient;
private readonly int waitAfterUpdate; private readonly QueryParser queryParser = new QueryParser(AzureIndexDefinition.GetFieldName);
public AzureTextIndex( public AzureTextIndex(
string serviceEndpoint, string serviceEndpoint,
string serviceApiKey, string serviceApiKey,
string indexName, string indexName)
int waitAfterUpdate = 0)
{ {
indexClient = new SearchIndexClient(new Uri(serviceEndpoint), new AzureKeyCredential(serviceApiKey)); indexClient = new SearchIndexClient(new Uri(serviceEndpoint), new AzureKeyCredential(serviceApiKey));
searchClient = indexClient.GetSearchClient(indexName); searchClient = indexClient.GetSearchClient(indexName);
this.waitAfterUpdate = waitAfterUpdate;
} }
public async Task InitializeAsync( public async Task InitializeAsync(
@ -76,20 +73,19 @@ namespace Squidex.Extensions.Text.Azure
} }
await searchClient.IndexDocumentsAsync(batch, cancellationToken: ct); await searchClient.IndexDocumentsAsync(batch, cancellationToken: ct);
if (waitAfterUpdate > 0)
{
await Task.Delay(waitAfterUpdate, ct);
}
} }
public Task<List<DomainId>> SearchAsync(IAppEntity app, GeoQuery query, SearchScope scope, public async Task<List<DomainId>> SearchAsync(IAppEntity app, GeoQuery query, SearchScope scope,
CancellationToken ct = default) CancellationToken ct = default)
{ {
Guard.NotNull(app, nameof(app)); Guard.NotNull(app, nameof(app));
Guard.NotNull(query, nameof(query)); Guard.NotNull(query, nameof(query));
return Task.FromResult<List<DomainId>>(null); var result = new List<(DomainId Id, double Score)>();
await SearchAsync(result, "*", BuildGeoQuery(query, scope), query.Take, 1, ct);
return result.OrderByDescending(x => x.Score).Select(x => x.Id).Distinct().ToList();
} }
public async Task<List<DomainId>> SearchAsync(IAppEntity app, TextQuery query, SearchScope scope, public async Task<List<DomainId>> SearchAsync(IAppEntity app, TextQuery query, SearchScope scope,
@ -98,57 +94,57 @@ namespace Squidex.Extensions.Text.Azure
Guard.NotNull(app, nameof(app)); Guard.NotNull(app, nameof(app));
Guard.NotNull(query, nameof(query)); Guard.NotNull(query, nameof(query));
if (string.IsNullOrWhiteSpace(query.Text)) var parsed = queryParser.Parse(query.Text);
if (parsed == null)
{ {
return null; return null;
} }
List<(DomainId, double)> documents; var result = new List<(DomainId Id, double Score)>();
if (query.RequiredSchemaIds?.Count > 0) if (query.RequiredSchemaIds?.Count > 0)
{ {
documents = await SearchBySchemaAsync(query.Text, query.RequiredSchemaIds, scope, query.Take, 1, ct); await SearchBySchemaAsync(result, parsed.Text, query.RequiredSchemaIds, scope, query.Take, 1, ct);
} }
else if (query.PreferredSchemaId == null) else if (query.PreferredSchemaId == null)
{ {
documents = await SearchByAppAsync(query.Text, app, scope, query.Take, 1, ct); await SearchByAppAsync(result, parsed.Text, app, scope, query.Take, 1, ct);
} }
else else
{ {
var halfBucket = query.Take / 2; var halfTake = query.Take / 2;
var schemaIds = Enumerable.Repeat(query.PreferredSchemaId.Value, 1); var schemaIds = Enumerable.Repeat(query.PreferredSchemaId.Value, 1);
documents = await SearchBySchemaAsync( await SearchBySchemaAsync(result, parsed.Text, schemaIds, scope, halfTake, 1.1, ct);
query.Text, await SearchByAppAsync(result, parsed.Text, app, scope, halfTake, 1, ct);
schemaIds,
scope,
halfBucket, 1,
ct);
documents.AddRange(await SearchByAppAsync(query.Text, app, scope, halfBucket, 1, ct));
} }
return documents.OrderByDescending(x => x.Item2).Select(x => x.Item1).Distinct().ToList(); return result.OrderByDescending(x => x.Score).Select(x => x.Id).Distinct().ToList();
} }
private Task<List<(DomainId, double)>> SearchBySchemaAsync(string search, IEnumerable<DomainId> schemaIds, SearchScope scope, int limit, double factor, private Task SearchBySchemaAsync(List<(DomainId, double)> result, string text, IEnumerable<DomainId> schemaIds, SearchScope scope, int take, double factor,
CancellationToken ct = default) CancellationToken ct = default)
{ {
var filter = $"{string.Join(" or ", schemaIds.Select(x => $"schemaId eq '{x}'"))} and {GetServeField(scope)} eq true"; var searchField = GetServeField(scope);
return SearchAsync(search, filter, limit, factor, ct); var filter = $"{string.Join(" or ", schemaIds.Select(x => $"schemaId eq '{x}'"))} and {searchField} eq true";
return SearchAsync(result, text, filter, take, factor, ct);
} }
private Task<List<(DomainId, double)>> SearchByAppAsync(string search, IAppEntity app, SearchScope scope, int limit, double factor, private Task SearchByAppAsync(List<(DomainId, double)> result, string text, IAppEntity app, SearchScope scope, int take, double factor,
CancellationToken ct = default) CancellationToken ct = default)
{ {
var filter = $"appId eq '{app.Id}' and {GetServeField(scope)} eq true"; var searchField = GetServeField(scope);
var filter = $"appId eq '{app.Id}' and {searchField} eq true";
return SearchAsync(search, filter, limit, factor, ct); return SearchAsync(result, text, filter, take, factor, ct);
} }
private async Task<List<(DomainId, double)>> SearchAsync(string search, string filter, int size, double factor, private async Task SearchAsync(List<(DomainId, double)> result, string text, string filter, int take, double factor,
CancellationToken ct = default) CancellationToken ct = default)
{ {
var searchOptions = new SearchOptions var searchOptions = new SearchOptions
@ -157,22 +153,30 @@ namespace Squidex.Extensions.Text.Azure
}; };
searchOptions.Select.Add("contentId"); searchOptions.Select.Add("contentId");
searchOptions.Size = size; searchOptions.Size = take;
searchOptions.QueryType = SearchQueryType.Full; searchOptions.QueryType = SearchQueryType.Full;
var results = await searchClient.SearchAsync<SearchDocument>(search, searchOptions, ct); var results = await searchClient.SearchAsync<SearchDocument>(text, searchOptions, ct);
var ids = new List<(DomainId, double)>();
await foreach (var item in results.Value.GetResultsAsync().WithCancellation(ct)) await foreach (var item in results.Value.GetResultsAsync().WithCancellation(ct))
{ {
if (item != null) if (item != null)
{ {
ids.Add((DomainId.Create(item.Document["contentId"].ToString()), factor * item.Score ?? 0)); var id = DomainId.Create(item.Document["contentId"].ToString());
result.Add((id, factor * item.Score ?? 0));
} }
} }
}
private static string BuildGeoQuery(GeoQuery query, SearchScope scope)
{
var (schema, field, lat, lng, radius, _) = query;
var searchField = GetServeField(scope);
var searchDistance = radius / 1000;
return ids; return $"schemaId eq '{schema}' and geoField eq '{field}' and geo.distance(geoObject, geography'POINT({lng} {lat})') lt {searchDistance} and {searchField} eq true";
} }
private static string GetServeField(SearchScope scope) private static string GetServeField(SearchScope scope)

82
backend/extensions/Squidex.Extensions/Text/Azure/CommandFactory.cs

@ -9,6 +9,7 @@ using System;
using System.Collections.Generic; using System.Collections.Generic;
using System.Text; using System.Text;
using Azure.Search.Documents.Models; using Azure.Search.Documents.Models;
using GeoJSON.Net.Geometry;
using Squidex.Domain.Apps.Entities.Contents.Text; using Squidex.Domain.Apps.Entities.Contents.Text;
namespace Squidex.Extensions.Text.Azure namespace Squidex.Extensions.Text.Azure
@ -20,57 +21,92 @@ namespace Squidex.Extensions.Text.Azure
switch (command) switch (command)
{ {
case UpsertIndexEntry upsert: case UpsertIndexEntry upsert:
batch.Add(UpsertEntry(upsert)); UpsertTextEntry(upsert, batch);
break; break;
case UpdateIndexEntry update: case UpdateIndexEntry update:
batch.Add(UpdateEntry(update)); UpdateEntry(update, batch);
break; break;
case DeleteIndexEntry delete: case DeleteIndexEntry delete:
batch.Add(DeleteEntry(delete)); DeleteEntry(delete, batch);
break; break;
} }
} }
private static IndexDocumentsAction<SearchDocument> UpsertEntry(UpsertIndexEntry upsert) private static void UpsertTextEntry(UpsertIndexEntry upsert, IList<IndexDocumentsAction<SearchDocument>> batch)
{ {
var searchDocument = new SearchDocument var geoField = string.Empty;
var geoObject = (object)null;
if (upsert.GeoObjects != null)
{ {
["docId"] = upsert.DocId.ToBase64(), foreach (var (key, value) in upsert.GeoObjects)
["appId"] = upsert.AppId.Id.ToString(), {
["appName"] = upsert.AppId.Name, if (value is Point point)
["contentId"] = upsert.ContentId.ToString(), {
["schemaId"] = upsert.SchemaId.Id.ToString(), geoField = key;
["schemaName"] = upsert.SchemaId.Name, geoObject = new
["serveAll"] = upsert.ServeAll, {
["servePublished"] = upsert.ServePublished type = "Point",
}; coordinates = new[]
{
point.Coordinates.Longitude,
point.Coordinates.Latitude
}
};
break;
}
}
}
if (upsert.Texts != null) if (upsert.Texts != null || geoObject != null)
{ {
var document = new SearchDocument
{
["docId"] = upsert.DocId.ToBase64(),
["appId"] = upsert.AppId.Id.ToString(),
["appName"] = upsert.AppId.Name,
["contentId"] = upsert.ContentId.ToString(),
["schemaId"] = upsert.SchemaId.Id.ToString(),
["schemaName"] = upsert.SchemaId.Name,
["serveAll"] = upsert.ServeAll,
["servePublished"] = upsert.ServePublished,
["geoField"] = geoField,
["geoObject"] = geoObject
};
foreach (var (key, value) in upsert.Texts) foreach (var (key, value) in upsert.Texts)
{ {
searchDocument[AzureIndexDefinition.GetTextField(key)] = value; var text = value;
var languageCode = AzureIndexDefinition.GetFieldName(key);
if (document.TryGetValue(languageCode, out var existing))
{
text = $"{existing} {value}";
}
document[languageCode] = text;
} }
}
return IndexDocumentsAction.MergeOrUpload(searchDocument); batch.Add(IndexDocumentsAction.MergeOrUpload(document));
}
} }
private static IndexDocumentsAction<SearchDocument> UpdateEntry(UpdateIndexEntry update) private static void UpdateEntry(UpdateIndexEntry update, IList<IndexDocumentsAction<SearchDocument>> batch)
{ {
var searchDocument = new SearchDocument var document = new SearchDocument
{ {
["docId"] = update.DocId.ToBase64(), ["docId"] = update.DocId.ToBase64(),
["serveAll"] = update.ServeAll, ["serveAll"] = update.ServeAll,
["servePublished"] = update.ServePublished, ["servePublished"] = update.ServePublished,
}; };
return IndexDocumentsAction.MergeOrUpload(searchDocument); batch.Add(IndexDocumentsAction.MergeOrUpload(document));
} }
private static IndexDocumentsAction<SearchDocument> DeleteEntry(DeleteIndexEntry delete) private static void DeleteEntry(DeleteIndexEntry delete, IList<IndexDocumentsAction<SearchDocument>> batch)
{ {
return IndexDocumentsAction.Delete("docId", delete.DocId.ToBase64()); batch.Add(IndexDocumentsAction.Delete("docId", delete.DocId.ToBase64()));
} }
private static string ToBase64(this string value) private static string ToBase64(this string value)

72
backend/extensions/Squidex.Extensions/Text/ElasticSearch/CommandFactory.cs

@ -6,6 +6,7 @@
// ========================================================================== // ==========================================================================
using System.Collections.Generic; using System.Collections.Generic;
using GeoJSON.Net.Geometry;
using Squidex.Domain.Apps.Entities.Contents.Text; using Squidex.Domain.Apps.Entities.Contents.Text;
namespace Squidex.Extensions.Text.ElasticSearch namespace Squidex.Extensions.Text.ElasticSearch
@ -30,26 +31,67 @@ namespace Squidex.Extensions.Text.ElasticSearch
private static void UpsertEntry(UpsertIndexEntry upsert, List<object> args, string indexName) private static void UpsertEntry(UpsertIndexEntry upsert, List<object> args, string indexName)
{ {
args.Add(new var geoField = string.Empty;
var geoObject = (object)null;
if (upsert.GeoObjects != null)
{ {
index = new foreach (var (key, value) in upsert.GeoObjects)
{ {
_id = upsert.DocId, if (value is Point point)
_index = indexName {
geoField = key;
geoObject = new
{
lat = point.Coordinates.Latitude,
lon = point.Coordinates.Longitude
};
break;
}
} }
}); }
args.Add(new if (upsert.Texts != null || geoObject != null)
{ {
appId = upsert.AppId.Id.ToString(), args.Add(new
appName = upsert.AppId.Name, {
contentId = upsert.ContentId.ToString(), index = new
schemaId = upsert.SchemaId.Id.ToString(), {
schemaName = upsert.SchemaId.Name, _id = upsert.DocId,
serveAll = upsert.ServeAll, _index = indexName
servePublished = upsert.ServePublished, }
texts = upsert.Texts });
});
var texts = new Dictionary<string, string>();
foreach (var (key, value) in upsert.Texts)
{
var text = value;
var languageCode = ElasticSearchIndexDefinition.GetFieldName(key);
if (texts.TryGetValue(languageCode, out var existing))
{
text = $"{existing} {value}";
}
texts[languageCode] = text;
}
args.Add(new
{
appId = upsert.AppId.Id.ToString(),
appName = upsert.AppId.Name,
contentId = upsert.ContentId.ToString(),
schemaId = upsert.SchemaId.Id.ToString(),
schemaName = upsert.SchemaId.Name,
serveAll = upsert.ServeAll,
servePublished = upsert.ServePublished,
texts,
geoField,
geoObject
});
}
} }
private static void UpdateEntry(UpdateIndexEntry update, List<object> args, string indexName) private static void UpdateEntry(UpdateIndexEntry update, List<object> args, string indexName)

140
backend/extensions/Squidex.Extensions/Text/ElasticSearch/ElasticSearchIndexDefinition.cs

@ -0,0 +1,140 @@
// ==========================================================================
// Squidex Headless CMS
// ==========================================================================
// Copyright (c) Squidex UG (haftungsbeschraenkt)
// All rights reserved. Licensed under the MIT license.
// ==========================================================================
using System;
using System.Collections.Generic;
using System.Linq;
using System.Threading;
using System.Threading.Tasks;
using Elasticsearch.Net;
namespace Squidex.Extensions.Text.ElasticSearch
{
public static class ElasticSearchIndexDefinition
{
private static readonly Dictionary<string, string> FieldPaths;
private static readonly Dictionary<string, string> FieldAnalyzers = new Dictionary<string, string>
{
["ar"] = "arabic",
["hy"] = "armenian",
["eu"] = "basque",
["bn"] = "bengali",
["br"] = "brazilian",
["bg"] = "bulgarian",
["ca"] = "catalan",
["zh"] = "cjk",
["ja"] = "cjk",
["ko"] = "cjk",
["cs"] = "czech",
["da"] = "danish",
["nl"] = "dutch",
["en"] = "english",
["fi"] = "finnish",
["fr"] = "french",
["gl"] = "galician",
["de"] = "german",
["el"] = "greek",
["hi"] = "hindi",
["hu"] = "hungarian",
["id"] = "indonesian",
["ga"] = "irish",
["it"] = "italian",
["lv"] = "latvian",
["lt"] = "lithuanian",
["no"] = "norwegian",
["pt"] = "portuguese",
["ro"] = "romanian",
["ru"] = "russian",
["ku"] = "sorani",
["es"] = "spanish",
["sv"] = "swedish",
["tr"] = "turkish",
["th"] = "thai"
};
static ElasticSearchIndexDefinition()
{
FieldPaths = FieldAnalyzers.ToDictionary(x => x.Key, x => $"texts.{x.Key}");
}
public static string GetFieldName(string key)
{
if (FieldAnalyzers.ContainsKey(key))
{
return key;
}
if (key.Length > 0)
{
var language = key[2..];
if (FieldAnalyzers.ContainsKey(language))
{
return language;
}
}
return "iv";
}
public static string GetFieldPath(string key)
{
if (FieldPaths.TryGetValue(key, out var path))
{
return path;
}
if (key.Length > 0)
{
var language = key[2..];
if (FieldPaths.TryGetValue(language, out path))
{
return path;
}
}
return "texts.iv";
}
public static async Task ApplyAsync(IElasticLowLevelClient elastic, string indexName,
CancellationToken ct = default)
{
var query = new
{
properties = new Dictionary<string, object>
{
["geoObject"] = new
{
type ="geo_point"
}
}
};
foreach (var (key, analyzer) in FieldAnalyzers)
{
query.properties[GetFieldPath(key)] = new
{
type = "text",
analyzer
};
}
var result = await elastic.Indices.PutMappingAsync<StringResponse>(indexName, CreatePost(query), ctx: ct);
if (!result.Success)
{
throw new InvalidOperationException($"Failed with ${result.Body}", result.OriginalException);
}
}
private static PostData CreatePost<T>(T data)
{
return new SerializableData<T>(data);
}
}
}

226
backend/extensions/Squidex.Extensions/Text/ElasticSearch/ElasticSearchMapping.cs

@ -1,226 +0,0 @@
// ==========================================================================
// Squidex Headless CMS
// ==========================================================================
// Copyright (c) Squidex UG (haftungsbeschraenkt)
// All rights reserved. Licensed under the MIT license.
// ==========================================================================
using System;
using System.Collections.Generic;
using System.Threading;
using System.Threading.Tasks;
using Elasticsearch.Net;
namespace Squidex.Extensions.Text.ElasticSearch
{
public static class ElasticSearchMapping
{
public static async Task ApplyAsync(IElasticLowLevelClient elastic, string indexName,
CancellationToken ct = default)
{
var query = new
{
properties = new Dictionary<string, object>
{
["texts.ar"] = new
{
type = "text",
analyzer = "arabic"
},
["texts.hy"] = new
{
type = "text",
analyzer = "armenian"
},
["texts.eu"] = new
{
type = "text",
analyzer = "basque"
},
["texts.bn"] = new
{
type = "text",
analyzer = "bengali"
},
["texts.br"] = new
{
type = "text",
analyzer = "brazilian"
},
["texts.bg"] = new
{
type = "text",
analyzer = "bulgarian"
},
["texts.ca"] = new
{
type = "text",
analyzer = "catalan"
},
["texts.zh"] = new
{
type = "text",
analyzer = "cjk"
},
["texts.ja"] = new
{
type = "text",
analyzer = "cjk"
},
["texts.ko"] = new
{
type = "text",
analyzer = "cjk"
},
["texts.cs"] = new
{
type = "text",
analyzer = "czech"
},
["texts.da"] = new
{
type = "text",
analyzer = "danish"
},
["texts.nl"] = new
{
type = "text",
analyzer = "dutch"
},
["texts.en"] = new
{
type = "text",
analyzer = "english"
},
["texts.fi"] = new
{
type = "text",
analyzer = "finnish"
},
["texts.fr"] = new
{
type = "text",
analyzer = "french"
},
["texts.gl"] = new
{
type = "text",
analyzer = "galician"
},
["texts.de"] = new
{
type = "text",
analyzer = "german"
},
["texts.el"] = new
{
type = "text",
analyzer = "greek"
},
["texts.hi"] = new
{
type = "text",
analyzer = "hindi"
},
["texts.hu"] = new
{
type = "text",
analyzer = "hungarian"
},
["texts.id"] = new
{
type = "text",
analyzer = "indonesian"
},
["texts.ga"] = new
{
type = "text",
analyzer = "irish"
},
["texts.it"] = new
{
type = "text",
analyzer = "italian"
},
["texts.lv"] = new
{
type = "text",
analyzer = "latvian"
},
["texts.lt"] = new
{
type = "text",
analyzer = "lithuanian"
},
["texts.nb"] = new
{
type = "text",
analyzer = "norwegian"
},
["texts.nn"] = new
{
type = "text",
analyzer = "norwegian"
},
["texts.no"] = new
{
type = "text",
analyzer = "norwegian"
},
["texts.pt"] = new
{
type = "text",
analyzer = "portuguese"
},
["texts.ro"] = new
{
type = "text",
analyzer = "romanian"
},
["texts.ru"] = new
{
type = "text",
analyzer = "russian"
},
["texts.ku"] = new
{
type = "text",
analyzer = "sorani"
},
["texts.es"] = new
{
type = "text",
analyzer = "spanish"
},
["texts.sv"] = new
{
type = "text",
analyzer = "swedish"
},
["texts.tr"] = new
{
type = "text",
analyzer = "turkish"
},
["texts.th"] = new
{
type = "text",
analyzer = "thai"
}
}
};
var result = await elastic.Indices.PutMappingAsync<StringResponse>(indexName, CreatePost(query), ctx: ct);
if (!result.Success)
{
throw new InvalidOperationException($"Failed with ${result.Body}", result.OriginalException);
}
}
private static PostData CreatePost<T>(T data)
{
return new SerializableData<T>(data);
}
}
}

135
backend/extensions/Squidex.Extensions/Text/ElasticSearch/ElasticSearchTextIndex.cs

@ -8,9 +8,11 @@
using System; using System;
using System.Collections.Generic; using System.Collections.Generic;
using System.Linq; using System.Linq;
using System.Text.RegularExpressions;
using System.Threading; using System.Threading;
using System.Threading.Tasks; using System.Threading.Tasks;
using Elasticsearch.Net; using Elasticsearch.Net;
using Newtonsoft.Json;
using Squidex.Domain.Apps.Entities.Apps; using Squidex.Domain.Apps.Entities.Apps;
using Squidex.Domain.Apps.Entities.Contents; using Squidex.Domain.Apps.Entities.Contents;
using Squidex.Domain.Apps.Entities.Contents.Text; using Squidex.Domain.Apps.Entities.Contents.Text;
@ -21,25 +23,25 @@ namespace Squidex.Extensions.Text.ElasticSearch
{ {
public sealed class ElasticSearchTextIndex : ITextIndex, IInitializable public sealed class ElasticSearchTextIndex : ITextIndex, IInitializable
{ {
private static readonly Regex LanguageRegex = new Regex(@"[^\w]+([a-z\-_]{2,}):", RegexOptions.Compiled | RegexOptions.ExplicitCapture);
private static readonly Regex LanguageRegexStart = new Regex(@"$^([a-z\-_]{2,}):", RegexOptions.Compiled | RegexOptions.ExplicitCapture);
private readonly ElasticLowLevelClient client; private readonly ElasticLowLevelClient client;
private readonly QueryParser queryParser = new QueryParser(ElasticSearchIndexDefinition.GetFieldPath);
private readonly string indexName; private readonly string indexName;
private readonly int waitAfterUpdate;
public ElasticSearchTextIndex(string configurationString, string indexName, int waitAfterUpdate = 0) public ElasticSearchTextIndex(string configurationString, string indexName)
{ {
var config = new ConnectionConfiguration(new Uri(configurationString)); var config = new ConnectionConfiguration(new Uri(configurationString));
client = new ElasticLowLevelClient(config); client = new ElasticLowLevelClient(config);
this.indexName = indexName; this.indexName = indexName;
this.waitAfterUpdate = waitAfterUpdate;
} }
public Task InitializeAsync( public Task InitializeAsync(
CancellationToken ct) CancellationToken ct)
{ {
return ElasticSearchMapping.ApplyAsync(client, indexName, ct); return ElasticSearchIndexDefinition.ApplyAsync(client, indexName, ct);
} }
public Task ClearAsync( public Task ClearAsync(
@ -69,17 +71,68 @@ namespace Squidex.Extensions.Text.ElasticSearch
{ {
throw new InvalidOperationException($"Failed with ${result.Body}", result.OriginalException); throw new InvalidOperationException($"Failed with ${result.Body}", result.OriginalException);
} }
if (waitAfterUpdate > 0)
{
await Task.Delay(waitAfterUpdate, ct);
}
} }
public Task<List<DomainId>> SearchAsync(IAppEntity app, GeoQuery query, SearchScope scope, public async Task<List<DomainId>> SearchAsync(IAppEntity app, GeoQuery query, SearchScope scope,
CancellationToken ct = default) CancellationToken ct = default)
{ {
return Task.FromResult<List<DomainId>>(null); Guard.NotNull(app, nameof(app));
Guard.NotNull(query, nameof(query));
var serveField = GetServeField(scope);
var elasticQuery = new
{
query = new
{
@bool = new
{
filter = new object[]
{
new
{
term = new Dictionary<string, object>
{
["schemaId.keyword"] = query.SchemaId.ToString()
}
},
new
{
term = new Dictionary<string, string>
{
["geoField.keyword"] = query.Field
}
},
new
{
term = new Dictionary<string, string>
{
[serveField] = "true"
}
},
new
{
geo_distance = new
{
geoObject = new
{
lat = query.Latitude,
lon = query.Longitude
},
distance = $"{query.Radius}m"
}
}
},
}
},
_source = new[]
{
"contentId"
},
size = query.Take
};
return await SearchAsync(elasticQuery, ct);
} }
public async Task<List<DomainId>> SearchAsync(IAppEntity app, TextQuery query, SearchScope scope, public async Task<List<DomainId>> SearchAsync(IAppEntity app, TextQuery query, SearchScope scope,
@ -88,34 +141,13 @@ namespace Squidex.Extensions.Text.ElasticSearch
Guard.NotNull(app, nameof(app)); Guard.NotNull(app, nameof(app));
Guard.NotNull(query, nameof(query)); Guard.NotNull(query, nameof(query));
var queryText = query.Text; var parsed = queryParser.Parse(query.Text);
if (string.IsNullOrWhiteSpace(queryText)) if (parsed == null)
{ {
return null; return null;
} }
var isFuzzy = queryText.EndsWith("~", StringComparison.OrdinalIgnoreCase);
if (isFuzzy)
{
queryText = queryText[..^1];
}
var field = "texts.*";
if (queryText.Length >= 4 && queryText.IndexOf(":", StringComparison.OrdinalIgnoreCase) == 2)
{
var candidateLanguage = queryText.Substring(0, 2);
if (Language.IsValidLanguage(candidateLanguage))
{
field = $"texts.{candidateLanguage}";
queryText = queryText[3..];
}
}
var serveField = GetServeField(scope); var serveField = GetServeField(scope);
var elasticQuery = new var elasticQuery = new
@ -124,7 +156,7 @@ namespace Squidex.Extensions.Text.ElasticSearch
{ {
@bool = new @bool = new
{ {
must = new List<object> filter = new List<object>
{ {
new new
{ {
@ -139,18 +171,13 @@ namespace Squidex.Extensions.Text.ElasticSearch
{ {
[serveField] = "true" [serveField] = "true"
} }
}, }
new },
must = new
{
query_string = new
{ {
multi_match = new query = parsed.Text
{
fuzziness = isFuzzy ? (object)"AUTO" : 0,
fields = new[]
{
field
},
query = query.Text
}
} }
}, },
should = new List<object>() should = new List<object>()
@ -160,7 +187,7 @@ namespace Squidex.Extensions.Text.ElasticSearch
{ {
"contentId" "contentId"
}, },
size = 2000 size = query.Take
}; };
if (query.RequiredSchemaIds?.Count > 0) if (query.RequiredSchemaIds?.Count > 0)
@ -173,7 +200,7 @@ namespace Squidex.Extensions.Text.ElasticSearch
} }
}; };
elasticQuery.query.@bool.must.Add(bySchema); elasticQuery.query.@bool.filter.Add(bySchema);
} }
else if (query.PreferredSchemaId.HasValue) else if (query.PreferredSchemaId.HasValue)
{ {
@ -188,7 +215,15 @@ namespace Squidex.Extensions.Text.ElasticSearch
elasticQuery.query.@bool.should.Add(bySchema); elasticQuery.query.@bool.should.Add(bySchema);
} }
var result = await client.SearchAsync<DynamicResponse>(indexName, CreatePost(elasticQuery), ctx: ct); var json = JsonConvert.SerializeObject(elasticQuery, Formatting.Indented);
return await SearchAsync(elasticQuery, ct);
}
private async Task<List<DomainId>> SearchAsync(object query,
CancellationToken ct)
{
var result = await client.SearchAsync<DynamicResponse>(indexName, CreatePost(query), ctx: ct);
if (!result.Success) if (!result.Success)
{ {

3
backend/src/Squidex.Domain.Apps.Entities.MongoDb/Squidex.Domain.Apps.Entities.MongoDb.csproj

@ -17,12 +17,13 @@
<ProjectReference Include="..\Squidex.Domain.Apps.Entities\Squidex.Domain.Apps.Entities.csproj" /> <ProjectReference Include="..\Squidex.Domain.Apps.Entities\Squidex.Domain.Apps.Entities.csproj" />
</ItemGroup> </ItemGroup>
<ItemGroup> <ItemGroup>
<PackageReference Include="Lucene.Net.QueryParser" Version="4.8.0-beta00015" />
<PackageReference Include="Meziantou.Analyzer" Version="1.0.670"> <PackageReference Include="Meziantou.Analyzer" Version="1.0.670">
<PrivateAssets>all</PrivateAssets> <PrivateAssets>all</PrivateAssets>
<IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets> <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
</PackageReference> </PackageReference>
<PackageReference Include="MongoDB.Driver" Version="2.13.1" /> <PackageReference Include="MongoDB.Driver" Version="2.13.1" />
<PackageReference Include="RefactoringEssentials" Version="5.6.0" PrivateAssets="all" /> <PackageReference Include="RefactoringEssentials" Version="5.6.0" PrivateAssets="all" />
<PackageReference Include="StyleCop.Analyzers" Version="1.1.118" PrivateAssets="all" /> <PackageReference Include="StyleCop.Analyzers" Version="1.1.118" PrivateAssets="all" />
<PackageReference Include="System.ValueTuple" Version="4.5.0" /> <PackageReference Include="System.ValueTuple" Version="4.5.0" />
</ItemGroup> </ItemGroup>

210
backend/src/Squidex.Domain.Apps.Entities.MongoDb/Text/AtlasIndexDefinition.cs

@ -0,0 +1,210 @@
// ==========================================================================
// Squidex Headless CMS
// ==========================================================================
// Copyright (c) Squidex UG (haftungsbeschraenkt)
// All rights reserved. Licensed under the MIT license.
// ==========================================================================
using System.Collections.Generic;
using System.Linq;
using System.Net;
using System.Net.Http;
using System.Net.Http.Json;
using System.Threading;
using System.Threading.Tasks;
using Squidex.Hosting.Configuration;
namespace Squidex.Domain.Apps.Entities.MongoDb.Text
{
public static class AtlasIndexDefinition
{
private static readonly Dictionary<string, string> FieldPaths = new Dictionary<string, string>();
private static readonly Dictionary<string, string> FieldAnalyzers = new Dictionary<string, string>
{
["iv"] = "lucene.standard",
["ar"] = "lucene.arabic",
["hy"] = "lucene.armenian",
["eu"] = "lucene.basque",
["bn"] = "lucene.bengali",
["br"] = "lucene.brazilian",
["bg"] = "lucene.bulgarian",
["ca"] = "lucene.catalan",
["ko"] = "lucene.cjk",
["da"] = "lucene.danish",
["nl"] = "lucene.dutch",
["en"] = "lucene.english",
["fi"] = "lucene.finnish",
["fr"] = "lucene.french",
["gl"] = "lucene.galician",
["de"] = "lucene.german",
["el"] = "lucene.greek",
["hi"] = "lucene.hindi",
["hu"] = "lucene.hungarian",
["id"] = "lucene.indonesian",
["ga"] = "lucene.irish",
["it"] = "lucene.italian",
["jp"] = "lucene.japanese",
["lv"] = "lucene.latvian",
["no"] = "lucene.norwegian",
["fa"] = "lucene.persian",
["pt"] = "lucene.portuguese",
["ro"] = "lucene.romanian",
["ru"] = "lucene.russian",
["zh"] = "lucene.smartcn",
["es"] = "lucene.spanish",
["sv"] = "lucene.swedish",
["th"] = "lucene.thai",
["tr"] = "lucene.turkish",
["uk"] = "lucene.ukrainian"
};
public sealed class ErrorResponse
{
public string Detail { get; set; }
public string ErrorCode { get; set; }
}
static AtlasIndexDefinition()
{
FieldPaths = FieldAnalyzers.ToDictionary(x => x.Key, x => $"t.{x.Key}");
}
public static string GetFieldName(string key)
{
if (FieldAnalyzers.ContainsKey(key))
{
return key;
}
if (key.Length > 0)
{
var language = key[2..];
if (FieldAnalyzers.ContainsKey(language))
{
return language;
}
}
return "iv";
}
public static string GetFieldPath(string key)
{
if (FieldPaths.TryGetValue(key, out var path))
{
return path;
}
if (key.Length > 0)
{
var language = key[2..];
if (FieldPaths.TryGetValue(language, out path))
{
return path;
}
}
return "t.iv";
}
public static async Task<string> CreateIndexAsync(AtlasOptions options,
string database,
string collectionName,
CancellationToken ct)
{
var (index, name) = Create(database, collectionName);
using (var httpClient = new HttpClient(new HttpClientHandler
{
Credentials = new NetworkCredential(options.PublicKey, options.PrivateKey, "cloud.mongodb.com")
}))
{
var url = $"https://cloud.mongodb.com/api/atlas/v1.0/groups/{options.GroupId}/clusters/{options.ClusterName}/fts/indexes";
var result = await httpClient.PostAsJsonAsync(url, index, ct);
if (result.IsSuccessStatusCode)
{
return name;
}
var error = await result.Content.ReadFromJsonAsync<ErrorResponse>(cancellationToken: ct);
if (error?.ErrorCode != "ATLAS_FTS_DUPLICATE_INDEX")
{
var message = new ConfigurationError($"Creating index failed with {result.StatusCode}: {error?.Detail}");
throw new ConfigurationException(message);
}
}
return name;
}
public static (object, string) Create(string database, string collectionName)
{
var name = $"{database}_{collectionName}_text".ToLowerInvariant();
var texts = new
{
type = "document",
fields = new Dictionary<string, object>(),
dynamic = false
};
var index = new
{
collectionName,
database,
name,
mappings = new
{
dynamic = false,
fields = new Dictionary<string, object>
{
["_ai"] = new
{
type = "string",
analyzer = "lucene.keyword"
},
["_si"] = new
{
type = "string",
analyzer = "lucene.keyword"
},
["_ci"] = new
{
type = "string",
analyzer = "lucene.keyword"
},
["fa"] = new
{
type = "boolean"
},
["fp"] = new
{
type = "boolean"
},
["t"] = texts
}
}
};
foreach (var (field, analyzer) in FieldAnalyzers)
{
texts.fields[field] = new
{
type = "string",
analyzer,
searchAnalyzer = analyzer,
store = false
};
}
return (index, name);
}
}
}

31
backend/src/Squidex.Domain.Apps.Entities.MongoDb/Text/AtlasOptions.cs

@ -0,0 +1,31 @@
// ==========================================================================
// Squidex Headless CMS
// ==========================================================================
// Copyright (c) Squidex UG (haftungsbeschraenkt)
// All rights reserved. Licensed under the MIT license.
// ==========================================================================
namespace Squidex.Domain.Apps.Entities.MongoDb.Text
{
public sealed class AtlasOptions
{
public string GroupId { get; set; }
public string ClusterName { get; set; }
public string PublicKey { get; set; }
public string PrivateKey { get; set; }
public bool FullTextEnabled { get; set; }
public bool IsConfigured()
{
return
!string.IsNullOrWhiteSpace(GroupId) &&
!string.IsNullOrWhiteSpace(ClusterName) &&
!string.IsNullOrWhiteSpace(PublicKey) &&
!string.IsNullOrWhiteSpace(PrivateKey);
}
}
}

164
backend/src/Squidex.Domain.Apps.Entities.MongoDb/Text/AtlasTextIndex.cs

@ -0,0 +1,164 @@
// ==========================================================================
// Squidex Headless CMS
// ==========================================================================
// Copyright (c) Squidex UG (haftungsbeschraenkt)
// All rights reserved. Licensed under the MIT license.
// ==========================================================================
using System.Collections.Generic;
using System.Linq;
using System.Threading;
using System.Threading.Tasks;
using Lucene.Net.Analysis.Standard;
using Lucene.Net.Analysis.Util;
using Lucene.Net.Util;
using Microsoft.Extensions.Options;
using MongoDB.Bson;
using MongoDB.Driver;
using Squidex.Domain.Apps.Entities.Apps;
using Squidex.Domain.Apps.Entities.Contents;
using Squidex.Domain.Apps.Entities.Contents.Text;
using Squidex.Infrastructure;
using LuceneQueryAnalyzer = Lucene.Net.QueryParsers.Classic.QueryParser;
namespace Squidex.Domain.Apps.Entities.MongoDb.Text
{
public sealed class AtlasTextIndex : MongoTextIndexBase<Dictionary<string, string>>
{
private static readonly LuceneQueryVisitor QueryVisitor = new LuceneQueryVisitor(AtlasIndexDefinition.GetFieldPath);
private static readonly LuceneQueryAnalyzer QueryParser =
new LuceneQueryAnalyzer(LuceneVersion.LUCENE_48, "*",
new StandardAnalyzer(LuceneVersion.LUCENE_48, CharArraySet.EMPTY_SET));
private readonly AtlasOptions options;
private string index;
public AtlasTextIndex(IMongoDatabase database, IOptions<AtlasOptions> options, bool setup = false)
: base(database, setup)
{
this.options = options.Value;
}
protected override async Task SetupCollectionAsync(IMongoCollection<MongoTextIndexEntity<Dictionary<string, string>>> collection,
CancellationToken ct)
{
await base.SetupCollectionAsync(collection, ct);
index = await AtlasIndexDefinition.CreateIndexAsync(options,
Database.DatabaseNamespace.DatabaseName, CollectionName(), ct);
}
protected override Dictionary<string, string> BuildTexts(Dictionary<string, string> source)
{
var texts = new Dictionary<string, string>();
foreach (var (key, value) in source)
{
var text = value;
var languageCode = AtlasIndexDefinition.GetFieldName(key);
if (texts.TryGetValue(languageCode, out var existing))
{
text = $"{existing} {value}";
}
texts[languageCode] = text;
}
return texts;
}
public override async Task<List<DomainId>?> SearchAsync(IAppEntity app, TextQuery query, SearchScope scope,
CancellationToken ct = default)
{
Guard.NotNull(app, nameof(app));
Guard.NotNull(query, nameof(query));
var (search, take) = query;
if (string.IsNullOrWhiteSpace(search))
{
return null;
}
var luceneQuery = QueryParser.Parse(search);
var serveField = scope == SearchScope.All ? "fa" : "fp";
var compound = new BsonDocument
{
["must"] = new BsonArray
{
QueryVisitor.Visit(luceneQuery)
},
["filter"] = new BsonArray
{
new BsonDocument
{
["text"] = new BsonDocument
{
["path"] = "_ai",
["query"] = app.Id.ToString()
}
},
new BsonDocument
{
["equals"] = new BsonDocument
{
["path"] = serveField,
["value"] = true
}
}
}
};
if (query.PreferredSchemaId != null)
{
compound["should"] = new BsonArray
{
new BsonDocument
{
["text"] = new BsonDocument
{
["path"] = "_si",
["query"] = query.PreferredSchemaId.Value.ToString()
}
}
};
}
else if (query.RequiredSchemaIds?.Count > 0)
{
compound["should"] = new BsonArray(query.RequiredSchemaIds.Select(x =>
new BsonDocument
{
["text"] = new BsonDocument
{
["path"] = "_si",
["query"] = x.ToString()
}
}));
compound["minimumShouldMatch"] = 1;
}
var searchQuery = new BsonDocument
{
["compound"] = compound,
};
if (index != null)
{
searchQuery["index"] = index;
}
var results =
await Collection.Aggregate().Search(searchQuery).Limit(take)
.Project<MongoTextResult>(
Projection.Include(x => x.ContentId)
)
.ToListAsync(ct);
return results.Select(x => x.ContentId).ToList();
}
}
}

44
backend/src/Squidex.Domain.Apps.Entities.MongoDb/FullText/CommandFactory.cs → backend/src/Squidex.Domain.Apps.Entities.MongoDb/Text/CommandFactory.cs

@ -11,14 +11,23 @@ using System.Linq;
using MongoDB.Driver; using MongoDB.Driver;
using Squidex.Domain.Apps.Entities.Contents.Text; using Squidex.Domain.Apps.Entities.Contents.Text;
namespace Squidex.Domain.Apps.Entities.MongoDb.FullText namespace Squidex.Domain.Apps.Entities.MongoDb.Text
{ {
public static class CommandFactory public sealed class CommandFactory<T> where T : class
{ {
private static readonly FilterDefinitionBuilder<MongoTextIndexEntity> Filter = Builders<MongoTextIndexEntity>.Filter; #pragma warning disable RECS0108 // Warns about static fields in generic types
private static readonly UpdateDefinitionBuilder<MongoTextIndexEntity> Update = Builders<MongoTextIndexEntity>.Update; private static readonly FilterDefinitionBuilder<MongoTextIndexEntity<T>> Filter = Builders<MongoTextIndexEntity<T>>.Filter;
private static readonly UpdateDefinitionBuilder<MongoTextIndexEntity<T>> Update = Builders<MongoTextIndexEntity<T>>.Update;
#pragma warning restore RECS0108 // Warns about static fields in generic types
public static void CreateCommands(IndexCommand command, List<WriteModel<MongoTextIndexEntity>> writes) private readonly Func<Dictionary<string, string>, T> textBuilder;
public CommandFactory(Func<Dictionary<string, string>, T> textBuilder)
{
this.textBuilder = textBuilder;
}
public void CreateCommands(IndexCommand command, List<WriteModel<MongoTextIndexEntity<T>>> writes)
{ {
switch (command) switch (command)
{ {
@ -34,10 +43,10 @@ namespace Squidex.Domain.Apps.Entities.MongoDb.FullText
} }
} }
private static void UpsertEntry(UpsertIndexEntry upsert, List<WriteModel<MongoTextIndexEntity>> writes) private void UpsertEntry(UpsertIndexEntry upsert, List<WriteModel<MongoTextIndexEntity<T>>> writes)
{ {
writes.Add( writes.Add(
new UpdateOneModel<MongoTextIndexEntity>( new UpdateOneModel<MongoTextIndexEntity<T>>(
Filter.And( Filter.And(
Filter.Eq(x => x.DocId, upsert.DocId), Filter.Eq(x => x.DocId, upsert.DocId),
Filter.Exists(x => x.GeoField, false), Filter.Exists(x => x.GeoField, false),
@ -45,7 +54,7 @@ namespace Squidex.Domain.Apps.Entities.MongoDb.FullText
Update Update
.Set(x => x.ServeAll, upsert.ServeAll) .Set(x => x.ServeAll, upsert.ServeAll)
.Set(x => x.ServePublished, upsert.ServePublished) .Set(x => x.ServePublished, upsert.ServePublished)
.Set(x => x.Texts, upsert.Texts?.Values.Select(MongoTextIndexEntityText.FromText).ToList()) .Set(x => x.Texts, BuildTexts(upsert))
.SetOnInsert(x => x.Id, Guid.NewGuid().ToString()) .SetOnInsert(x => x.Id, Guid.NewGuid().ToString())
.SetOnInsert(x => x.DocId, upsert.DocId) .SetOnInsert(x => x.DocId, upsert.DocId)
.SetOnInsert(x => x.AppId, upsert.AppId.Id) .SetOnInsert(x => x.AppId, upsert.AppId.Id)
@ -60,7 +69,7 @@ namespace Squidex.Domain.Apps.Entities.MongoDb.FullText
if (!upsert.IsNew) if (!upsert.IsNew)
{ {
writes.Add( writes.Add(
new DeleteOneModel<MongoTextIndexEntity>( new DeleteOneModel<MongoTextIndexEntity<T>>(
Filter.And( Filter.And(
Filter.Eq(x => x.DocId, upsert.DocId), Filter.Eq(x => x.DocId, upsert.DocId),
Filter.Exists(x => x.GeoField), Filter.Exists(x => x.GeoField),
@ -70,8 +79,8 @@ namespace Squidex.Domain.Apps.Entities.MongoDb.FullText
foreach (var (field, geoObject) in upsert.GeoObjects) foreach (var (field, geoObject) in upsert.GeoObjects)
{ {
writes.Add( writes.Add(
new InsertOneModel<MongoTextIndexEntity>( new InsertOneModel<MongoTextIndexEntity<T>>(
new MongoTextIndexEntity new MongoTextIndexEntity<T>
{ {
Id = Guid.NewGuid().ToString(), Id = Guid.NewGuid().ToString(),
AppId = upsert.AppId.Id, AppId = upsert.AppId.Id,
@ -87,20 +96,25 @@ namespace Squidex.Domain.Apps.Entities.MongoDb.FullText
} }
} }
private static void UpdateEntry(UpdateIndexEntry update, List<WriteModel<MongoTextIndexEntity>> writes) private T? BuildTexts(UpsertIndexEntry upsert)
{
return upsert.Texts == null ? null : textBuilder(upsert.Texts);
}
private static void UpdateEntry(UpdateIndexEntry update, List<WriteModel<MongoTextIndexEntity<T>>> writes)
{ {
writes.Add( writes.Add(
new UpdateOneModel<MongoTextIndexEntity>( new UpdateOneModel<MongoTextIndexEntity<T>>(
Filter.Eq(x => x.DocId, update.DocId), Filter.Eq(x => x.DocId, update.DocId),
Update Update
.Set(x => x.ServeAll, update.ServeAll) .Set(x => x.ServeAll, update.ServeAll)
.Set(x => x.ServePublished, update.ServePublished))); .Set(x => x.ServePublished, update.ServePublished)));
} }
private static void DeleteEntry(DeleteIndexEntry delete, List<WriteModel<MongoTextIndexEntity>> writes) private static void DeleteEntry(DeleteIndexEntry delete, List<WriteModel<MongoTextIndexEntity<T>>> writes)
{ {
writes.Add( writes.Add(
new DeleteOneModel<MongoTextIndexEntity>( new DeleteOneModel<MongoTextIndexEntity<T>>(
Filter.Eq(x => x.DocId, delete.DocId))); Filter.Eq(x => x.DocId, delete.DocId)));
} }
} }

329
backend/src/Squidex.Domain.Apps.Entities.MongoDb/Text/LuceneQueryVisitor.cs

@ -0,0 +1,329 @@
// ==========================================================================
// Squidex Headless CMS
// ==========================================================================
// Copyright (c) Squidex UG (haftungsbeschraenkt)
// All rights reserved. Licensed under the MIT license.
// ==========================================================================
using System;
using System.Globalization;
using System.Linq;
using System.Text;
using Lucene.Net.Index;
using Lucene.Net.Search;
using Lucene.Net.Util;
using MongoDB.Bson;
namespace Squidex.Domain.Apps.Entities.MongoDb.Text
{
public sealed class LuceneQueryVisitor
{
private readonly Func<string, string>? fieldConverter;
public LuceneQueryVisitor(Func<string, string>? fieldConverter = null)
{
this.fieldConverter = fieldConverter;
}
public BsonDocument Visit(Query query)
{
switch (query)
{
case BooleanQuery booleanQuery:
return VisitBoolean(booleanQuery);
case TermQuery termQuery:
return VisitTerm(termQuery);
case PhraseQuery phraseQuery:
return VisitPhrase(phraseQuery);
case WildcardQuery wildcardQuery:
return VisitWilcard(wildcardQuery);
case PrefixQuery prefixQuery:
return VisitPrefix(prefixQuery);
case FuzzyQuery fuzzyQuery:
return VisitFuzzy(fuzzyQuery);
case NumericRangeQuery<float> rangeQuery:
return VisitNumericRange(rangeQuery);
case NumericRangeQuery<double> rangeQuery:
return VisitNumericRange(rangeQuery);
case NumericRangeQuery<int> rangeQuery:
return VisitNumericRange(rangeQuery);
case NumericRangeQuery<long> rangeQuery:
return VisitNumericRange(rangeQuery);
case TermRangeQuery termRangeQuery:
return VisitTermRange(termRangeQuery);
default:
throw new NotSupportedException();
}
}
private BsonDocument VisitTermRange(TermRangeQuery termRangeQuery)
{
if (!TryParseValue(termRangeQuery.LowerTerm, out var min) ||
!TryParseValue(termRangeQuery.UpperTerm, out var max))
{
throw new NotSupportedException();
}
var minField = termRangeQuery.IncludesLower ? "gte" : "gt";
var maxField = termRangeQuery.IncludesUpper ? "lte" : "lt";
var doc = new BsonDocument
{
["path"] = GetPath(termRangeQuery.Field),
[minField] = BsonValue.Create(min),
[maxField] = BsonValue.Create(max)
};
ApplyBoost(termRangeQuery, doc);
return new BsonDocument
{
["range"] = doc
};
}
private BsonDocument VisitNumericRange<T>(NumericRangeQuery<T> numericRangeQuery) where T : struct, IComparable<T>
{
var minField = numericRangeQuery.IncludesMin ? "gte" : "gt";
var maxField = numericRangeQuery.IncludesMin ? "lte" : "lt";
var doc = new BsonDocument
{
["path"] = GetPath(numericRangeQuery.Field),
[minField] = BsonValue.Create(numericRangeQuery.Min),
[maxField] = BsonValue.Create(numericRangeQuery.Max)
};
ApplyBoost(numericRangeQuery, doc);
return new BsonDocument
{
["range"] = doc
};
}
private BsonDocument VisitFuzzy(FuzzyQuery fuzzyQuery)
{
var doc = CreateDefaultDoc(fuzzyQuery, fuzzyQuery.Term);
if (fuzzyQuery.MaxEdits > 0)
{
var fuzzy = new BsonDocument
{
["maxEdits"] = fuzzyQuery.MaxEdits,
};
if (fuzzyQuery.PrefixLength > 0)
{
fuzzy["prefixLength"] = fuzzyQuery.PrefixLength;
}
doc["fuzzy"] = fuzzy;
}
return new BsonDocument
{
["text"] = doc
};
}
private BsonDocument VisitPrefix(PrefixQuery prefixQuery)
{
var doc = CreateDefaultDoc(prefixQuery, new Term(prefixQuery.Prefix.Field, prefixQuery.Prefix.Text + "*"));
return new BsonDocument
{
["wildcard"] = doc
};
}
private BsonDocument VisitWilcard(WildcardQuery wildcardQuery)
{
var doc = CreateDefaultDoc(wildcardQuery, wildcardQuery.Term);
return new BsonDocument
{
["wildcard"] = doc
};
}
private BsonDocument VisitPhrase(PhraseQuery phraseQuery)
{
var terms = phraseQuery.GetTerms();
var doc = new BsonDocument
{
["path"] = GetPath(terms[0].Field),
};
if (terms.Length == 1)
{
doc["query"] = terms[0].Text;
}
else
{
doc["query"] = new BsonArray(terms.Select(x => x.Text));
}
if (phraseQuery.Slop != 0)
{
doc["slop"] = phraseQuery.Slop;
}
ApplyBoost(phraseQuery, doc);
return new BsonDocument
{
["phrase"] = doc
};
}
private BsonDocument VisitTerm(TermQuery termQuery)
{
var doc = CreateDefaultDoc(termQuery, termQuery.Term);
return new BsonDocument
{
["text"] = doc
};
}
private BsonDocument VisitBoolean(BooleanQuery booleanQuery)
{
var doc = new BsonDocument();
BsonArray? musts = null;
BsonArray? mustNots = null;
BsonArray? shoulds = null;
foreach (var clause in booleanQuery.Clauses)
{
var converted = Visit(clause.Query);
switch (clause.Occur)
{
case Occur.MUST:
musts ??= new BsonArray();
musts.Add(converted);
break;
case Occur.SHOULD:
shoulds ??= new BsonArray();
shoulds.Add(converted);
break;
case Occur.MUST_NOT:
mustNots ??= new BsonArray();
mustNots.Add(converted);
break;
}
}
if (musts != null)
{
doc.Add("must", musts);
}
if (mustNots != null)
{
doc.Add("mustNot", mustNots);
}
if (shoulds != null)
{
doc.Add("should", shoulds);
}
if (booleanQuery.MinimumNumberShouldMatch > 0)
{
doc["minimumShouldMatch"] = booleanQuery.MinimumNumberShouldMatch;
}
return new BsonDocument
{
["compound"] = doc
};
}
private BsonDocument CreateDefaultDoc(Query query, Term term)
{
var doc = new BsonDocument
{
["path"] = GetPath(term.Field),
["query"] = term.Text
};
ApplyBoost(query, doc);
return doc;
}
private BsonValue GetPath(string field)
{
if (field != "*" && fieldConverter != null)
{
field = fieldConverter(field);
}
if (field.Contains('*', StringComparison.Ordinal))
{
return new BsonDocument
{
["wildcard"] = field
};
}
return field;
}
#pragma warning disable RECS0018 // Comparison of floating point numbers with equality operator
private static void ApplyBoost(Query query, BsonDocument doc)
{
if (query.Boost != 1)
{
doc["score"] = new BsonDocument
{
["boost"] = query.Boost
};
}
}
private static bool TryParseValue(BytesRef bytes, out object result)
{
result = null!;
try
{
var text = Encoding.ASCII.GetString(bytes.Bytes, bytes.Offset, bytes.Length);
if (!double.TryParse(text, NumberStyles.Any, CultureInfo.InvariantCulture, out var number))
{
return false;
}
var integer = (long)number;
if (number == integer)
{
if (integer <= int.MaxValue && integer >= int.MinValue)
{
result = (int)integer;
}
else
{
result = integer;
}
}
else
{
result = number;
}
return true;
}
catch (Exception)
{
return false;
}
}
#pragma warning restore RECS0018 // Comparison of floating point numbers with equality operator
}
}

57
backend/src/Squidex.Domain.Apps.Entities.MongoDb/Text/LuceneSearchDefinitionExtensions.cs

@ -0,0 +1,57 @@
// ==========================================================================
// Squidex Headless CMS
// ==========================================================================
// Copyright (c) Squidex UG (haftungsbeschraenkt)
// All rights reserved. Licensed under the MIT license.
// ==========================================================================
using System;
using MongoDB.Bson;
using MongoDB.Bson.Serialization;
using MongoDB.Driver;
namespace Squidex.Domain.Apps.Entities.MongoDb.Text
{
public static class LuceneSearchDefinitionExtensions
{
public static IAggregateFluent<TResult> Search<TResult>(
this IAggregateFluent<TResult> aggregate,
BsonDocument search)
{
const string OperatorName = "$search";
var stage = new DelegatedPipelineStageDefinition<TResult, TResult>(
OperatorName,
serializer =>
{
var document = new BsonDocument(OperatorName, search);
return new RenderedPipelineStageDefinition<TResult>(OperatorName, document, serializer);
});
return aggregate.AppendStage(stage);
}
private sealed class DelegatedPipelineStageDefinition<TInput, TOutput> : PipelineStageDefinition<TInput, TOutput>
{
private readonly Func<IBsonSerializer<TInput>, RenderedPipelineStageDefinition<TOutput>> renderer;
public override string OperatorName { get; }
public DelegatedPipelineStageDefinition(string operatorName,
Func<IBsonSerializer<TInput>, RenderedPipelineStageDefinition<TOutput>> renderer)
{
this.renderer = renderer;
OperatorName = operatorName;
}
public override RenderedPipelineStageDefinition<TOutput> Render(
IBsonSerializer<TInput> inputSerializer,
IBsonSerializerRegistry serializerRegistry)
{
return renderer(inputSerializer);
}
}
}
}

44
backend/src/Squidex.Domain.Apps.Entities.MongoDb/Text/MongoTextIndex.cs

@ -0,0 +1,44 @@
// ==========================================================================
// Squidex Headless CMS
// ==========================================================================
// Copyright (c) Squidex UG (haftungsbeschraenkt)
// All rights reserved. Licensed under the MIT license.
// ==========================================================================
using System.Collections.Generic;
using System.Linq;
using System.Threading;
using System.Threading.Tasks;
using MongoDB.Driver;
namespace Squidex.Domain.Apps.Entities.MongoDb.Text
{
public sealed class MongoTextIndex : MongoTextIndexBase<List<MongoTextIndexEntityText>>
{
public MongoTextIndex(IMongoDatabase database, bool setup = false)
: base(database, setup)
{
}
protected override async Task SetupCollectionAsync(IMongoCollection<MongoTextIndexEntity<List<MongoTextIndexEntityText>>> collection,
CancellationToken ct)
{
await base.SetupCollectionAsync(collection, ct);
await collection.Indexes.CreateOneAsync(
new CreateIndexModel<MongoTextIndexEntity<List<MongoTextIndexEntityText>>>(
Index
.Text("t.t")
.Ascending(x => x.AppId)
.Ascending(x => x.ServeAll)
.Ascending(x => x.ServePublished)
.Ascending(x => x.SchemaId)),
cancellationToken: ct);
}
protected override List<MongoTextIndexEntityText> BuildTexts(Dictionary<string, string> source)
{
return source.Select(x => new MongoTextIndexEntityText { Text = x.Value }).ToList();
}
}
}

92
backend/src/Squidex.Domain.Apps.Entities.MongoDb/FullText/MongoTextIndex.cs → backend/src/Squidex.Domain.Apps.Entities.MongoDb/Text/MongoTextIndexBase.cs

@ -18,14 +18,15 @@ using Squidex.Domain.Apps.Entities.Contents.Text;
using Squidex.Infrastructure; using Squidex.Infrastructure;
using Squidex.Infrastructure.MongoDb; using Squidex.Infrastructure.MongoDb;
namespace Squidex.Domain.Apps.Entities.MongoDb.FullText namespace Squidex.Domain.Apps.Entities.MongoDb.Text
{ {
public sealed class MongoTextIndex : MongoRepositoryBase<MongoTextIndexEntity>, ITextIndex, IDeleter public abstract class MongoTextIndexBase<T> : MongoRepositoryBase<MongoTextIndexEntity<T>>, ITextIndex, IDeleter where T : class
{ {
private readonly ProjectionDefinition<MongoTextIndexEntity> searchTextProjection; private readonly ProjectionDefinition<MongoTextIndexEntity<T>> searchTextProjection;
private readonly ProjectionDefinition<MongoTextIndexEntity> searchGeoProjection; private readonly ProjectionDefinition<MongoTextIndexEntity<T>> searchGeoProjection;
private readonly CommandFactory<T> commandFactory;
private sealed class MongoTextResult protected sealed class MongoTextResult
{ {
[BsonId] [BsonId]
[BsonElement] [BsonElement]
@ -42,30 +43,26 @@ namespace Squidex.Domain.Apps.Entities.MongoDb.FullText
public double Score { get; set; } public double Score { get; set; }
} }
public MongoTextIndex(IMongoDatabase database, bool setup = false) protected MongoTextIndexBase(IMongoDatabase database, bool setup = false)
: base(database, setup) : base(database, setup)
{ {
searchGeoProjection = Projection.Include(x => x.ContentId); searchGeoProjection = Projection.Include(x => x.ContentId);
searchTextProjection = Projection.Include(x => x.ContentId).MetaTextScore("score"); searchTextProjection = Projection.Include(x => x.ContentId).MetaTextScore("score");
#pragma warning disable MA0056 // Do not call overridable members in constructor
commandFactory = new CommandFactory<T>(BuildTexts);
#pragma warning restore MA0056 // Do not call overridable members in constructor
} }
protected override Task SetupCollectionAsync(IMongoCollection<MongoTextIndexEntity> collection, protected override Task SetupCollectionAsync(IMongoCollection<MongoTextIndexEntity<T>> collection,
CancellationToken ct) CancellationToken ct)
{ {
return collection.Indexes.CreateManyAsync(new[] return collection.Indexes.CreateManyAsync(new[]
{ {
new CreateIndexModel<MongoTextIndexEntity>( new CreateIndexModel<MongoTextIndexEntity<T>>(
Index.Ascending(x => x.DocId)), Index.Ascending(x => x.DocId)),
new CreateIndexModel<MongoTextIndexEntity>( new CreateIndexModel<MongoTextIndexEntity<T>>(
Index
.Text("t.t")
.Ascending(x => x.AppId)
.Ascending(x => x.ServeAll)
.Ascending(x => x.ServePublished)
.Ascending(x => x.SchemaId)),
new CreateIndexModel<MongoTextIndexEntity>(
Index Index
.Ascending(x => x.AppId) .Ascending(x => x.AppId)
.Ascending(x => x.ServeAll) .Ascending(x => x.ServeAll)
@ -81,20 +78,22 @@ namespace Squidex.Domain.Apps.Entities.MongoDb.FullText
return "TextIndex"; return "TextIndex";
} }
protected abstract T BuildTexts(Dictionary<string, string> source);
async Task IDeleter.DeleteAppAsync(IAppEntity app, async Task IDeleter.DeleteAppAsync(IAppEntity app,
CancellationToken ct) CancellationToken ct)
{ {
await Collection.DeleteManyAsync(Filter.Eq(x => x.AppId, app.Id), ct); await Collection.DeleteManyAsync(Filter.Eq(x => x.AppId, app.Id), ct);
} }
public Task ExecuteAsync(IndexCommand[] commands, public virtual Task ExecuteAsync(IndexCommand[] commands,
CancellationToken ct = default) CancellationToken ct = default)
{ {
var writes = new List<WriteModel<MongoTextIndexEntity>>(commands.Length); var writes = new List<WriteModel<MongoTextIndexEntity<T>>>(commands.Length);
foreach (var command in commands) foreach (var command in commands)
{ {
CommandFactory.CreateCommands(command, writes); commandFactory.CreateCommands(command, writes);
} }
if (writes.Count == 0) if (writes.Count == 0)
@ -105,9 +104,12 @@ namespace Squidex.Domain.Apps.Entities.MongoDb.FullText
return Collection.BulkWriteAsync(writes, BulkUnordered, ct); return Collection.BulkWriteAsync(writes, BulkUnordered, ct);
} }
public async Task<List<DomainId>?> SearchAsync(IAppEntity app, GeoQuery query, SearchScope scope, public virtual async Task<List<DomainId>?> SearchAsync(IAppEntity app, GeoQuery query, SearchScope scope,
CancellationToken ct = default) CancellationToken ct = default)
{ {
Guard.NotNull(app, nameof(app));
Guard.NotNull(query, nameof(query));
var findFilter = var findFilter =
Filter.And( Filter.And(
Filter.Eq(x => x.AppId, app.Id), Filter.Eq(x => x.AppId, app.Id),
@ -122,45 +124,43 @@ namespace Squidex.Domain.Apps.Entities.MongoDb.FullText
return byGeo.Select(x => x.ContentId).ToList(); return byGeo.Select(x => x.ContentId).ToList();
} }
public async Task<List<DomainId>?> SearchAsync(IAppEntity app, TextQuery query, SearchScope scope, public virtual async Task<List<DomainId>?> SearchAsync(IAppEntity app, TextQuery query, SearchScope scope,
CancellationToken ct = default) CancellationToken ct = default)
{ {
if (string.IsNullOrWhiteSpace(query.Text)) Guard.NotNull(app, nameof(app));
Guard.NotNull(query, nameof(query));
var (search, take) = query;
if (string.IsNullOrWhiteSpace(search))
{ {
return null; return null;
} }
List<(DomainId, double)> documents; var result = new List<(DomainId Id, double Score)>();
if (query.RequiredSchemaIds?.Count > 0) if (query.RequiredSchemaIds?.Count > 0)
{ {
documents = await SearchBySchemaAsync(query.Text, app, query.RequiredSchemaIds, scope, query.Take, 1, ct); await SearchBySchemaAsync(result, search, app, query.RequiredSchemaIds, scope, take, 1, ct);
} }
else if (query.PreferredSchemaId == null) else if (query.PreferredSchemaId == null)
{ {
documents = await SearchByAppAsync(query.Text, app, scope, query.Take, 1, ct); await SearchByAppAsync(result, search, app, scope, take, 1, ct);
} }
else else
{ {
var halfBucket = query.Take / 2; var halfBucket = take / 2;
var schemaIds = Enumerable.Repeat(query.PreferredSchemaId.Value, 1); var schemaIds = Enumerable.Repeat(query.PreferredSchemaId.Value, 1);
documents = await SearchBySchemaAsync( await SearchBySchemaAsync(result, search, app, schemaIds, scope, halfBucket, 1.1, ct);
query.Text, await SearchByAppAsync(result, search, app, scope, halfBucket, 1, ct);
app,
schemaIds,
scope,
halfBucket, 1,
ct);
documents.AddRange(await SearchByAppAsync(query.Text, app, scope, halfBucket, 1, ct));
} }
return documents.OrderByDescending(x => x.Item2).Select(x => x.Item1).Distinct().ToList(); return result.OrderByDescending(x => x.Score).Select(x => x.Id).Distinct().ToList();
} }
private Task<List<(DomainId, double)>> SearchBySchemaAsync(string text, IAppEntity app, IEnumerable<DomainId> schemaIds, SearchScope scope, int limit, double factor, private Task SearchBySchemaAsync(List<(DomainId, double)> result, string text, IAppEntity app, IEnumerable<DomainId> schemaIds, SearchScope scope, int take, double factor,
CancellationToken ct = default) CancellationToken ct = default)
{ {
var filter = var filter =
@ -170,10 +170,10 @@ namespace Squidex.Domain.Apps.Entities.MongoDb.FullText
Filter_ByScope(scope), Filter_ByScope(scope),
Filter.Text(text, "none")); Filter.Text(text, "none"));
return SearchAsync(filter, scope, limit, factor, ct); return SearchAsync(result, filter, scope, take, factor, ct);
} }
private Task<List<(DomainId, double)>> SearchByAppAsync(string text, IAppEntity app, SearchScope scope, int limit, double factor, private Task SearchByAppAsync(List<(DomainId, double)> result, string text, IAppEntity app, SearchScope scope, int take, double factor,
CancellationToken ct = default) CancellationToken ct = default)
{ {
var filter = var filter =
@ -183,24 +183,24 @@ namespace Squidex.Domain.Apps.Entities.MongoDb.FullText
Filter_ByScope(scope), Filter_ByScope(scope),
Filter.Text(text, "none")); Filter.Text(text, "none"));
return SearchAsync(filter, scope, limit, factor, ct); return SearchAsync(result, filter, scope, take, factor, ct);
} }
private async Task<List<(DomainId, double)>> SearchAsync(FilterDefinition<MongoTextIndexEntity> filter, SearchScope scope, int limit, double factor, private async Task SearchAsync(List<(DomainId, double)> result, FilterDefinition<MongoTextIndexEntity<T>> filter, SearchScope scope, int take, double factor,
CancellationToken ct = default) CancellationToken ct = default)
{ {
var collection = GetCollection(scope); var collection = GetCollection(scope);
var find = var find =
collection.Find(filter).Limit(limit) collection.Find(filter).Limit(take)
.Project<MongoTextResult>(searchTextProjection).Sort(Sort.MetaTextScore("score")); .Project<MongoTextResult>(searchTextProjection).Sort(Sort.MetaTextScore("score"));
var documents = await find.ToListAsync(ct); var documents = await find.ToListAsync(ct);
return documents.Select(x => (x.ContentId, x.Score * factor)).ToList(); result.AddRange(documents.Select(x => (x.ContentId, x.Score * factor)));
} }
private static FilterDefinition<MongoTextIndexEntity> Filter_ByScope(SearchScope scope) private static FilterDefinition<MongoTextIndexEntity<T>> Filter_ByScope(SearchScope scope)
{ {
if (scope == SearchScope.All) if (scope == SearchScope.All)
{ {
@ -212,7 +212,7 @@ namespace Squidex.Domain.Apps.Entities.MongoDb.FullText
} }
} }
private IMongoCollection<MongoTextIndexEntity> GetCollection(SearchScope scope) private IMongoCollection<MongoTextIndexEntity<T>> GetCollection(SearchScope scope)
{ {
if (scope == SearchScope.All) if (scope == SearchScope.All)
{ {

7
backend/src/Squidex.Domain.Apps.Entities.MongoDb/FullText/MongoTextIndexEntity.cs → backend/src/Squidex.Domain.Apps.Entities.MongoDb/Text/MongoTextIndexEntity.cs

@ -5,16 +5,15 @@
// All rights reserved. Licensed under the MIT license. // All rights reserved. Licensed under the MIT license.
// ========================================================================== // ==========================================================================
using System.Collections.Generic;
using GeoJSON.Net; using GeoJSON.Net;
using MongoDB.Bson; using MongoDB.Bson;
using MongoDB.Bson.Serialization.Attributes; using MongoDB.Bson.Serialization.Attributes;
using Squidex.Infrastructure; using Squidex.Infrastructure;
using Squidex.Infrastructure.MongoDb; using Squidex.Infrastructure.MongoDb;
namespace Squidex.Domain.Apps.Entities.MongoDb.FullText namespace Squidex.Domain.Apps.Entities.MongoDb.Text
{ {
public sealed class MongoTextIndexEntity public sealed class MongoTextIndexEntity<T>
{ {
[BsonId] [BsonId]
[BsonElement] [BsonElement]
@ -50,7 +49,7 @@ namespace Squidex.Domain.Apps.Entities.MongoDb.FullText
[BsonIgnoreIfNull] [BsonIgnoreIfNull]
[BsonElement("t")] [BsonElement("t")]
public List<MongoTextIndexEntityText> Texts { get; set; } public T Texts { get; set; }
[BsonIgnoreIfNull] [BsonIgnoreIfNull]
[BsonElement("gf")] [BsonElement("gf")]

2
backend/src/Squidex.Domain.Apps.Entities.MongoDb/FullText/MongoTextIndexEntityText.cs → backend/src/Squidex.Domain.Apps.Entities.MongoDb/Text/MongoTextIndexEntityText.cs

@ -7,7 +7,7 @@
using MongoDB.Bson.Serialization.Attributes; using MongoDB.Bson.Serialization.Attributes;
namespace Squidex.Domain.Apps.Entities.MongoDb.FullText namespace Squidex.Domain.Apps.Entities.MongoDb.Text
{ {
public sealed class MongoTextIndexEntityText public sealed class MongoTextIndexEntityText
{ {

2
backend/src/Squidex.Domain.Apps.Entities.MongoDb/FullText/MongoTextIndexerState.cs → backend/src/Squidex.Domain.Apps.Entities.MongoDb/Text/MongoTextIndexerState.cs

@ -16,7 +16,7 @@ using Squidex.Domain.Apps.Entities.Contents.Text.State;
using Squidex.Infrastructure; using Squidex.Infrastructure;
using Squidex.Infrastructure.MongoDb; using Squidex.Infrastructure.MongoDb;
namespace Squidex.Domain.Apps.Entities.MongoDb.FullText namespace Squidex.Domain.Apps.Entities.MongoDb.Text
{ {
public sealed class MongoTextIndexerState : MongoRepositoryBase<TextContentState>, ITextIndexerState, IDeleter public sealed class MongoTextIndexerState : MongoRepositoryBase<TextContentState>, ITextIndexerState, IDeleter
{ {

14
backend/src/Squidex.Domain.Apps.Entities/Contents/Text/Query.cs

@ -0,0 +1,14 @@
// ==========================================================================
// Squidex Headless CMS
// ==========================================================================
// Copyright (c) Squidex UG (haftungsbeschraenkt)
// All rights reserved. Licensed under the MIT license.
// ==========================================================================
namespace Squidex.Domain.Apps.Entities.Contents.Text
{
public sealed class Query
{
public string Text { get; init; }
}
}

82
backend/src/Squidex.Domain.Apps.Entities/Contents/Text/QueryParser.cs

@ -0,0 +1,82 @@
// ==========================================================================
// Squidex Headless CMS
// ==========================================================================
// Copyright (c) Squidex UG (haftungsbeschraenkt)
// All rights reserved. Licensed under the MIT license.
// ==========================================================================
using System;
using System.Text;
namespace Squidex.Domain.Apps.Entities.Contents.Text
{
public sealed class QueryParser
{
private readonly Func<string, string> fieldProvider;
public QueryParser(Func<string, string> fieldProvider)
{
this.fieldProvider = fieldProvider;
}
public Query? Parse(string text)
{
if (string.IsNullOrWhiteSpace(text))
{
return null;
}
text = text.Trim();
text = ConvertFieldNames(text);
return new Query
{
Text = text
};
}
private string ConvertFieldNames(string query)
{
var indexOfColon = query.IndexOf(':', StringComparison.Ordinal);
if (indexOfColon < 0)
{
return query;
}
var sb = new StringBuilder();
int position = 0, lastIndexOfColon = 0;
while (indexOfColon >= 0)
{
lastIndexOfColon = indexOfColon;
var i = 0;
for (i = indexOfColon - 1; i >= position; i--)
{
var c = query[i];
if (!char.IsLetter(c) && c != '-' && c != '_')
{
break;
}
}
i++;
sb.Append(query[position..i]);
sb.Append(fieldProvider(query[i..indexOfColon]));
position = indexOfColon + 1;
indexOfColon = query.IndexOf(':', position);
}
sb.Append(query[lastIndexOfColon..]);
return sb.ToString();
}
}
}

20
backend/src/Squidex/Config/Domain/StoreServices.cs

@ -24,10 +24,10 @@ using Squidex.Domain.Apps.Entities.History.Repositories;
using Squidex.Domain.Apps.Entities.MongoDb.Apps; using Squidex.Domain.Apps.Entities.MongoDb.Apps;
using Squidex.Domain.Apps.Entities.MongoDb.Assets; using Squidex.Domain.Apps.Entities.MongoDb.Assets;
using Squidex.Domain.Apps.Entities.MongoDb.Contents; using Squidex.Domain.Apps.Entities.MongoDb.Contents;
using Squidex.Domain.Apps.Entities.MongoDb.FullText;
using Squidex.Domain.Apps.Entities.MongoDb.History; using Squidex.Domain.Apps.Entities.MongoDb.History;
using Squidex.Domain.Apps.Entities.MongoDb.Rules; using Squidex.Domain.Apps.Entities.MongoDb.Rules;
using Squidex.Domain.Apps.Entities.MongoDb.Schemas; using Squidex.Domain.Apps.Entities.MongoDb.Schemas;
using Squidex.Domain.Apps.Entities.MongoDb.Text;
using Squidex.Domain.Apps.Entities.Rules.DomainObject; using Squidex.Domain.Apps.Entities.Rules.DomainObject;
using Squidex.Domain.Apps.Entities.Rules.Repositories; using Squidex.Domain.Apps.Entities.Rules.Repositories;
using Squidex.Domain.Apps.Entities.Schemas; using Squidex.Domain.Apps.Entities.Schemas;
@ -135,9 +135,6 @@ namespace Squidex.Config.Domain
services.AddSingletonAs<MongoSchemasHash>() services.AddSingletonAs<MongoSchemasHash>()
.AsOptional<ISchemasHash>().As<IEventConsumer>().As<IDeleter>(); .AsOptional<ISchemasHash>().As<IEventConsumer>().As<IDeleter>();
services.AddSingletonAs<MongoTextIndex>()
.AsOptional<ITextIndex>().As<IDeleter>();
services.AddSingletonAs<MongoTextIndexerState>() services.AddSingletonAs<MongoTextIndexerState>()
.As<ITextIndexerState>().As<IDeleter>(); .As<ITextIndexerState>().As<IDeleter>();
@ -151,6 +148,21 @@ namespace Squidex.Config.Domain
builder.SetDefaultScopeEntity<ImmutableScope>(); builder.SetDefaultScopeEntity<ImmutableScope>();
builder.SetDefaultApplicationEntity<ImmutableApplication>(); builder.SetDefaultApplicationEntity<ImmutableApplication>();
}); });
var atlasOptions = config.GetSection("store:mongoDb:atlas").Get<AtlasOptions>() ?? new ();
if (atlasOptions.IsConfigured() && atlasOptions.FullTextEnabled)
{
services.Configure<AtlasOptions>(config.GetSection("store:mongoDb:atlas"));
services.AddSingletonAs<AtlasTextIndex>()
.AsOptional<ITextIndex>().As<IDeleter>();
}
else
{
services.AddSingletonAs<MongoTextIndex>()
.AsOptional<ITextIndex>().As<IDeleter>();
}
} }
}); });

2
backend/src/Squidex/Squidex.csproj

@ -161,4 +161,6 @@
<PropertyGroup> <PropertyGroup>
<NoWarn>$(NoWarn);CS1591;1591;1573;1572;NU1605;IDE0060</NoWarn> <NoWarn>$(NoWarn);CS1591;1591;1573;1572;NU1605;IDE0060</NoWarn>
</PropertyGroup> </PropertyGroup>
<ProjectExtensions><VisualStudio><UserProperties appsettings_1json__JsonSchema="https://json.schemastore.org/bamboo-spec.json" /></VisualStudio></ProjectExtensions>
</Project> </Project>

17
backend/src/Squidex/appsettings.json

@ -437,7 +437,22 @@
"contentDatabase": "SquidexContent", "contentDatabase": "SquidexContent",
// The database for all your other read collections. // The database for all your other read collections.
"database": "Squidex" "database": "Squidex",
"atlas": {
// The organization id.
"groupId": "",
// The name of the cluster.
"clusterName": "",
// Credentials to your account.
"publicKey": "",
"privateKey": "",
// True, if you want to enable mongo atlas for full text search instead of MongoDB.
"fullTextEnabled": false
}
} }
}, },

24
backend/tests/Squidex.Domain.Apps.Entities.Tests/Assets/MongoDb/AssetQueryTests.cs

@ -28,8 +28,6 @@ namespace Squidex.Domain.Apps.Entities.Assets.MongoDb
{ {
public class AssetQueryTests public class AssetQueryTests
{ {
private static readonly IBsonSerializerRegistry Registry = BsonSerializer.SerializerRegistry;
private static readonly IBsonSerializer<MongoAssetEntity> Serializer = BsonSerializer.SerializerRegistry.GetSerializer<MongoAssetEntity>();
private readonly DomainId appId = DomainId.NewGuid(); private readonly DomainId appId = DomainId.NewGuid();
static AssetQueryTests() static AssetQueryTests()
@ -212,13 +210,15 @@ namespace Squidex.Domain.Apps.Entities.Assets.MongoDb
private void AssertQuery(string expected, ClrQuery query, object? arg = null) private void AssertQuery(string expected, ClrQuery query, object? arg = null)
{ {
var rendered = var filter = query.AdjustToModel(appId).BuildFilter<MongoAssetEntity>(false).Filter!;
query.AdjustToModel(appId).BuildFilter<MongoAssetEntity>(false).Filter!
.Render(Serializer, Registry).ToString();
var expectation = Cleanup(expected, arg); var rendered =
filter.Render(
BsonSerializer.SerializerRegistry.GetSerializer<MongoAssetEntity>(),
BsonSerializer.SerializerRegistry)
.ToString();
Assert.Equal(expectation, rendered); Assert.Equal(Cleanup(expected, arg), rendered);
} }
private void AssertSorting(string expected, params SortNode[] sort) private void AssertSorting(string expected, params SortNode[] sort)
@ -230,14 +230,16 @@ namespace Squidex.Domain.Apps.Entities.Assets.MongoDb
A.CallTo(() => cursor.Sort(A<SortDefinition<MongoAssetEntity>>._)) A.CallTo(() => cursor.Sort(A<SortDefinition<MongoAssetEntity>>._))
.Invokes((SortDefinition<MongoAssetEntity> sortDefinition) => .Invokes((SortDefinition<MongoAssetEntity> sortDefinition) =>
{ {
rendered = sortDefinition.Render(Serializer, Registry).ToString(); rendered =
sortDefinition.Render(
BsonSerializer.SerializerRegistry.GetSerializer<MongoAssetEntity>(),
BsonSerializer.SerializerRegistry)
.ToString();
}); });
cursor.QuerySort(new ClrQuery { Sort = sort.ToList() }.AdjustToModel(appId)); cursor.QuerySort(new ClrQuery { Sort = sort.ToList() }.AdjustToModel(appId));
var expectation = Cleanup(expected); Assert.Equal(Cleanup(expected), rendered);
Assert.Equal(expectation, rendered);
} }
private static string Cleanup(string filter, object? arg = null) private static string Cleanup(string filter, object? arg = null)

6
backend/tests/Squidex.Domain.Apps.Entities.Tests/Assets/MongoDb/AssetsQueryFixture.cs

@ -15,6 +15,7 @@ using Newtonsoft.Json;
using Squidex.Domain.Apps.Core.Assets; using Squidex.Domain.Apps.Core.Assets;
using Squidex.Domain.Apps.Core.TestHelpers; using Squidex.Domain.Apps.Core.TestHelpers;
using Squidex.Domain.Apps.Entities.MongoDb.Assets; using Squidex.Domain.Apps.Entities.MongoDb.Assets;
using Squidex.Domain.Apps.Entities.TestHelpers;
using Squidex.Infrastructure; using Squidex.Infrastructure;
using Squidex.Infrastructure.Json.Objects; using Squidex.Infrastructure.Json.Objects;
using Squidex.Infrastructure.MongoDb; using Squidex.Infrastructure.MongoDb;
@ -25,7 +26,7 @@ namespace Squidex.Domain.Apps.Entities.Assets.MongoDb
{ {
private readonly Random random = new Random(); private readonly Random random = new Random();
private readonly int numValues = 250; private readonly int numValues = 250;
private readonly IMongoClient mongoClient = new MongoClient("mongodb://localhost"); private readonly IMongoClient mongoClient;
private readonly IMongoDatabase mongoDatabase; private readonly IMongoDatabase mongoDatabase;
public MongoAssetRepository AssetRepository { get; } public MongoAssetRepository AssetRepository { get; }
@ -38,7 +39,8 @@ namespace Squidex.Domain.Apps.Entities.Assets.MongoDb
public AssetsQueryFixture() public AssetsQueryFixture()
{ {
mongoDatabase = mongoClient.GetDatabase("Squidex_Testing"); mongoClient = new MongoClient(TestConfig.Configuration["mongodb:configuration"]);
mongoDatabase = mongoClient.GetDatabase(TestConfig.Configuration["mongodb:database"]);
SetupJson(); SetupJson();

24
backend/tests/Squidex.Domain.Apps.Entities.Tests/Contents/MongoDb/ContentQueryTests.cs

@ -32,8 +32,6 @@ namespace Squidex.Domain.Apps.Entities.Contents.MongoDb
{ {
public class ContentQueryTests public class ContentQueryTests
{ {
private static readonly IBsonSerializerRegistry Registry = BsonSerializer.SerializerRegistry;
private static readonly IBsonSerializer<MongoContentEntity> Serializer = BsonSerializer.SerializerRegistry.GetSerializer<MongoContentEntity>();
private readonly DomainId appId = DomainId.NewGuid(); private readonly DomainId appId = DomainId.NewGuid();
private readonly Schema schemaDef; private readonly Schema schemaDef;
private readonly LanguagesConfig languagesConfig = LanguagesConfig.English.Set(Language.DE); private readonly LanguagesConfig languagesConfig = LanguagesConfig.English.Set(Language.DE);
@ -233,13 +231,15 @@ namespace Squidex.Domain.Apps.Entities.Contents.MongoDb
private void AssertQuery(ClrQuery query, string expected, object? arg = null) private void AssertQuery(ClrQuery query, string expected, object? arg = null)
{ {
var rendered = var filter = query.AdjustToModel(appId).BuildFilter<MongoContentEntity>(false).Filter!;
query.AdjustToModel(appId).BuildFilter<MongoContentEntity>().Filter!
.Render(Serializer, Registry).ToString();
var expectation = Cleanup(expected, arg); var rendered =
filter.Render(
BsonSerializer.SerializerRegistry.GetSerializer<MongoContentEntity>(),
BsonSerializer.SerializerRegistry)
.ToString();
Assert.Equal(expectation, rendered); Assert.Equal(Cleanup(expected, arg), rendered);
} }
private void AssertSorting(string expected, params SortNode[] sort) private void AssertSorting(string expected, params SortNode[] sort)
@ -251,14 +251,16 @@ namespace Squidex.Domain.Apps.Entities.Contents.MongoDb
A.CallTo(() => cursor.Sort(A<SortDefinition<MongoContentEntity>>._)) A.CallTo(() => cursor.Sort(A<SortDefinition<MongoContentEntity>>._))
.Invokes((SortDefinition<MongoContentEntity> sortDefinition) => .Invokes((SortDefinition<MongoContentEntity> sortDefinition) =>
{ {
rendered = sortDefinition.Render(Serializer, Registry).ToString(); rendered =
sortDefinition.Render(
BsonSerializer.SerializerRegistry.GetSerializer<MongoContentEntity>(),
BsonSerializer.SerializerRegistry)
.ToString();
}); });
cursor.QuerySort(new ClrQuery { Sort = sort.ToList() }.AdjustToModel(appId)); cursor.QuerySort(new ClrQuery { Sort = sort.ToList() }.AdjustToModel(appId));
var expectation = Cleanup(expected); Assert.Equal(Cleanup(expected), rendered);
Assert.Equal(expectation, rendered);
} }
private static string Cleanup(string filter, object? arg = null) private static string Cleanup(string filter, object? arg = null)

5
backend/tests/Squidex.Domain.Apps.Entities.Tests/Contents/MongoDb/ContentsQueryFixture.cs

@ -33,7 +33,7 @@ namespace Squidex.Domain.Apps.Entities.Contents.MongoDb
{ {
private readonly Random random = new Random(); private readonly Random random = new Random();
private readonly int numValues = 10000; private readonly int numValues = 10000;
private readonly IMongoClient mongoClient = new MongoClient("mongodb://localhost"); private readonly IMongoClient mongoClient;
private readonly IMongoDatabase mongoDatabase; private readonly IMongoDatabase mongoDatabase;
public MongoContentRepository ContentRepository { get; } public MongoContentRepository ContentRepository { get; }
@ -55,7 +55,8 @@ namespace Squidex.Domain.Apps.Entities.Contents.MongoDb
public ContentsQueryFixture() public ContentsQueryFixture()
{ {
mongoDatabase = mongoClient.GetDatabase("Squidex_Testing"); mongoClient = new MongoClient(TestConfig.Configuration["mongodb:configuration"]);
mongoDatabase = mongoClient.GetDatabase(TestConfig.Configuration["mongodb:database"]);
SetupJson(); SetupJson();

423
backend/tests/Squidex.Domain.Apps.Entities.Tests/Contents/Text/AtlasParsingTests.cs

@ -0,0 +1,423 @@
// ==========================================================================
// Squidex Headless CMS
// ==========================================================================
// Copyright (c) Squidex UG (haftungsbeschraenkt)
// All rights reserved. Licensed under the MIT license.
// ==========================================================================
using System.IO;
using System.Text;
using System.Text.Json;
using Lucene.Net.Analysis.Standard;
using Lucene.Net.Analysis.Util;
using Lucene.Net.Util;
using MongoDB.Bson;
using Squidex.Domain.Apps.Entities.MongoDb.Text;
using Xunit;
using LuceneQueryAnalyzer = Lucene.Net.QueryParsers.Classic.QueryParser;
namespace Squidex.Domain.Apps.Entities.Contents.Text
{
public class AtlasParsingTests
{
private static readonly LuceneQueryVisitor QueryVisitor = new LuceneQueryVisitor();
private static readonly LuceneQueryAnalyzer QueryParser =
new LuceneQueryAnalyzer(LuceneVersion.LUCENE_48, "*",
new StandardAnalyzer(LuceneVersion.LUCENE_48, CharArraySet.EMPTY_SET));
private static readonly JsonSerializerOptions JsonSerializerOptions = new JsonSerializerOptions
{
WriteIndented = true
};
[Fact]
public void Should_parse_term_query()
{
var actual = ParseQuery("hello");
var expected = CreateQuery(new
{
text = new
{
path = new
{
wildcard = "*"
},
query = "hello"
}
});
Assert.Equal(expected, actual);
}
[Fact]
public void Should_parse_phrase_query()
{
var actual = ParseQuery("\"hello dolly\"");
var expected = CreateQuery(new
{
phrase = new
{
path = new
{
wildcard = "*"
},
query = new[] { "hello", "dolly" }
}
});
Assert.Equal(expected, actual);
}
[Fact]
public void Should_parse_compound_phrase_query()
{
var actual = ParseQuery("title:\"The Right Way\" AND text:go");
var expected = CreateQuery(new
{
compound = new
{
must = new object[]
{
new
{
phrase = new
{
path = "title",
query = new[]
{
"the",
"right",
"way"
}
}
},
new
{
text = new
{
path = "text",
query = "go"
}
}
}
}
});
Assert.Equal(expected, actual);
}
[Fact]
public void Should_parse_compound_phrase_query_with_widldcard()
{
var actual = ParseQuery("title:\"Do it right\" AND right");
var expected = CreateQuery(new
{
compound = new
{
must = new object[]
{
new
{
phrase = new
{
path = "title",
query = new[]
{
"do",
"it",
"right"
}
}
},
new
{
text = new
{
path = new
{
wildcard = "*"
},
query = "right"
}
}
}
}
});
Assert.Equal(expected, actual);
}
[Fact]
public void Should_parse_wildcard_query()
{
var actual = ParseQuery("te?t");
var expected = CreateQuery(new
{
wildcard = new
{
path = new
{
wildcard = "*"
},
query = "te?t"
}
});
Assert.Equal(expected, actual);
}
[Fact]
public void Should_parse_prefix_query()
{
var actual = ParseQuery("test*");
var expected = CreateQuery(new
{
wildcard = new
{
path = new
{
wildcard = "*"
},
query = "test*"
}
});
Assert.Equal(expected, actual);
}
[Fact]
public void Should_parse_fuzzy_query()
{
var actual = ParseQuery("roam~");
var expected = CreateQuery(new
{
text = new
{
path = new
{
wildcard = "*"
},
query = "roam",
fuzzy = new
{
maxEdits = 2
}
}
});
Assert.Equal(expected, actual);
}
[Fact]
public void Should_parse_fuzzy_query_with_max_edits()
{
var actual = ParseQuery("roam~1");
var expected = CreateQuery(new
{
text = new
{
path = new
{
wildcard = "*"
},
query = "roam",
fuzzy = new
{
maxEdits = 1
}
}
});
Assert.Equal(expected, actual);
}
[Fact]
public void Should_parse_fuzzy_phrase_query_with_slop()
{
var actual = ParseQuery("\"jakarta apache\"~10");
var expected = CreateQuery(new
{
phrase = new
{
path = new
{
wildcard = "*"
},
query = new[]
{
"jakarta",
"apache"
},
slop = 10
}
});
Assert.Equal(expected, actual);
}
[Fact]
public void Should_parse_compound_query_with_brackets()
{
var actual = ParseQuery("(jakarta OR apache) AND website");
var expected = CreateQuery(new
{
compound = new
{
must = new object[]
{
new
{
compound = new
{
should = new object[]
{
new
{
text = new
{
path = new
{
wildcard = "*"
},
query = "jakarta"
}
},
new
{
text = new
{
path = new
{
wildcard = "*"
},
query = "apache"
}
}
}
}
},
new
{
text = new
{
path = new
{
wildcard = "*"
},
query = "website"
}
}
}
}
});
Assert.Equal(expected, actual);
}
[Fact]
public void Should_parse_compound_query_and_optimize()
{
var actual = ParseQuery("title:(+return +\"pink panther\")");
var expected = CreateQuery(new
{
compound = new
{
must = new object[]
{
new
{
text = new
{
path = "title",
query = "return"
}
},
new
{
phrase = new
{
path = "title",
query = new[]
{
"pink",
"panther"
}
}
}
}
}
});
Assert.Equal(expected, actual);
}
[Fact]
public void Should_parse_range_query()
{
var actual = ParseQuery("mod_date:[20020101 TO 20030101]");
var expected = CreateQuery(new
{
range = new
{
path = "mod_date",
gte = 20020101,
lte = 20030101
}
});
Assert.Equal(expected, actual);
}
[Fact]
public void Should_parse_open_range_query()
{
var actual = ParseQuery("mod_date:{20020101 TO 20030101}");
var expected = CreateQuery(new
{
range = new
{
path = "mod_date",
gt = 20020101,
lt = 20030101
}
});
Assert.Equal(expected, actual);
}
private static object CreateQuery(object query)
{
return JsonSerializer.Serialize(query, JsonSerializerOptions);
}
private static object ParseQuery(string query)
{
var luceneQuery = QueryParser.Parse(query);
var rendered = QueryVisitor.Visit(luceneQuery);
var jsonStream = new MemoryStream();
var jsonDocument = JsonDocument.Parse(rendered.ToJson());
var jsonWriter = new Utf8JsonWriter(jsonStream, new JsonWriterOptions { Indented = true });
jsonDocument.WriteTo(jsonWriter);
jsonWriter.Flush();
return Encoding.UTF8.GetString(jsonStream.ToArray());
}
}
}

38
backend/tests/Squidex.Domain.Apps.Entities.Tests/Contents/Text/AtlasTextIndexFixture.cs

@ -0,0 +1,38 @@
// ==========================================================================
// Squidex Headless CMS
// ==========================================================================
// Copyright (c) Squidex UG (haftungsbeschraenkt)
// All rights reserved. Licensed under the MIT license.
// ==========================================================================
using Microsoft.Extensions.Configuration;
using Microsoft.Extensions.Options;
using MongoDB.Driver;
using Newtonsoft.Json;
using Squidex.Domain.Apps.Core.TestHelpers;
using Squidex.Domain.Apps.Entities.MongoDb.Text;
using Squidex.Domain.Apps.Entities.TestHelpers;
using Squidex.Infrastructure.MongoDb;
namespace Squidex.Domain.Apps.Entities.Contents.Text
{
public sealed class AtlasTextIndexFixture
{
public AtlasTextIndex Index { get; }
public AtlasTextIndexFixture()
{
BsonJsonConvention.Register(JsonSerializer.Create(TestUtils.CreateSerializerSettings()));
DomainIdSerializer.Register();
var mongoClient = new MongoClient(TestConfig.Configuration["atlas:configuration"]);
var mongoDatabase = mongoClient.GetDatabase(TestConfig.Configuration["atlas:database"]);
var options = TestConfig.Configuration.GetSection("atlas").Get<AtlasOptions>();
Index = new AtlasTextIndex(mongoDatabase, Options.Create(options), false);
Index.InitializeAsync(default).Wait();
}
}
}

62
backend/tests/Squidex.Domain.Apps.Entities.Tests/Contents/Text/AtlasTextIndexTests.cs

@ -0,0 +1,62 @@
// ==========================================================================
// Squidex Headless CMS
// ==========================================================================
// Copyright (c) Squidex UG (haftungsbeschraenkt)
// All rights reserved. Licensed under the MIT license.
// ==========================================================================
using System.Threading.Tasks;
using Xunit;
#pragma warning disable SA1300 // Element should begin with upper-case letter
namespace Squidex.Domain.Apps.Entities.Contents.Text
{
[Trait("Category", "Dependencies")]
public class AtlasTextIndexTests : TextIndexerTestsBase, IClassFixture<AtlasTextIndexFixture>
{
public override bool SupportsQuerySyntax => true;
public override bool SupportsGeo => true;
public override int WaitAfterUpdate => 2000;
public AtlasTextIndexFixture _ { get; }
public AtlasTextIndexTests(AtlasTextIndexFixture fixture)
{
_ = fixture;
}
public override ITextIndex CreateIndex()
{
return _.Index;
}
[Fact]
public async Task Should_retrieve_english_stopword_only_for_german_query()
{
await CreateTextAsync(ids1[0], "de", "and und");
await CreateTextAsync(ids2[0], "en", "and und");
await SearchText(expected: ids2, text: "und");
}
[Fact]
public async Task Should_retrieve_german_stopword_only_for_english_query()
{
await CreateTextAsync(ids1[0], "de", "and und");
await CreateTextAsync(ids2[0], "en", "and und");
await SearchText(expected: ids1, text: "and");
}
[Fact]
public async Task Should_index_cjk_content_and_retrieve()
{
await CreateTextAsync(ids1[0], "zh", "東京大学");
await SearchText(expected: ids1, text: "東京");
}
}
}

26
backend/tests/Squidex.Domain.Apps.Entities.Tests/Contents/Text/AzureTextIndexFixture.cs

@ -0,0 +1,26 @@
// ==========================================================================
// Squidex Headless CMS
// ==========================================================================
// Copyright (c) Squidex UG (haftungsbeschraenkt)
// All rights reserved. Licensed under the MIT license.
// ==========================================================================
using Squidex.Domain.Apps.Entities.TestHelpers;
using Squidex.Extensions.Text.Azure;
namespace Squidex.Domain.Apps.Entities.Contents.Text
{
public sealed class AzureTextIndexFixture
{
public AzureTextIndex Index { get; }
public AzureTextIndexFixture()
{
Index = new AzureTextIndex(
TestConfig.Configuration["azureText:serviceEndpoint"],
TestConfig.Configuration["azureText:apiKey"],
TestConfig.Configuration["azureText:indexName"]);
Index.InitializeAsync(default).Wait();
}
}
}

22
backend/tests/Squidex.Domain.Apps.Entities.Tests/Contents/Text/TextIndexerTests_Elastic.cs → backend/tests/Squidex.Domain.Apps.Entities.Tests/Contents/Text/AzureTextIndexTests.cs

@ -6,21 +6,29 @@
// ========================================================================== // ==========================================================================
using System.Threading.Tasks; using System.Threading.Tasks;
using Squidex.Extensions.Text.ElasticSearch;
using Xunit; using Xunit;
#pragma warning disable SA1300 // Element should begin with upper-case letter
namespace Squidex.Domain.Apps.Entities.Contents.Text namespace Squidex.Domain.Apps.Entities.Contents.Text
{ {
[Trait("Category", "Dependencies")] [Trait("Category", "Dependencies")]
public class TextIndexerTests_Elastic : TextIndexerTestsBase public class AzureTextIndexTests : TextIndexerTestsBase, IClassFixture<AzureTextIndexFixture>
{ {
public override ITextIndex CreateIndex() public override bool SupportsGeo => true;
{
var index = new ElasticSearchTextIndex("http://localhost:9200", "squidex", 1000);
index.InitializeAsync(default).Wait(); public override int WaitAfterUpdate => 2000;
return index; public AzureTextIndexFixture _ { get; }
public AzureTextIndexTests(AzureTextIndexFixture fixture)
{
_ = fixture;
}
public override ITextIndex CreateIndex()
{
return _.Index;
} }
[Fact] [Fact]

25
backend/tests/Squidex.Domain.Apps.Entities.Tests/Contents/Text/ElasticTextIndexFixture.cs

@ -0,0 +1,25 @@
// ==========================================================================
// Squidex Headless CMS
// ==========================================================================
// Copyright (c) Squidex UG (haftungsbeschraenkt)
// All rights reserved. Licensed under the MIT license.
// ==========================================================================
using Squidex.Domain.Apps.Entities.TestHelpers;
using Squidex.Extensions.Text.ElasticSearch;
namespace Squidex.Domain.Apps.Entities.Contents.Text
{
public sealed class ElasticTextIndexFixture
{
public ElasticSearchTextIndex Index { get; }
public ElasticTextIndexFixture()
{
Index = new ElasticSearchTextIndex(
TestConfig.Configuration["elastic:configuration"],
TestConfig.Configuration["elastic:indexName"]);
Index.InitializeAsync(default).Wait();
}
}
}

22
backend/tests/Squidex.Domain.Apps.Entities.Tests/Contents/Text/TextIndexerTests_Azure.cs → backend/tests/Squidex.Domain.Apps.Entities.Tests/Contents/Text/ElasticTextIndexTests.cs

@ -6,21 +6,29 @@
// ========================================================================== // ==========================================================================
using System.Threading.Tasks; using System.Threading.Tasks;
using Squidex.Extensions.Text.Azure;
using Xunit; using Xunit;
#pragma warning disable SA1300 // Element should begin with upper-case letter
namespace Squidex.Domain.Apps.Entities.Contents.Text namespace Squidex.Domain.Apps.Entities.Contents.Text
{ {
[Trait("Category", "Dependencies")] [Trait("Category", "Dependencies")]
public class TextIndexerTests_Azure : TextIndexerTestsBase public class ElasticTextIndexTests : TextIndexerTestsBase, IClassFixture<ElasticTextIndexFixture>
{ {
public override ITextIndex CreateIndex() public override bool SupportsGeo => true;
{
var index = new AzureTextIndex("https://squidex.search.windows.net", "API_KEY", "test", 2000);
index.InitializeAsync(default).Wait(); public override int WaitAfterUpdate => 2000;
return index; public ElasticTextIndexFixture _ { get; }
public ElasticTextIndexTests(ElasticTextIndexFixture fixture)
{
_ = fixture;
}
public override ITextIndex CreateIndex()
{
return _.Index;
} }
[Fact] [Fact]

34
backend/tests/Squidex.Domain.Apps.Entities.Tests/Contents/Text/MongoTextIndexFixture.cs

@ -0,0 +1,34 @@
// ==========================================================================
// Squidex Headless CMS
// ==========================================================================
// Copyright (c) Squidex UG (haftungsbeschraenkt)
// All rights reserved. Licensed under the MIT license.
// ==========================================================================
using MongoDB.Driver;
using Newtonsoft.Json;
using Squidex.Domain.Apps.Core.TestHelpers;
using Squidex.Domain.Apps.Entities.MongoDb.Text;
using Squidex.Domain.Apps.Entities.TestHelpers;
using Squidex.Infrastructure.MongoDb;
namespace Squidex.Domain.Apps.Entities.Contents.Text
{
public sealed class MongoTextIndexFixture
{
public MongoTextIndex Index { get; }
public MongoTextIndexFixture()
{
BsonJsonConvention.Register(JsonSerializer.Create(TestUtils.CreateSerializerSettings()));
DomainIdSerializer.Register();
var mongoClient = new MongoClient(TestConfig.Configuration["mongodb:configuration"]);
var mongoDatabase = mongoClient.GetDatabase(TestConfig.Configuration["mongodb:database"]);
Index = new MongoTextIndex(mongoDatabase, false);
Index.InitializeAsync(default).Wait();
}
}
}

25
backend/tests/Squidex.Domain.Apps.Entities.Tests/Contents/Text/TextIndexerTests_Mongo.cs → backend/tests/Squidex.Domain.Apps.Entities.Tests/Contents/Text/MongoTextIndexTests.cs

@ -8,38 +8,29 @@
using System.Linq; using System.Linq;
using System.Threading.Tasks; using System.Threading.Tasks;
using MongoDB.Driver; using MongoDB.Driver;
using Newtonsoft.Json;
using Squidex.Domain.Apps.Core.TestHelpers;
using Squidex.Domain.Apps.Entities.MongoDb.FullText;
using Squidex.Infrastructure.MongoDb;
using Xunit; using Xunit;
#pragma warning disable SA1300 // Element should begin with upper-case letter
namespace Squidex.Domain.Apps.Entities.Contents.Text namespace Squidex.Domain.Apps.Entities.Contents.Text
{ {
[Trait("Category", "Dependencies")] [Trait("Category", "Dependencies")]
public class TextIndexerTests_Mongo : TextIndexerTestsBase public class MongoTextIndexTests : TextIndexerTestsBase, IClassFixture<MongoTextIndexFixture>
{ {
public override bool SupportsQuerySyntax => false; public override bool SupportsQuerySyntax => false;
public override bool SupportsGeo => true; public override bool SupportsGeo => true;
static TextIndexerTests_Mongo() public MongoTextIndexFixture _ { get; }
{
BsonJsonConvention.Register(JsonSerializer.Create(TestUtils.CreateSerializerSettings()));
DomainIdSerializer.Register(); public MongoTextIndexTests(MongoTextIndexFixture fixture)
{
_ = fixture;
} }
public override ITextIndex CreateIndex() public override ITextIndex CreateIndex()
{ {
var mongoClient = new MongoClient("mongodb://localhost"); return _.Index;
var mongoDatabase = mongoClient.GetDatabase("Squidex_Testing");
var index = new MongoTextIndex(mongoDatabase, false);
index.InitializeAsync(default).Wait();
return index;
} }
[Fact] [Fact]

48
backend/tests/Squidex.Domain.Apps.Entities.Tests/Contents/Text/QueryParserTests.cs

@ -0,0 +1,48 @@
// ==========================================================================
// Squidex Headless CMS
// ==========================================================================
// Copyright (c) Squidex UG (haftungsbeschraenkt)
// All rights reserved. Licensed under the MIT license.
// ==========================================================================
using Xunit;
namespace Squidex.Domain.Apps.Entities.Contents.Text
{
public class QueryParserTests
{
private readonly QueryParser sut = new QueryParser(x => $"texts.{x}");
[Fact]
public void Should_prefix_field_query()
{
var source = "en:Hello";
Assert.Equal("texts.en:Hello", sut.Parse(source)?.Text);
}
[Fact]
public void Should_prefix_field_with_complex_language()
{
var source = "en-EN:Hello";
Assert.Equal("texts.en-EN:Hello", sut.Parse(source)?.Text);
}
[Fact]
public void Should_prefix_field_query_within_query()
{
var source = "Hello en:World";
Assert.Equal("Hello texts.en:World", sut.Parse(source)?.Text);
}
[Fact]
public void Should_prefix_field_query_within_complex_query()
{
var source = "Hallo OR (Hello en:World)";
Assert.Equal("Hallo OR (Hello texts.en:World)", sut.Parse(source)?.Text);
}
}
}

81
backend/tests/Squidex.Domain.Apps.Entities.Tests/Contents/Text/TextIndexerTestsBase.cs

@ -5,6 +5,7 @@
// All rights reserved. Licensed under the MIT license. // All rights reserved. Licensed under the MIT license.
// ========================================================================== // ==========================================================================
using System;
using System.Collections.Generic; using System.Collections.Generic;
using System.Linq; using System.Linq;
using System.Threading.Tasks; using System.Threading.Tasks;
@ -32,12 +33,19 @@ namespace Squidex.Domain.Apps.Entities.Contents.Text
private readonly NamedId<DomainId> appId = NamedId.Of(DomainId.NewGuid(), "my-app"); private readonly NamedId<DomainId> appId = NamedId.Of(DomainId.NewGuid(), "my-app");
private readonly NamedId<DomainId> schemaId = NamedId.Of(DomainId.NewGuid(), "my-schema"); private readonly NamedId<DomainId> schemaId = NamedId.Of(DomainId.NewGuid(), "my-schema");
private readonly IAppEntity app; private readonly IAppEntity app;
private readonly TextIndexingProcess sut; private readonly Lazy<TextIndexingProcess> sut;
protected TextIndexingProcess Sut
{
get { return sut.Value; }
}
public virtual bool SupportsQuerySyntax => true; public virtual bool SupportsQuerySyntax => true;
public virtual bool SupportsGeo => false; public virtual bool SupportsGeo => false;
public virtual int WaitAfterUpdate => 0;
protected TextIndexerTestsBase() protected TextIndexerTestsBase()
{ {
app = app =
@ -45,58 +53,65 @@ namespace Squidex.Domain.Apps.Entities.Contents.Text
Language.DE, Language.DE,
Language.EN); Language.EN);
#pragma warning disable MA0056 // Do not call overridable members in constructor sut = new Lazy<TextIndexingProcess>(CreateSut);
sut = new TextIndexingProcess(TestUtils.DefaultSerializer, CreateIndex(), new InMemoryTextIndexerState());
#pragma warning restore MA0056 // Do not call overridable members in constructor
} }
public abstract ITextIndex CreateIndex(); private TextIndexingProcess CreateSut()
[SkippableFact]
public async Task Should_index_invariant_content_and_retrieve_with_fuzzy()
{ {
Skip.IfNot(SupportsQuerySyntax); var index = CreateIndex();
await CreateTextAsync(ids1[0], "iv", "Hello"); return new TextIndexingProcess(TestUtils.DefaultSerializer, index, new InMemoryTextIndexerState());
await SearchText(expected: ids1, text: "helo~");
} }
[SkippableFact] public abstract ITextIndex CreateIndex();
public async Task Should_index_invariant_content_and_retrieve_with_fuzzy_with_full_scope()
[Fact]
public async Task Should_search_with_fuzzy()
{ {
Skip.IfNot(SupportsQuerySyntax); if (!SupportsQuerySyntax)
{
return;
}
await CreateTextAsync(ids2[0], "iv", "World"); await CreateTextAsync(ids1[0], "iv", "Hello");
await SearchText(expected: ids2, text: "wold~", SearchScope.All); await SearchText(expected: ids1, text: "helo~");
} }
[SkippableFact] [Fact]
public async Task Should_search_by_field() public async Task Should_search_by_field()
{ {
Skip.IfNot(SupportsQuerySyntax); if (!SupportsQuerySyntax)
{
return;
}
await CreateTextAsync(ids1[0], "en", "City"); await CreateTextAsync(ids1[0], "en", "City");
await SearchText(expected: ids1, text: "en:city"); await SearchText(expected: ids1, text: "en:city");
} }
[SkippableFact] [Fact]
public async Task Should_search_by_geo() public async Task Should_search_by_geo()
{ {
Skip.IfNot(SupportsGeo); if (!SupportsGeo)
{
return;
}
var field = Guid.NewGuid().ToString();
// Within radius // Within search radius
await CreateGeoAsync(ids1[0], "geo", 51.343391192211506, 12.401476788622826); await CreateGeoAsync(ids1[0], field, 51.343391192211506, 12.401476788622826);
// Not in radius // Outside of search radius
await CreateGeoAsync(ids2[0], "geo", 51.30765141427311, 12.379631713912486); await CreateGeoAsync(ids2[0], field, 51.30765141427311, 12.379631713912486);
await SearchGeo(expected: ids1, "geo.iv", 51.34641682574934, 12.401965298137707); // Within search radius and correct field.
await SearchGeo(expected: ids1, $"{field}.iv", 51.34641682574934, 12.401965298137707);
// Wrong field // Within search radius but incorrect field.
await SearchGeo(expected: null, "abc.iv", 51.48596429889613, 12.102629469505713); await SearchGeo(expected: null, "other.iv", 51.48596429889613, 12.102629469505713);
} }
[Fact] [Fact]
@ -346,13 +361,15 @@ namespace Squidex.Domain.Apps.Entities.Contents.Text
return UpdateAsync(id, new ContentDeleted()); return UpdateAsync(id, new ContentDeleted());
} }
private Task UpdateAsync(DomainId id, ContentEvent contentEvent) private async Task UpdateAsync(DomainId id, ContentEvent contentEvent)
{ {
contentEvent.ContentId = id; contentEvent.ContentId = id;
contentEvent.AppId = appId; contentEvent.AppId = appId;
contentEvent.SchemaId = schemaId; contentEvent.SchemaId = schemaId;
return sut.On(Enumerable.Repeat(Envelope.Create<IEvent>(contentEvent), 1)); await Sut.On(Enumerable.Repeat(Envelope.Create<IEvent>(contentEvent), 1));
await Task.Delay(WaitAfterUpdate);
} }
private static ContentData TextData(string language, string text) private static ContentData TextData(string language, string text)
@ -375,7 +392,7 @@ namespace Squidex.Domain.Apps.Entities.Contents.Text
{ {
var query = new GeoQuery(schemaId.Id, field, latitude, longitude, 1000, 1000); var query = new GeoQuery(schemaId.Id, field, latitude, longitude, 1000, 1000);
var result = await sut.TextIndex.SearchAsync(app, query, target); var result = await Sut.TextIndex.SearchAsync(app, query, target);
if (expected != null) if (expected != null)
{ {
@ -394,7 +411,7 @@ namespace Squidex.Domain.Apps.Entities.Contents.Text
RequiredSchemaIds = new List<DomainId> { schemaId.Id } RequiredSchemaIds = new List<DomainId> { schemaId.Id }
}; };
var result = await sut.TextIndex.SearchAsync(app, query, target); var result = await Sut.TextIndex.SearchAsync(app, query, target);
if (expected != null) if (expected != null)
{ {

7
backend/tests/Squidex.Domain.Apps.Entities.Tests/Schemas/MongoDb/SchemasHashFixture.cs

@ -8,22 +8,21 @@
using System.Threading.Tasks; using System.Threading.Tasks;
using MongoDB.Driver; using MongoDB.Driver;
using Squidex.Domain.Apps.Entities.MongoDb.Schemas; using Squidex.Domain.Apps.Entities.MongoDb.Schemas;
using Squidex.Domain.Apps.Entities.TestHelpers;
using Squidex.Infrastructure.MongoDb; using Squidex.Infrastructure.MongoDb;
namespace Squidex.Domain.Apps.Entities.Schemas.MongoDb namespace Squidex.Domain.Apps.Entities.Schemas.MongoDb
{ {
public sealed class SchemasHashFixture public sealed class SchemasHashFixture
{ {
private readonly IMongoClient mongoClient = new MongoClient("mongodb://localhost");
private readonly IMongoDatabase mongoDatabase;
public MongoSchemasHash SchemasHash { get; } public MongoSchemasHash SchemasHash { get; }
public SchemasHashFixture() public SchemasHashFixture()
{ {
InstantSerializer.Register(); InstantSerializer.Register();
mongoDatabase = mongoClient.GetDatabase("Squidex_Testing"); var mongoClient = new MongoClient(TestConfig.Configuration["mongodb:configuration"]);
var mongoDatabase = mongoClient.GetDatabase(TestConfig.Configuration["mongodb:database"]);
var schemasHash = new MongoSchemasHash(mongoDatabase); var schemasHash = new MongoSchemasHash(mongoDatabase);

1
backend/tests/Squidex.Domain.Apps.Entities.Tests/Squidex.Domain.Apps.Entities.Tests.csproj

@ -41,7 +41,6 @@
<PrivateAssets>all</PrivateAssets> <PrivateAssets>all</PrivateAssets>
<IncludeAssets>runtime; build; native; contentfiles; analyzers</IncludeAssets> <IncludeAssets>runtime; build; native; contentfiles; analyzers</IncludeAssets>
</PackageReference> </PackageReference>
<PackageReference Include="Xunit.SkippableFact" Version="1.4.13" />
</ItemGroup> </ItemGroup>
<ItemGroup> <ItemGroup>
<AdditionalFiles Include="..\..\stylecop.json" Link="stylecop.json" /> <AdditionalFiles Include="..\..\stylecop.json" Link="stylecop.json" />

29
backend/tests/Squidex.Domain.Apps.Entities.Tests/TestHelpers/TestConfig.cs

@ -0,0 +1,29 @@
// ==========================================================================
// Squidex Headless CMS
// ==========================================================================
// Copyright (c) Squidex UG (haftungsbeschraenkt)
// All rights reserved. Licensed under the MIT license.
// ==========================================================================
using System.IO;
using Microsoft.Extensions.Configuration;
namespace Squidex.Domain.Apps.Entities.TestHelpers
{
public static class TestConfig
{
public static IConfiguration Configuration { get; }
static TestConfig()
{
var basePath = Path.GetFullPath("../../../");
Configuration = new ConfigurationBuilder()
.SetBasePath(basePath)
.AddJsonFile("appsettings.json", true)
.AddJsonFile("appsettings.Development.json", true)
.AddEnvironmentVariables()
.Build();
}
}
}

53
backend/tests/Squidex.Domain.Apps.Entities.Tests/appSettings.json

@ -0,0 +1,53 @@
{
"mongoDb": {
// The connection string to your Mongo Server.
//
// Read More: https://docs.mongodb.com/manual/reference/connection-string/
"configuration": "mongodb://localhost",
// The name of the event store database.
"database": "Squidex_Testing"
},
"atlas": {
// The connection string to your Mongo Server.
//
// Read More: https://docs.mongodb.com/manual/reference/connection-string/
"configuration": "mongodb://localhost",
// The name of the event store database.
"database": "Squidex_Testing",
// The organization id.
"groupId": "",
// The name of the cluster.
"clusterName": "",
// Credentials to your account.
"publicKey": "",
"privateKey": "",
"fullTextEnabled": true
},
"elastic": {
// The configuration to your elastic search cluster.
//
// Read More: https://www.elastic.co/guide/en/elasticsearch/client/javascript-api/current/client-configuration.html
"configuration": "http://localhost:9200",
// The name of the test index.
"indexName": "test"
},
"azureText": {
// The URL to your azure search instance.
//
// Read More: https://docs.microsoft.com/en-us/azure/search/search-create-service-portal#get-a-key-and-url-endpoint
"serviceEndpoint": "https://<name>.search.windows.net",
// The api key. See link above.
"apiKey": "",
// The name of the index.
"indexName": "test"
}
}

22
backend/tests/Squidex.Infrastructure.Tests/Email/SmtpEmailSenderTests.cs

@ -5,8 +5,11 @@
// All rights reserved. Licensed under the MIT license. // All rights reserved. Licensed under the MIT license.
// ========================================================================== // ==========================================================================
using System.Threading;
using System.Threading.Tasks; using System.Threading.Tasks;
using Microsoft.Extensions.Configuration;
using Microsoft.Extensions.Options; using Microsoft.Extensions.Options;
using Squidex.Infrastructure.TestHelpers;
using Xunit; using Xunit;
namespace Squidex.Infrastructure.Email namespace Squidex.Infrastructure.Email
@ -17,18 +20,19 @@ namespace Squidex.Infrastructure.Email
[Fact] [Fact]
public async Task Should_handle_timeout_properly() public async Task Should_handle_timeout_properly()
{ {
var sut = new SmtpEmailSender(Options.Create(new SmtpOptions var options = TestConfig.Configuration.GetSection("email:smtp").Get<SmtpOptions>();
{
Sender = "sebastian@squidex.io", var recipient = TestConfig.Configuration["email:smtp:recipient"];
Server = "invalid",
Timeout = 1000
}));
var timer = Task.Delay(5000); var testSubject = TestConfig.Configuration["email:smtp:testSubject"];
var testBody = TestConfig.Configuration["email:smtp:testBody"];
var result = await Task.WhenAny(timer, sut.SendAsync("hello@squidex.io", "TEST", "TEST")); var sut = new SmtpEmailSender(Options.Create(options));
Assert.NotSame(timer, result); using (var cts = new CancellationTokenSource(5000))
{
await sut.SendAsync(recipient, testSubject, testBody, cts.Token);
}
} }
} }
} }

2
backend/tests/Squidex.Infrastructure.Tests/EventSourcing/GetEventStoreFixture.cs

@ -22,7 +22,7 @@ namespace Squidex.Infrastructure.EventSourcing
{ {
AppContext.SetSwitch("System.Net.Http.SocketsHttpHandler.Http2UnencryptedSupport", true); AppContext.SetSwitch("System.Net.Http.SocketsHttpHandler.Http2UnencryptedSupport", true);
settings = EventStoreClientSettings.Create("esdb://admin:changeit@127.0.0.1:2113?tls=false"); settings = EventStoreClientSettings.Create(TestConfig.Configuration["eventStore:configuration"]);
EventStore = new GetEventStore(settings, TestUtils.DefaultSerializer); EventStore = new GetEventStore(settings, TestUtils.DefaultSerializer);
EventStore.InitializeAsync(default).Wait(); EventStore.InitializeAsync(default).Wait();

6
backend/tests/Squidex.Infrastructure.Tests/EventSourcing/MongoEventStoreFixture.cs

@ -25,7 +25,7 @@ namespace Squidex.Infrastructure.EventSourcing
protected MongoEventStoreFixture(string connectionString) protected MongoEventStoreFixture(string connectionString)
{ {
mongoClient = new MongoClient(connectionString); mongoClient = new MongoClient(connectionString);
mongoDatabase = mongoClient.GetDatabase("Squidex_Testing"); mongoDatabase = mongoClient.GetDatabase(TestConfig.Configuration["mongodb:database"]);
BsonJsonConvention.Register(JsonSerializer.Create(TestUtils.DefaultSettings())); BsonJsonConvention.Register(JsonSerializer.Create(TestUtils.DefaultSettings()));
@ -49,7 +49,7 @@ namespace Squidex.Infrastructure.EventSourcing
public sealed class MongoEventStoreDirectFixture : MongoEventStoreFixture public sealed class MongoEventStoreDirectFixture : MongoEventStoreFixture
{ {
public MongoEventStoreDirectFixture() public MongoEventStoreDirectFixture()
: base("mongodb://localhost:27019") : base(TestConfig.Configuration["mongodb:configuration"])
{ {
} }
} }
@ -57,7 +57,7 @@ namespace Squidex.Infrastructure.EventSourcing
public sealed class MongoEventStoreReplicaSetFixture : MongoEventStoreFixture public sealed class MongoEventStoreReplicaSetFixture : MongoEventStoreFixture
{ {
public MongoEventStoreReplicaSetFixture() public MongoEventStoreReplicaSetFixture()
: base("mongodb://localhost:27017") : base(TestConfig.Configuration["mongodb:configurationReplica"])
{ {
} }
} }

29
backend/tests/Squidex.Infrastructure.Tests/MongoDb/MongoQueryTests.cs

@ -23,9 +23,6 @@ namespace Squidex.Infrastructure.MongoDb
{ {
public class MongoQueryTests public class MongoQueryTests
{ {
private readonly IBsonSerializerRegistry registry = BsonSerializer.SerializerRegistry;
private readonly IBsonSerializer<TestEntity> serializer = BsonSerializer.SerializerRegistry.GetSerializer<TestEntity>();
public class TestEntity public class TestEntity
{ {
public DomainId Id { get; set; } public DomainId Id { get; set; }
@ -322,18 +319,20 @@ namespace Squidex.Infrastructure.MongoDb
AssertQuery(new ClrQuery { Filter = filter }, expected, arg); AssertQuery(new ClrQuery { Filter = filter }, expected, arg);
} }
private void AssertQuery(ClrQuery query, string expected, object? arg = null) private static void AssertQuery(ClrQuery query, string expected, object? arg = null)
{ {
var rendered = var filter = query.BuildFilter<TestEntity>().Filter!;
query.BuildFilter<TestEntity>().Filter!
.Render(serializer, registry).ToString();
var expectation = Cleanup(expected, arg); var rendered =
filter.Render(
BsonSerializer.SerializerRegistry.GetSerializer<TestEntity>(),
BsonSerializer.SerializerRegistry)
.ToString();
Assert.Equal(expectation, rendered); Assert.Equal(Cleanup(expected, arg), rendered);
} }
private void AssertSorting(string expected, params SortNode[] sort) private static void AssertSorting(string expected, params SortNode[] sort)
{ {
var cursor = A.Fake<IFindFluent<TestEntity, TestEntity>>(); var cursor = A.Fake<IFindFluent<TestEntity, TestEntity>>();
@ -342,14 +341,16 @@ namespace Squidex.Infrastructure.MongoDb
A.CallTo(() => cursor.Sort(A<SortDefinition<TestEntity>>._)) A.CallTo(() => cursor.Sort(A<SortDefinition<TestEntity>>._))
.Invokes((SortDefinition<TestEntity> sortDefinition) => .Invokes((SortDefinition<TestEntity> sortDefinition) =>
{ {
rendered = sortDefinition.Render(serializer, registry).ToString(); rendered =
sortDefinition.Render(
BsonSerializer.SerializerRegistry.GetSerializer<TestEntity>(),
BsonSerializer.SerializerRegistry)
.ToString();
}); });
cursor.QuerySort(new ClrQuery { Sort = sort.ToList() }); cursor.QuerySort(new ClrQuery { Sort = sort.ToList() });
var expectation = Cleanup(expected); Assert.Equal(Cleanup(expected), rendered);
Assert.Equal(expectation, rendered);
} }
private static string Cleanup(string filter, object? arg = null) private static string Cleanup(string filter, object? arg = null)

29
backend/tests/Squidex.Infrastructure.Tests/TestHelpers/TestConfig.cs

@ -0,0 +1,29 @@
// ==========================================================================
// Squidex Headless CMS
// ==========================================================================
// Copyright (c) Squidex UG (haftungsbeschraenkt)
// All rights reserved. Licensed under the MIT license.
// ==========================================================================
using System.IO;
using Microsoft.Extensions.Configuration;
namespace Squidex.Infrastructure.TestHelpers
{
public static class TestConfig
{
public static IConfiguration Configuration { get; }
static TestConfig()
{
var basePath = Path.GetFullPath("../../../");
Configuration = new ConfigurationBuilder()
.SetBasePath(basePath)
.AddJsonFile("appsettings.json", true)
.AddJsonFile("appsettings.Development.json", true)
.AddEnvironmentVariables()
.Build();
}
}
}

15
backend/tests/Squidex.Infrastructure.Tests/appSettings.json

@ -0,0 +1,15 @@
{
"mongoDb": {
// The connection string to your Mongo Server.
//
// Read More: https://docs.mongodb.com/manual/reference/connection-string/
"configuration": "mongodb://localhost",
"configurationReplica": "mongodb://localhost",
// The name of the event store database.
"database": "Squidex_Testing"
},
"eventStore": {
"configuration": "esdb://admin:changeit@127.0.0.1:2113?tls=false"
}
}
Loading…
Cancel
Save