mirror of https://github.com/Squidex/squidex.git
Browse Source
* Update helm. * Fix DocumentDB search. * Fix tests * Fix compare. * Revert settings. * Fixespull/1267/head
Sebastian Stehle
9 months ago
committed by
GitHub
GitHub
23 changed files with 371 additions and 54 deletions
@ -0,0 +1,179 @@ |
|||
// ==========================================================================
|
|||
// Squidex Headless CMS
|
|||
// ==========================================================================
|
|||
// Copyright (c) Squidex UG (haftungsbeschraenkt)
|
|||
// All rights reserved. Licensed under the MIT license.
|
|||
// ==========================================================================
|
|||
|
|||
using MongoDB.Driver; |
|||
using MongoDB.Driver.GeoJsonObjectModel; |
|||
using Squidex.Domain.Apps.Core.Apps; |
|||
using Squidex.Infrastructure; |
|||
using Squidex.Infrastructure.ObjectPool; |
|||
using Squidex.Infrastructure.Translations; |
|||
|
|||
namespace Squidex.Domain.Apps.Entities.Contents.Text; |
|||
|
|||
public sealed class DocumentDbTextIndex(IMongoDatabase database, string shardKey) |
|||
: MongoTextIndexBase<string>(database, shardKey, new CommandFactory<string>(BuildTexts)) |
|||
{ |
|||
private record struct SearchOperation |
|||
{ |
|||
required public App App { get; init; } |
|||
|
|||
required public List<(DomainId Id, double Score)> Results { get; init; } |
|||
|
|||
required public string SearchTerms { get; init; } |
|||
|
|||
required public int Take { get; set; } |
|||
|
|||
required public SearchScope SearchScope { get; init; } |
|||
} |
|||
|
|||
protected override async Task SetupCollectionAsync(IMongoCollection<MongoTextIndexEntity<string>> collection, |
|||
CancellationToken ct) |
|||
{ |
|||
await collection.Indexes.CreateOneAsync( |
|||
new CreateIndexModel<MongoTextIndexEntity<string>>( |
|||
Index.Text(x => x.Texts)), |
|||
cancellationToken: ct); |
|||
|
|||
|
|||
await collection.Indexes.CreateManyAsync( |
|||
[ |
|||
new CreateIndexModel<MongoTextIndexEntity<string>>( |
|||
Index |
|||
.Ascending(x => x.AppId) |
|||
.Ascending(x => x.ContentId)), |
|||
|
|||
new CreateIndexModel<MongoTextIndexEntity<string>>( |
|||
Index |
|||
.Ascending(x => x.AppId) |
|||
.Ascending(x => x.SchemaId) |
|||
.Ascending(x => x.GeoField) |
|||
.Geo2DSphere(x => x.GeoObject)), |
|||
], ct); |
|||
} |
|||
|
|||
public override async Task<List<DomainId>?> SearchAsync(App app, GeoQuery query, SearchScope scope, |
|||
CancellationToken ct = default) |
|||
{ |
|||
Guard.NotNull(app); |
|||
Guard.NotNull(query); |
|||
|
|||
var point = new GeoJsonPoint<GeoJson2DCoordinates>(new GeoJson2DCoordinates(query.Longitude, query.Latitude)); |
|||
|
|||
// Use the filter in the correct order to leverage the index in the best way.
|
|||
var findFilter = |
|||
Filter.And( |
|||
Filter.Eq(x => x.AppId, app.Id), |
|||
Filter.Eq(x => x.SchemaId, query.SchemaId), |
|||
Filter.Eq(x => x.GeoField, query.Field), |
|||
Filter.NearSphere(x => x.GeoObject, point, query.Radius), |
|||
FilterByScope(scope)); |
|||
|
|||
var byGeo = |
|||
await GetCollection(scope).Find(findFilter).Limit(query.Take) |
|||
.Project<MongoTextResult>(Projection.Include(x => x.ContentId)) |
|||
.ToListAsync(ct); |
|||
|
|||
return byGeo.Select(x => x.ContentId).ToList(); |
|||
} |
|||
|
|||
public override async Task<List<DomainId>?> SearchAsync(App app, TextQuery query, SearchScope scope, |
|||
CancellationToken ct = default) |
|||
{ |
|||
Guard.NotNull(app); |
|||
Guard.NotNull(query); |
|||
|
|||
if (string.IsNullOrWhiteSpace(query.Text)) |
|||
{ |
|||
return null; |
|||
} |
|||
|
|||
// Use a custom tokenizer to leverage stop words from multiple languages.
|
|||
var search = new SearchOperation |
|||
{ |
|||
App = app, |
|||
SearchTerms = Tokenizer.Query(query.Text), |
|||
SearchScope = scope, |
|||
Results = [], |
|||
Take = query.Take, |
|||
}; |
|||
|
|||
if (query.RequiredSchemaIds?.Count > 0) |
|||
{ |
|||
await SearchBySchemaAsync(search, query.RequiredSchemaIds, 1, ct); |
|||
} |
|||
else if (query.PreferredSchemaId == null) |
|||
{ |
|||
await SearchByAppAsync(search, 1, ct); |
|||
} |
|||
else |
|||
{ |
|||
// We cannot write queries that prefer results from the same schema, therefore make two queries.
|
|||
search.Take /= 2; |
|||
|
|||
// Increasing the scoring of the results from the schema by 10 percent.
|
|||
await SearchBySchemaAsync(search, Enumerable.Repeat(query.PreferredSchemaId.Value, 1), 1.1, ct); |
|||
await SearchByAppAsync(search, 1, ct); |
|||
} |
|||
|
|||
return search.Results.OrderByDescending(x => x.Score).Select(x => x.Id).Distinct().ToList(); |
|||
} |
|||
|
|||
private Task SearchBySchemaAsync(SearchOperation search, IEnumerable<DomainId> schemaIds, double factor, |
|||
CancellationToken ct) |
|||
{ |
|||
var filter = |
|||
Filter.And( |
|||
Filter.Eq(x => x.AppId, search.App.Id), |
|||
Filter.Text(search.SearchTerms), |
|||
Filter.In(x => x.SchemaId, schemaIds), |
|||
FilterByScope(search.SearchScope)); |
|||
|
|||
return SearchAsync(search, filter, factor, ct); |
|||
} |
|||
|
|||
private Task SearchByAppAsync(SearchOperation search, double factor, |
|||
CancellationToken ct) |
|||
{ |
|||
var filter = |
|||
Filter.And( |
|||
Filter.Eq(x => x.AppId, search.App.Id), |
|||
Filter.Text(search.SearchTerms), |
|||
FilterByScope(search.SearchScope)); |
|||
|
|||
return SearchAsync(search, filter, factor, ct); |
|||
} |
|||
|
|||
private async Task SearchAsync(SearchOperation search, FilterDefinition<MongoTextIndexEntity<string>> filter, double factor, |
|||
CancellationToken ct) |
|||
{ |
|||
var byText = |
|||
await GetCollection(search.SearchScope).Find(filter).Limit(search.Take) |
|||
.Project<MongoTextResult>(Projection.Include(x => x.ContentId).MetaTextScore("score")).Sort(Sort.MetaTextScore("score")) |
|||
.ToListAsync(ct); |
|||
|
|||
search.Results.AddRange(byText.Select(x => (x.ContentId, x.Score * factor))); |
|||
} |
|||
|
|||
private static string BuildTexts(Dictionary<string, string> source) |
|||
{ |
|||
var sb = DefaultPools.StringBuilder.Get(); |
|||
try |
|||
{ |
|||
foreach (var (key, value) in source) |
|||
{ |
|||
sb.Append(' '); |
|||
sb.Append(Tokenizer.Terms(value, key)); |
|||
} |
|||
|
|||
return sb.ToString(); |
|||
} |
|||
finally |
|||
{ |
|||
DefaultPools.StringBuilder.Return(sb); |
|||
} |
|||
} |
|||
} |
|||
@ -0,0 +1,37 @@ |
|||
// ==========================================================================
|
|||
// Squidex Headless CMS
|
|||
// ==========================================================================
|
|||
// Copyright (c) Squidex UG (haftungsbeschraenkt)
|
|||
// All rights reserved. Licensed under the MIT license.
|
|||
// ==========================================================================
|
|||
|
|||
using Squidex.Domain.Apps.Entities.Contents.Text; |
|||
using Squidex.MongoDb.TestHelpers; |
|||
|
|||
namespace Squidex.MongoDb.Domain.Contents.Text; |
|||
|
|||
[Trait("Category", "Dependencies")] |
|||
[Collection(DocumentDbCollection.Name)] |
|||
public sealed class DocumentDbTextIndexTests(DocumentDbFixture fixture) : TextIndexerTests |
|||
{ |
|||
public override bool SupportsQuerySyntax => false; |
|||
|
|||
public override bool SupportsGeo => true; |
|||
|
|||
public override async Task<ITextIndex> CreateSutAsync() |
|||
{ |
|||
var sut = new DocumentDbTextIndex(fixture.Database, string.Empty); |
|||
|
|||
await sut.InitializeAsync(default); |
|||
return sut; |
|||
} |
|||
|
|||
[Fact] |
|||
public async Task Should_retrieve_all_stopwords_for_german_query() |
|||
{ |
|||
await CreateTextAsync(Ids1[0], "de", "and und"); |
|||
await CreateTextAsync(Ids2[0], "en", "and und"); |
|||
|
|||
await SearchText(expected: Ids2, text: "en:und"); |
|||
} |
|||
} |
|||
@ -0,0 +1,50 @@ |
|||
// ==========================================================================
|
|||
// Squidex Headless CMS
|
|||
// ==========================================================================
|
|||
// Copyright (c) Squidex UG (haftungsbeschraenkt)
|
|||
// All rights reserved. Licensed under the MIT license.
|
|||
// ==========================================================================
|
|||
|
|||
using System.Security.Cryptography.X509Certificates; |
|||
using Microsoft.Extensions.Configuration; |
|||
using MongoDB.Driver; |
|||
using Squidex.Domain.Apps.Entities.TestHelpers; |
|||
|
|||
#pragma warning disable MA0048 // File name must match type name
|
|||
|
|||
namespace Squidex.MongoDb.TestHelpers; |
|||
|
|||
[CollectionDefinition(Name)] |
|||
public sealed class DocumentDbCollection : ICollectionFixture<DocumentDbFixture> |
|||
{ |
|||
public const string Name = "DocumentDb"; |
|||
} |
|||
|
|||
public class DocumentDbFixture |
|||
{ |
|||
public IMongoClient Client { get; private set; } |
|||
|
|||
public IMongoDatabase Database => Client.GetDatabase($"Test_{Guid.NewGuid()}"); |
|||
|
|||
public DocumentDbFixture() |
|||
{ |
|||
MongoTestUtils.SetupBson(); |
|||
|
|||
var settings = MongoClientSettings.FromConnectionString( |
|||
TestConfig.Configuration.GetValue<string>("documentDb:configuration") |
|||
); |
|||
|
|||
var cert = new X509Certificate2(TestConfig.Configuration.GetValue<string>("documentDb:keyFile")!); |
|||
|
|||
settings.RetryWrites = false; |
|||
settings.RetryReads = false; |
|||
settings.SslSettings = new SslSettings |
|||
{ |
|||
ClientCertificates = [cert], |
|||
CheckCertificateRevocation = false, |
|||
ServerCertificateValidationCallback = (sender, certificate, chain, errors) => true, |
|||
}; |
|||
|
|||
Client = new MongoClient(settings); |
|||
} |
|||
} |
|||
Loading…
Reference in new issue