Browse Source

Fix fulltext (#1197)

pull/1200/head
Sebastian Stehle 1 year ago
committed by GitHub
parent
commit
497af8b56a
No known key found for this signature in database GPG Key ID: B5690EEEBB952194
  1. 2
      backend/extensions/Squidex.Extensions/Text/Azure/AzureTextIndex.cs
  2. 64
      backend/extensions/Squidex.Extensions/Text/Azure/CommandFactory.cs
  3. 85
      backend/extensions/Squidex.Extensions/Text/ElasticSearch/CommandFactory.cs
  4. 12
      backend/src/Squidex.Data.EntityFramework/Squidex.Data.EntityFramework.csproj
  5. 10
      backend/src/Squidex.Data.MongoDb/Squidex.Data.MongoDb.csproj
  6. 4
      backend/src/Squidex.Domain.Apps.Core.Operations/Squidex.Domain.Apps.Core.Operations.csproj
  7. 9
      backend/src/Squidex.Domain.Apps.Entities/Contents/Text/TextIndexingProcess.cs
  8. 14
      backend/src/Squidex.Infrastructure/Squidex.Infrastructure.csproj
  9. 22
      backend/src/Squidex/Squidex.csproj

2
backend/extensions/Squidex.Extensions/Text/Azure/AzureTextIndex.cs

@ -67,7 +67,7 @@ public sealed class AzureTextIndex : IInitializable, ITextIndex
return; return;
} }
await searchClient.IndexDocumentsAsync(batch, cancellationToken: ct); var x = await searchClient.IndexDocumentsAsync(batch, cancellationToken: ct);
} }
public async Task<List<DomainId>?> SearchAsync(App app, GeoQuery query, SearchScope scope, public async Task<List<DomainId>?> SearchAsync(App app, GeoQuery query, SearchScope scope,

64
backend/extensions/Squidex.Extensions/Text/Azure/CommandFactory.cs

@ -32,58 +32,72 @@ public static class CommandFactory
private static void UpsertTextEntry(UpsertIndexEntry upsert, IList<IndexDocumentsAction<SearchDocument>> batch) private static void UpsertTextEntry(UpsertIndexEntry upsert, IList<IndexDocumentsAction<SearchDocument>> batch)
{ {
var geoField = string.Empty;
var geoObject = (object?)null;
if (upsert.GeoObjects != null) if (upsert.GeoObjects != null)
{ {
foreach (var (key, value) in upsert.GeoObjects) foreach (var (key, value) in upsert.GeoObjects)
{ {
if (value is Point point) if (value is not Point point)
{ {
geoField = key; continue;
geoObject = new
{
type = "Point",
coordinates = new[]
{
point.Coordinate.X,
point.Coordinate.Y,
},
};
break;
} }
var geoField = key;
var geoObject = new
{
type = "Point",
coordinates = new[]
{
point.Coordinate.X,
point.Coordinate.Y,
},
};
var geoDocument = new SearchDocument
{
["docId"] = upsert.ToDocId(),
["appId"] = upsert.UniqueContentId.AppId.ToString(),
["appName"] = string.Empty,
["contentId"] = upsert.UniqueContentId.ContentId.ToString(),
["schemaId"] = upsert.SchemaId.Id.ToString(),
["schemaName"] = upsert.SchemaId.Name,
["serveAll"] = upsert.ServeAll,
["servePublished"] = upsert.ServePublished,
["geoField"] = geoField,
["geoObject"] = geoObject,
};
batch.Add(IndexDocumentsAction.MergeOrUpload(geoDocument));
} }
} }
if (upsert.Texts != null && geoObject != null) if (upsert.Texts is { Count: > 0 })
{ {
var document = new SearchDocument var document = new SearchDocument
{ {
["docId"] = upsert.ToDocId(), ["docId"] = upsert.ToDocId(),
["appId"] = upsert.UniqueContentId.AppId.ToString(), ["appId"] = upsert.UniqueContentId.AppId.ToString(),
["appName"] = string.Empty, ["appName"] = string.Empty,
["contentId"] = upsert.UniqueContentId.ToString(), ["contentId"] = upsert.UniqueContentId.ContentId.ToString(),
["schemaId"] = upsert.SchemaId.Id.ToString(), ["schemaId"] = upsert.SchemaId.Id.ToString(),
["schemaName"] = upsert.SchemaId.Name, ["schemaName"] = upsert.SchemaId.Name,
["serveAll"] = upsert.ServeAll, ["serveAll"] = upsert.ServeAll,
["servePublished"] = upsert.ServePublished, ["servePublished"] = upsert.ServePublished,
["geoField"] = geoField,
["geoObject"] = geoObject,
}; };
foreach (var (key, value) in upsert.Texts) foreach (var (key, value) in upsert.Texts)
{ {
var text = value; var textMerged = value;
var textLanguage = AzureIndexDefinition.GetFieldName(key);
var languageCode = AzureIndexDefinition.GetFieldName(key);
if (document.TryGetValue(languageCode, out var existing)) if (document.TryGetValue(textLanguage, out var existing))
{ {
text = $"{existing} {value}"; textMerged = $"{existing} {value}";
} }
document[languageCode] = text; if (!string.IsNullOrWhiteSpace(textMerged))
{
document[textLanguage] = textMerged;
}
} }
batch.Add(IndexDocumentsAction.MergeOrUpload(document)); batch.Add(IndexDocumentsAction.MergeOrUpload(document));

85
backend/extensions/Squidex.Extensions/Text/ElasticSearch/CommandFactory.cs

@ -30,57 +30,78 @@ public static class CommandFactory
private static void UpsertEntry(UpsertIndexEntry upsert, List<object> args, string indexName) private static void UpsertEntry(UpsertIndexEntry upsert, List<object> args, string indexName)
{ {
var geoField = string.Empty; var hasAddedDeletion = false;
var geoObject = (object?)null;
if (upsert.GeoObjects != null) void AddArgs(object arg)
{ {
foreach (var (key, value) in upsert.GeoObjects) if (!hasAddedDeletion)
{ {
if (value is Point point) args.Add(new
{ {
geoField = key; index = new
geoObject = new
{ {
lat = point.Coordinate.X, _id = upsert.ToDocId(),
lon = point.Coordinate.Y, _index = indexName,
}; },
break; });
}
} }
args.Add(arg);
hasAddedDeletion = true;
} }
if (upsert.Texts != null || geoObject != null) if (upsert.GeoObjects != null)
{ {
args.Add(new foreach (var (key, value) in upsert.GeoObjects)
{ {
index = new if (value is not Point point)
{ {
_id = upsert.ToDocId(), continue;
_index = indexName, }
},
});
var texts = new Dictionary<string, string>(); var geoField = key;
var geoObject = new
{
lat = point.Coordinate.X,
lon = point.Coordinate.Y,
};
if (upsert.Texts != null) AddArgs(new
{
foreach (var (key, value) in upsert.Texts)
{ {
var text = value; appId = upsert.UniqueContentId.AppId.ToString(),
appName = string.Empty,
contentId = upsert.UniqueContentId.ContentId.ToString(),
schemaId = upsert.SchemaId.Id.ToString(),
schemaName = upsert.SchemaId.Name,
serveAll = upsert.ServeAll,
servePublished = upsert.ServePublished,
geoField,
geoObject,
});
}
}
var languageCode = ElasticSearchIndexDefinition.GetFieldName(key); if (upsert.Texts is { Count: > 0 })
{
var texts = new Dictionary<string, string>();
if (texts.TryGetValue(languageCode, out var existing)) foreach (var (key, value) in upsert.Texts)
{ {
text = $"{existing} {value}"; var textMerged = value;
} var textLanguage = ElasticSearchIndexDefinition.GetFieldName(key);
texts[languageCode] = text; if (texts.TryGetValue(textLanguage, out var existing))
{
textMerged = $"{existing} {value}";
}
if (!string.IsNullOrWhiteSpace(textMerged))
{
texts[textLanguage] = textMerged;
} }
} }
args.Add(new AddArgs(new
{ {
appId = upsert.UniqueContentId.AppId.ToString(), appId = upsert.UniqueContentId.AppId.ToString(),
appName = string.Empty, appName = string.Empty,
@ -90,8 +111,6 @@ public static class CommandFactory
serveAll = upsert.ServeAll, serveAll = upsert.ServeAll,
servePublished = upsert.ServePublished, servePublished = upsert.ServePublished,
texts, texts,
geoField,
geoObject,
}); });
} }
} }

12
backend/src/Squidex.Data.EntityFramework/Squidex.Data.EntityFramework.csproj

@ -35,13 +35,13 @@
<PackageReference Include="Pomelo.EntityFrameworkCore.MySql.Json.Microsoft" Version="8.0.2" /> <PackageReference Include="Pomelo.EntityFrameworkCore.MySql.Json.Microsoft" Version="8.0.2" />
<PackageReference Include="Pomelo.EntityFrameworkCore.MySql.NetTopologySuite" Version="8.0.2" /> <PackageReference Include="Pomelo.EntityFrameworkCore.MySql.NetTopologySuite" Version="8.0.2" />
<PackageReference Include="RefactoringEssentials" Version="5.6.0" PrivateAssets="all" /> <PackageReference Include="RefactoringEssentials" Version="5.6.0" PrivateAssets="all" />
<PackageReference Include="Squidex.AI.EntityFramework" Version="6.33.0" /> <PackageReference Include="Squidex.AI.EntityFramework" Version="6.35.0" />
<PackageReference Include="Squidex.Assets.EntityFramework" Version="6.33.0" /> <PackageReference Include="Squidex.Assets.EntityFramework" Version="6.35.0" />
<PackageReference Include="Squidex.Assets.TusAdapter" Version="6.33.0" /> <PackageReference Include="Squidex.Assets.TusAdapter" Version="6.35.0" />
<PackageReference Include="Squidex.EFCore.BulkExtensions" Version="8.1.6" /> <PackageReference Include="Squidex.EFCore.BulkExtensions" Version="8.1.6" />
<PackageReference Include="Squidex.Events.EntityFramework" Version="6.33.0" /> <PackageReference Include="Squidex.Events.EntityFramework" Version="6.35.0" />
<PackageReference Include="Squidex.Hosting" Version="6.33.0" /> <PackageReference Include="Squidex.Hosting" Version="6.35.0" />
<PackageReference Include="Squidex.Messaging.EntityFramework" Version="6.33.0" /> <PackageReference Include="Squidex.Messaging.EntityFramework" Version="6.35.0" />
<PackageReference Include="Squidex.OpenIdDict.EntityFramework" Version="5.8.4" /> <PackageReference Include="Squidex.OpenIdDict.EntityFramework" Version="5.8.4" />
<PackageReference Include="StyleCop.Analyzers" Version="1.1.118" PrivateAssets="all" /> <PackageReference Include="StyleCop.Analyzers" Version="1.1.118" PrivateAssets="all" />
<PackageReference Include="System.ValueTuple" Version="4.5.0" /> <PackageReference Include="System.ValueTuple" Version="4.5.0" />

10
backend/src/Squidex.Data.MongoDb/Squidex.Data.MongoDb.csproj

@ -24,11 +24,11 @@
<PackageReference Include="MongoDB.Driver.Core.Extensions.DiagnosticSources" Version="1.5.0" /> <PackageReference Include="MongoDB.Driver.Core.Extensions.DiagnosticSources" Version="1.5.0" />
<PackageReference Include="MongoDB.Driver.GridFS" Version="2.30.0" /> <PackageReference Include="MongoDB.Driver.GridFS" Version="2.30.0" />
<PackageReference Include="RefactoringEssentials" Version="5.6.0" PrivateAssets="all" /> <PackageReference Include="RefactoringEssentials" Version="5.6.0" PrivateAssets="all" />
<PackageReference Include="Squidex.AI.Mongo" Version="6.33.0" /> <PackageReference Include="Squidex.AI.Mongo" Version="6.35.0" />
<PackageReference Include="Squidex.Assets.Mongo" Version="6.33.0" /> <PackageReference Include="Squidex.Assets.Mongo" Version="6.35.0" />
<PackageReference Include="Squidex.Events.Mongo" Version="6.33.0" /> <PackageReference Include="Squidex.Events.Mongo" Version="6.35.0" />
<PackageReference Include="Squidex.Hosting" Version="6.33.0" /> <PackageReference Include="Squidex.Hosting" Version="6.35.0" />
<PackageReference Include="Squidex.Messaging.Mongo" Version="6.33.0" /> <PackageReference Include="Squidex.Messaging.Mongo" Version="6.35.0" />
<PackageReference Include="Squidex.OpenIddict.MongoDb" Version="5.8.4" /> <PackageReference Include="Squidex.OpenIddict.MongoDb" Version="5.8.4" />
<PackageReference Include="StyleCop.Analyzers" Version="1.1.118" PrivateAssets="all" /> <PackageReference Include="StyleCop.Analyzers" Version="1.1.118" PrivateAssets="all" />
<PackageReference Include="System.ValueTuple" Version="4.5.0" /> <PackageReference Include="System.ValueTuple" Version="4.5.0" />

4
backend/src/Squidex.Domain.Apps.Core.Operations/Squidex.Domain.Apps.Core.Operations.csproj

@ -28,8 +28,8 @@
<PackageReference Include="Microsoft.Extensions.Http" Version="8.0.1" /> <PackageReference Include="Microsoft.Extensions.Http" Version="8.0.1" />
<PackageReference Include="NJsonSchema" Version="11.0.2" /> <PackageReference Include="NJsonSchema" Version="11.0.2" />
<PackageReference Include="RefactoringEssentials" Version="5.6.0" PrivateAssets="all" /> <PackageReference Include="RefactoringEssentials" Version="5.6.0" PrivateAssets="all" />
<PackageReference Include="Squidex.AI" Version="6.33.0" /> <PackageReference Include="Squidex.AI" Version="6.35.0" />
<PackageReference Include="Squidex.Messaging.Subscriptions" Version="6.33.0" /> <PackageReference Include="Squidex.Messaging.Subscriptions" Version="6.35.0" />
<PackageReference Include="StyleCop.Analyzers" Version="1.1.118" PrivateAssets="all" /> <PackageReference Include="StyleCop.Analyzers" Version="1.1.118" PrivateAssets="all" />
<PackageReference Include="System.Collections.Immutable" Version="8.0.0" /> <PackageReference Include="System.Collections.Immutable" Version="8.0.0" />
<PackageReference Include="System.Linq.Async" Version="6.0.1" /> <PackageReference Include="System.Linq.Async" Version="6.0.1" />

9
backend/src/Squidex.Domain.Apps.Entities/Contents/Text/TextIndexingProcess.cs

@ -331,16 +331,15 @@ public sealed class TextIndexingProcess(
public async Task On(IEnumerable<Envelope<IEvent>> events) public async Task On(IEnumerable<Envelope<IEvent>> events)
{ {
var states = await QueryStatesAsync(events); var textStates = await QueryStatesAsync(events);
var textBatch = new Updates(textStates, serializer);
var updates = new Updates(states, serializer);
foreach (var @event in events) foreach (var @event in events)
{ {
updates.On(@event); textBatch.On(@event);
} }
await updates.WriteAsync(textIndex, textIndexerState); await textBatch.WriteAsync(textIndex, textIndexerState);
} }
private Task<Dictionary<UniqueContentId, TextContentState>> QueryStatesAsync(IEnumerable<Envelope<IEvent>> events) private Task<Dictionary<UniqueContentId, TextContentState>> QueryStatesAsync(IEnumerable<Envelope<IEvent>> events)

14
backend/src/Squidex.Infrastructure/Squidex.Infrastructure.csproj

@ -24,13 +24,13 @@
<PackageReference Include="NodaTime" Version="3.2.0" /> <PackageReference Include="NodaTime" Version="3.2.0" />
<PackageReference Include="OpenTelemetry.Api" Version="1.9.0" /> <PackageReference Include="OpenTelemetry.Api" Version="1.9.0" />
<PackageReference Include="RefactoringEssentials" Version="5.6.0" PrivateAssets="all" /> <PackageReference Include="RefactoringEssentials" Version="5.6.0" PrivateAssets="all" />
<PackageReference Include="Squidex.Assets" Version="6.33.0" /> <PackageReference Include="Squidex.Assets" Version="6.35.0" />
<PackageReference Include="Squidex.Caching" Version="6.33.0" /> <PackageReference Include="Squidex.Caching" Version="6.35.0" />
<PackageReference Include="Squidex.Events" Version="6.33.0" /> <PackageReference Include="Squidex.Events" Version="6.35.0" />
<PackageReference Include="Squidex.Hosting.Abstractions" Version="6.33.0" /> <PackageReference Include="Squidex.Hosting.Abstractions" Version="6.35.0" />
<PackageReference Include="Squidex.Log" Version="6.33.0" /> <PackageReference Include="Squidex.Log" Version="6.35.0" />
<PackageReference Include="Squidex.Messaging" Version="6.33.0" /> <PackageReference Include="Squidex.Messaging" Version="6.35.0" />
<PackageReference Include="Squidex.Text" Version="6.33.0" /> <PackageReference Include="Squidex.Text" Version="6.35.0" />
<PackageReference Include="StyleCop.Analyzers" Version="1.1.118" PrivateAssets="all" /> <PackageReference Include="StyleCop.Analyzers" Version="1.1.118" PrivateAssets="all" />
<PackageReference Include="System.Collections.Immutable" Version="8.0.0" /> <PackageReference Include="System.Collections.Immutable" Version="8.0.0" />
<PackageReference Include="System.ComponentModel.Annotations" Version="5.0.0" /> <PackageReference Include="System.ComponentModel.Annotations" Version="5.0.0" />

22
backend/src/Squidex/Squidex.csproj

@ -62,18 +62,18 @@
<PackageReference Include="OpenTelemetry.Instrumentation.Runtime" Version="1.9.0" /> <PackageReference Include="OpenTelemetry.Instrumentation.Runtime" Version="1.9.0" />
<PackageReference Include="RefactoringEssentials" Version="5.6.0" PrivateAssets="all" /> <PackageReference Include="RefactoringEssentials" Version="5.6.0" PrivateAssets="all" />
<PackageReference Include="ReportGenerator" Version="5.4.1" PrivateAssets="all" /> <PackageReference Include="ReportGenerator" Version="5.4.1" PrivateAssets="all" />
<PackageReference Include="Squidex.Assets.Azure" Version="6.33.0" /> <PackageReference Include="Squidex.Assets.Azure" Version="6.35.0" />
<PackageReference Include="Squidex.Assets.GoogleCloud" Version="6.33.0" /> <PackageReference Include="Squidex.Assets.GoogleCloud" Version="6.35.0" />
<PackageReference Include="Squidex.Assets.FTP" Version="6.33.0" /> <PackageReference Include="Squidex.Assets.FTP" Version="6.35.0" />
<PackageReference Include="Squidex.Assets.ImageMagick" Version="6.33.0" /> <PackageReference Include="Squidex.Assets.ImageMagick" Version="6.35.0" />
<PackageReference Include="Squidex.Assets.ImageSharp" Version="6.33.0" /> <PackageReference Include="Squidex.Assets.ImageSharp" Version="6.35.0" />
<PackageReference Include="Squidex.Assets.S3" Version="6.33.0" /> <PackageReference Include="Squidex.Assets.S3" Version="6.35.0" />
<PackageReference Include="Squidex.Assets.TusAdapter" Version="6.33.0" /> <PackageReference Include="Squidex.Assets.TusAdapter" Version="6.35.0" />
<PackageReference Include="Squidex.ClientLibrary" Version="20.1.0" /> <PackageReference Include="Squidex.ClientLibrary" Version="20.1.0" />
<PackageReference Include="Squidex.Events.GetEventStore" Version="6.33.0" /> <PackageReference Include="Squidex.Events.GetEventStore" Version="6.35.0" />
<PackageReference Include="Squidex.Hosting" Version="6.33.0" /> <PackageReference Include="Squidex.Hosting" Version="6.35.0" />
<PackageReference Include="Squidex.Messaging.All" Version="6.33.0" /> <PackageReference Include="Squidex.Messaging.All" Version="6.35.0" />
<PackageReference Include="Squidex.Messaging.Subscriptions" Version="6.33.0" /> <PackageReference Include="Squidex.Messaging.Subscriptions" Version="6.35.0" />
<PackageReference Include="StyleCop.Analyzers" Version="1.1.118" PrivateAssets="all" /> <PackageReference Include="StyleCop.Analyzers" Version="1.1.118" PrivateAssets="all" />
<PackageReference Include="YDotNet" Version="0.4.3" /> <PackageReference Include="YDotNet" Version="0.4.3" />
<PackageReference Include="YDotNet.Native" Version="0.4.3" /> <PackageReference Include="YDotNet.Native" Version="0.4.3" />

Loading…
Cancel
Save