Browse Source

Fix fulltext (#1197)

pull/1200/head
Sebastian Stehle 1 year ago
committed by GitHub
parent
commit
497af8b56a
No known key found for this signature in database GPG Key ID: B5690EEEBB952194
  1. 2
      backend/extensions/Squidex.Extensions/Text/Azure/AzureTextIndex.cs
  2. 64
      backend/extensions/Squidex.Extensions/Text/Azure/CommandFactory.cs
  3. 85
      backend/extensions/Squidex.Extensions/Text/ElasticSearch/CommandFactory.cs
  4. 12
      backend/src/Squidex.Data.EntityFramework/Squidex.Data.EntityFramework.csproj
  5. 10
      backend/src/Squidex.Data.MongoDb/Squidex.Data.MongoDb.csproj
  6. 4
      backend/src/Squidex.Domain.Apps.Core.Operations/Squidex.Domain.Apps.Core.Operations.csproj
  7. 9
      backend/src/Squidex.Domain.Apps.Entities/Contents/Text/TextIndexingProcess.cs
  8. 14
      backend/src/Squidex.Infrastructure/Squidex.Infrastructure.csproj
  9. 22
      backend/src/Squidex/Squidex.csproj

2
backend/extensions/Squidex.Extensions/Text/Azure/AzureTextIndex.cs

@ -67,7 +67,7 @@ public sealed class AzureTextIndex : IInitializable, ITextIndex
return;
}
await searchClient.IndexDocumentsAsync(batch, cancellationToken: ct);
var x = await searchClient.IndexDocumentsAsync(batch, cancellationToken: ct);
}
public async Task<List<DomainId>?> SearchAsync(App app, GeoQuery query, SearchScope scope,

64
backend/extensions/Squidex.Extensions/Text/Azure/CommandFactory.cs

@ -32,58 +32,72 @@ public static class CommandFactory
private static void UpsertTextEntry(UpsertIndexEntry upsert, IList<IndexDocumentsAction<SearchDocument>> batch)
{
var geoField = string.Empty;
var geoObject = (object?)null;
if (upsert.GeoObjects != null)
{
foreach (var (key, value) in upsert.GeoObjects)
{
if (value is Point point)
if (value is not Point point)
{
geoField = key;
geoObject = new
{
type = "Point",
coordinates = new[]
{
point.Coordinate.X,
point.Coordinate.Y,
},
};
break;
continue;
}
var geoField = key;
var geoObject = new
{
type = "Point",
coordinates = new[]
{
point.Coordinate.X,
point.Coordinate.Y,
},
};
var geoDocument = new SearchDocument
{
["docId"] = upsert.ToDocId(),
["appId"] = upsert.UniqueContentId.AppId.ToString(),
["appName"] = string.Empty,
["contentId"] = upsert.UniqueContentId.ContentId.ToString(),
["schemaId"] = upsert.SchemaId.Id.ToString(),
["schemaName"] = upsert.SchemaId.Name,
["serveAll"] = upsert.ServeAll,
["servePublished"] = upsert.ServePublished,
["geoField"] = geoField,
["geoObject"] = geoObject,
};
batch.Add(IndexDocumentsAction.MergeOrUpload(geoDocument));
}
}
if (upsert.Texts != null && geoObject != null)
if (upsert.Texts is { Count: > 0 })
{
var document = new SearchDocument
{
["docId"] = upsert.ToDocId(),
["appId"] = upsert.UniqueContentId.AppId.ToString(),
["appName"] = string.Empty,
["contentId"] = upsert.UniqueContentId.ToString(),
["contentId"] = upsert.UniqueContentId.ContentId.ToString(),
["schemaId"] = upsert.SchemaId.Id.ToString(),
["schemaName"] = upsert.SchemaId.Name,
["serveAll"] = upsert.ServeAll,
["servePublished"] = upsert.ServePublished,
["geoField"] = geoField,
["geoObject"] = geoObject,
};
foreach (var (key, value) in upsert.Texts)
{
var text = value;
var languageCode = AzureIndexDefinition.GetFieldName(key);
var textMerged = value;
var textLanguage = AzureIndexDefinition.GetFieldName(key);
if (document.TryGetValue(languageCode, out var existing))
if (document.TryGetValue(textLanguage, out var existing))
{
text = $"{existing} {value}";
textMerged = $"{existing} {value}";
}
document[languageCode] = text;
if (!string.IsNullOrWhiteSpace(textMerged))
{
document[textLanguage] = textMerged;
}
}
batch.Add(IndexDocumentsAction.MergeOrUpload(document));

85
backend/extensions/Squidex.Extensions/Text/ElasticSearch/CommandFactory.cs

@ -30,57 +30,78 @@ public static class CommandFactory
private static void UpsertEntry(UpsertIndexEntry upsert, List<object> args, string indexName)
{
var geoField = string.Empty;
var geoObject = (object?)null;
var hasAddedDeletion = false;
if (upsert.GeoObjects != null)
void AddArgs(object arg)
{
foreach (var (key, value) in upsert.GeoObjects)
if (!hasAddedDeletion)
{
if (value is Point point)
args.Add(new
{
geoField = key;
geoObject = new
index = new
{
lat = point.Coordinate.X,
lon = point.Coordinate.Y,
};
break;
}
_id = upsert.ToDocId(),
_index = indexName,
},
});
}
args.Add(arg);
hasAddedDeletion = true;
}
if (upsert.Texts != null || geoObject != null)
if (upsert.GeoObjects != null)
{
args.Add(new
foreach (var (key, value) in upsert.GeoObjects)
{
index = new
if (value is not Point point)
{
_id = upsert.ToDocId(),
_index = indexName,
},
});
continue;
}
var texts = new Dictionary<string, string>();
var geoField = key;
var geoObject = new
{
lat = point.Coordinate.X,
lon = point.Coordinate.Y,
};
if (upsert.Texts != null)
{
foreach (var (key, value) in upsert.Texts)
AddArgs(new
{
var text = value;
appId = upsert.UniqueContentId.AppId.ToString(),
appName = string.Empty,
contentId = upsert.UniqueContentId.ContentId.ToString(),
schemaId = upsert.SchemaId.Id.ToString(),
schemaName = upsert.SchemaId.Name,
serveAll = upsert.ServeAll,
servePublished = upsert.ServePublished,
geoField,
geoObject,
});
}
}
var languageCode = ElasticSearchIndexDefinition.GetFieldName(key);
if (upsert.Texts is { Count: > 0 })
{
var texts = new Dictionary<string, string>();
if (texts.TryGetValue(languageCode, out var existing))
{
text = $"{existing} {value}";
}
foreach (var (key, value) in upsert.Texts)
{
var textMerged = value;
var textLanguage = ElasticSearchIndexDefinition.GetFieldName(key);
texts[languageCode] = text;
if (texts.TryGetValue(textLanguage, out var existing))
{
textMerged = $"{existing} {value}";
}
if (!string.IsNullOrWhiteSpace(textMerged))
{
texts[textLanguage] = textMerged;
}
}
args.Add(new
AddArgs(new
{
appId = upsert.UniqueContentId.AppId.ToString(),
appName = string.Empty,
@ -90,8 +111,6 @@ public static class CommandFactory
serveAll = upsert.ServeAll,
servePublished = upsert.ServePublished,
texts,
geoField,
geoObject,
});
}
}

12
backend/src/Squidex.Data.EntityFramework/Squidex.Data.EntityFramework.csproj

@ -35,13 +35,13 @@
<PackageReference Include="Pomelo.EntityFrameworkCore.MySql.Json.Microsoft" Version="8.0.2" />
<PackageReference Include="Pomelo.EntityFrameworkCore.MySql.NetTopologySuite" Version="8.0.2" />
<PackageReference Include="RefactoringEssentials" Version="5.6.0" PrivateAssets="all" />
<PackageReference Include="Squidex.AI.EntityFramework" Version="6.33.0" />
<PackageReference Include="Squidex.Assets.EntityFramework" Version="6.33.0" />
<PackageReference Include="Squidex.Assets.TusAdapter" Version="6.33.0" />
<PackageReference Include="Squidex.AI.EntityFramework" Version="6.35.0" />
<PackageReference Include="Squidex.Assets.EntityFramework" Version="6.35.0" />
<PackageReference Include="Squidex.Assets.TusAdapter" Version="6.35.0" />
<PackageReference Include="Squidex.EFCore.BulkExtensions" Version="8.1.6" />
<PackageReference Include="Squidex.Events.EntityFramework" Version="6.33.0" />
<PackageReference Include="Squidex.Hosting" Version="6.33.0" />
<PackageReference Include="Squidex.Messaging.EntityFramework" Version="6.33.0" />
<PackageReference Include="Squidex.Events.EntityFramework" Version="6.35.0" />
<PackageReference Include="Squidex.Hosting" Version="6.35.0" />
<PackageReference Include="Squidex.Messaging.EntityFramework" Version="6.35.0" />
<PackageReference Include="Squidex.OpenIdDict.EntityFramework" Version="5.8.4" />
<PackageReference Include="StyleCop.Analyzers" Version="1.1.118" PrivateAssets="all" />
<PackageReference Include="System.ValueTuple" Version="4.5.0" />

10
backend/src/Squidex.Data.MongoDb/Squidex.Data.MongoDb.csproj

@ -24,11 +24,11 @@
<PackageReference Include="MongoDB.Driver.Core.Extensions.DiagnosticSources" Version="1.5.0" />
<PackageReference Include="MongoDB.Driver.GridFS" Version="2.30.0" />
<PackageReference Include="RefactoringEssentials" Version="5.6.0" PrivateAssets="all" />
<PackageReference Include="Squidex.AI.Mongo" Version="6.33.0" />
<PackageReference Include="Squidex.Assets.Mongo" Version="6.33.0" />
<PackageReference Include="Squidex.Events.Mongo" Version="6.33.0" />
<PackageReference Include="Squidex.Hosting" Version="6.33.0" />
<PackageReference Include="Squidex.Messaging.Mongo" Version="6.33.0" />
<PackageReference Include="Squidex.AI.Mongo" Version="6.35.0" />
<PackageReference Include="Squidex.Assets.Mongo" Version="6.35.0" />
<PackageReference Include="Squidex.Events.Mongo" Version="6.35.0" />
<PackageReference Include="Squidex.Hosting" Version="6.35.0" />
<PackageReference Include="Squidex.Messaging.Mongo" Version="6.35.0" />
<PackageReference Include="Squidex.OpenIddict.MongoDb" Version="5.8.4" />
<PackageReference Include="StyleCop.Analyzers" Version="1.1.118" PrivateAssets="all" />
<PackageReference Include="System.ValueTuple" Version="4.5.0" />

4
backend/src/Squidex.Domain.Apps.Core.Operations/Squidex.Domain.Apps.Core.Operations.csproj

@ -28,8 +28,8 @@
<PackageReference Include="Microsoft.Extensions.Http" Version="8.0.1" />
<PackageReference Include="NJsonSchema" Version="11.0.2" />
<PackageReference Include="RefactoringEssentials" Version="5.6.0" PrivateAssets="all" />
<PackageReference Include="Squidex.AI" Version="6.33.0" />
<PackageReference Include="Squidex.Messaging.Subscriptions" Version="6.33.0" />
<PackageReference Include="Squidex.AI" Version="6.35.0" />
<PackageReference Include="Squidex.Messaging.Subscriptions" Version="6.35.0" />
<PackageReference Include="StyleCop.Analyzers" Version="1.1.118" PrivateAssets="all" />
<PackageReference Include="System.Collections.Immutable" Version="8.0.0" />
<PackageReference Include="System.Linq.Async" Version="6.0.1" />

9
backend/src/Squidex.Domain.Apps.Entities/Contents/Text/TextIndexingProcess.cs

@ -331,16 +331,15 @@ public sealed class TextIndexingProcess(
public async Task On(IEnumerable<Envelope<IEvent>> events)
{
var states = await QueryStatesAsync(events);
var updates = new Updates(states, serializer);
var textStates = await QueryStatesAsync(events);
var textBatch = new Updates(textStates, serializer);
foreach (var @event in events)
{
updates.On(@event);
textBatch.On(@event);
}
await updates.WriteAsync(textIndex, textIndexerState);
await textBatch.WriteAsync(textIndex, textIndexerState);
}
private Task<Dictionary<UniqueContentId, TextContentState>> QueryStatesAsync(IEnumerable<Envelope<IEvent>> events)

14
backend/src/Squidex.Infrastructure/Squidex.Infrastructure.csproj

@ -24,13 +24,13 @@
<PackageReference Include="NodaTime" Version="3.2.0" />
<PackageReference Include="OpenTelemetry.Api" Version="1.9.0" />
<PackageReference Include="RefactoringEssentials" Version="5.6.0" PrivateAssets="all" />
<PackageReference Include="Squidex.Assets" Version="6.33.0" />
<PackageReference Include="Squidex.Caching" Version="6.33.0" />
<PackageReference Include="Squidex.Events" Version="6.33.0" />
<PackageReference Include="Squidex.Hosting.Abstractions" Version="6.33.0" />
<PackageReference Include="Squidex.Log" Version="6.33.0" />
<PackageReference Include="Squidex.Messaging" Version="6.33.0" />
<PackageReference Include="Squidex.Text" Version="6.33.0" />
<PackageReference Include="Squidex.Assets" Version="6.35.0" />
<PackageReference Include="Squidex.Caching" Version="6.35.0" />
<PackageReference Include="Squidex.Events" Version="6.35.0" />
<PackageReference Include="Squidex.Hosting.Abstractions" Version="6.35.0" />
<PackageReference Include="Squidex.Log" Version="6.35.0" />
<PackageReference Include="Squidex.Messaging" Version="6.35.0" />
<PackageReference Include="Squidex.Text" Version="6.35.0" />
<PackageReference Include="StyleCop.Analyzers" Version="1.1.118" PrivateAssets="all" />
<PackageReference Include="System.Collections.Immutable" Version="8.0.0" />
<PackageReference Include="System.ComponentModel.Annotations" Version="5.0.0" />

22
backend/src/Squidex/Squidex.csproj

@ -62,18 +62,18 @@
<PackageReference Include="OpenTelemetry.Instrumentation.Runtime" Version="1.9.0" />
<PackageReference Include="RefactoringEssentials" Version="5.6.0" PrivateAssets="all" />
<PackageReference Include="ReportGenerator" Version="5.4.1" PrivateAssets="all" />
<PackageReference Include="Squidex.Assets.Azure" Version="6.33.0" />
<PackageReference Include="Squidex.Assets.GoogleCloud" Version="6.33.0" />
<PackageReference Include="Squidex.Assets.FTP" Version="6.33.0" />
<PackageReference Include="Squidex.Assets.ImageMagick" Version="6.33.0" />
<PackageReference Include="Squidex.Assets.ImageSharp" Version="6.33.0" />
<PackageReference Include="Squidex.Assets.S3" Version="6.33.0" />
<PackageReference Include="Squidex.Assets.TusAdapter" Version="6.33.0" />
<PackageReference Include="Squidex.Assets.Azure" Version="6.35.0" />
<PackageReference Include="Squidex.Assets.GoogleCloud" Version="6.35.0" />
<PackageReference Include="Squidex.Assets.FTP" Version="6.35.0" />
<PackageReference Include="Squidex.Assets.ImageMagick" Version="6.35.0" />
<PackageReference Include="Squidex.Assets.ImageSharp" Version="6.35.0" />
<PackageReference Include="Squidex.Assets.S3" Version="6.35.0" />
<PackageReference Include="Squidex.Assets.TusAdapter" Version="6.35.0" />
<PackageReference Include="Squidex.ClientLibrary" Version="20.1.0" />
<PackageReference Include="Squidex.Events.GetEventStore" Version="6.33.0" />
<PackageReference Include="Squidex.Hosting" Version="6.33.0" />
<PackageReference Include="Squidex.Messaging.All" Version="6.33.0" />
<PackageReference Include="Squidex.Messaging.Subscriptions" Version="6.33.0" />
<PackageReference Include="Squidex.Events.GetEventStore" Version="6.35.0" />
<PackageReference Include="Squidex.Hosting" Version="6.35.0" />
<PackageReference Include="Squidex.Messaging.All" Version="6.35.0" />
<PackageReference Include="Squidex.Messaging.Subscriptions" Version="6.35.0" />
<PackageReference Include="StyleCop.Analyzers" Version="1.1.118" PrivateAssets="all" />
<PackageReference Include="YDotNet" Version="0.4.3" />
<PackageReference Include="YDotNet.Native" Version="0.4.3" />

Loading…
Cancel
Save