Feature/azure search (#786)

* Azure cognitive search.

* Azure search finalized.

backend/extensions/Squidex.Extensions/Squidex.Extensions.csproj

@@ -9,6 +9,7 @@
   </ItemGroup>
   <ItemGroup>
     <PackageReference Include="Algolia.Search" Version="6.10.1" />
+    <PackageReference Include="Azure.Search.Documents" Version="11.3.0" />
     <PackageReference Include="Confluent.Apache.Avro" Version="1.7.7.7" />
     <PackageReference Include="Confluent.Kafka" Version="1.7.0" />
     <PackageReference Include="Confluent.SchemaRegistry.Serdes" Version="1.3.0" />

backend/extensions/Squidex.Extensions/Text/Azure/AzureIndexDefinition.cs

@@ -0,0 +1,130 @@
+// ==========================================================================
+//  Squidex Headless CMS
+// ==========================================================================
+//  Copyright (c) Squidex UG (haftungsbeschraenkt)
+//  All rights reserved. Licensed under the MIT license.
+// ==========================================================================
+
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Reflection;
+using Azure.Search.Documents.Indexes.Models;
+
+namespace Squidex.Extensions.Text.Azure
+{
+    public static class AzureIndexDefinition
+    {
+        private static readonly Dictionary<string, (string Field, string Analyzer)> AllowedLanguages = new Dictionary<string, (string Field, string Analyzer)>(StringComparer.OrdinalIgnoreCase)
+        {
+            ["iv"] = ("text_iv", LexicalAnalyzerName.StandardLucene.ToString()),
+            ["zh"] = ("text_zh", LexicalAnalyzerName.ZhHansLucene.ToString())
+        };
+
+        static AzureIndexDefinition()
+        {
+            var analyzers =
+                typeof(LexicalAnalyzerName)
+                    .GetProperties(BindingFlags.Public | BindingFlags.Static)
+                    .Select(x => x.GetValue(null))
+                    .Select(x => x.ToString())
+                    .OrderBy(x => x)
+                    .ToList();
+
+            var addedLanguage = new HashSet<string>();
+
+            foreach (var analyzer in analyzers)
+            {
+                var indexOfDot = analyzer.IndexOf('.', StringComparison.Ordinal);
+
+                if (indexOfDot > 0)
+                {
+                    var language = analyzer[0..indexOfDot];
+
+                    var isValidLanguage =
+                        language.Length == 2 ||
+                        language.StartsWith("zh-", StringComparison.Ordinal);
+
+                    if (isValidLanguage && addedLanguage.Add(language))
+                    {
+                        var fieldName = $"text_{language.Replace('-', '_')}";
+
+                        AllowedLanguages[language] = (fieldName, analyzer);
+                    }
+                }
+            }
+        }
+
+        public static string GetTextField(string key)
+        {
+            if (AllowedLanguages.TryGetValue(key, out var field))
+            {
+                return field.Field;
+            }
+
+            if (key.Length > 2 && AllowedLanguages.TryGetValue(key[2..], out field))
+            {
+                return field.Field;
+            }
+
+            return AllowedLanguages["iv"].Field;
+        }
+
+        public static SearchIndex Create(string indexName)
+        {
+            var fields = new List<SearchField>
+            {
+                new SimpleField("docId", SearchFieldDataType.String)
+                {
+                    IsKey = true,
+                },
+                new SimpleField("appId", SearchFieldDataType.String)
+                {
+                    IsFilterable = true,
+                },
+                new SimpleField("appName", SearchFieldDataType.String)
+                {
+                    IsFilterable = false
+                },
+                new SimpleField("contentId", SearchFieldDataType.String)
+                {
+                    IsFilterable = false
+                },
+                new SearchableField("schemaId")
+                {
+                    IsFilterable = true
+                },
+                new SimpleField("schemaName", SearchFieldDataType.String)
+                {
+                    IsFilterable = false
+                },
+                new SimpleField("serveAll", SearchFieldDataType.Boolean)
+                {
+                    IsFilterable = true
+                },
+                new SimpleField("servePublished", SearchFieldDataType.Boolean)
+                {
+                    IsFilterable = true
+                }
+            };
+
+            foreach (var (field, analyzer) in AllowedLanguages.Values)
+            {
+                fields.Add(
+                    new SearchableField(field)
+                    {
+                        IsFilterable = false,
+                        IsFacetable = false,
+                        AnalyzerName = analyzer
+                    });
+            }
+
+            var index = new SearchIndex(indexName)
+            {
+                Fields = fields
+            };
+
+            return index;
+        }
+    }
+}

backend/extensions/Squidex.Extensions/Text/Azure/AzureTextIndex.cs

@@ -0,0 +1,147 @@
+// ==========================================================================
+//  Squidex Headless CMS
+// ==========================================================================
+//  Copyright (c) Squidex UG (haftungsbeschraenkt)
+//  All rights reserved. Licensed under the MIT license.
+// ==========================================================================
+
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading;
+using System.Threading.Tasks;
+using Azure;
+using Azure.Search.Documents;
+using Azure.Search.Documents.Indexes;
+using Azure.Search.Documents.Models;
+using Squidex.Domain.Apps.Entities.Apps;
+using Squidex.Domain.Apps.Entities.Contents;
+using Squidex.Domain.Apps.Entities.Contents.Text;
+using Squidex.Hosting;
+using Squidex.Infrastructure;
+
+namespace Squidex.Extensions.Text.Azure
+{
+    public sealed class AzureTextIndex : IInitializable, ITextIndex
+    {
+        private readonly SearchIndexClient indexClient;
+        private readonly SearchClient searchClient;
+
+        public AzureTextIndex(
+            string serviceEndpoint,
+            string serviceApiKey,
+            string indexName)
+        {
+            indexClient = new SearchIndexClient(new Uri(serviceEndpoint), new AzureKeyCredential(serviceApiKey));
+
+            searchClient = indexClient.GetSearchClient(indexName);
+        }
+
+        public async Task InitializeAsync(
+            CancellationToken ct)
+        {
+            await CreateIndexAsync(ct);
+        }
+
+        public async Task ClearAsync(
+            CancellationToken ct = default)
+        {
+            await indexClient.DeleteIndexAsync(searchClient.IndexName, ct);
+
+            await CreateIndexAsync(ct);
+        }
+
+        private async Task CreateIndexAsync(
+            CancellationToken ct)
+        {
+            var index = AzureIndexDefinition.Create(searchClient.IndexName);
+
+            await indexClient.CreateOrUpdateIndexAsync(index, true, true, ct);
+        }
+
+        public async Task ExecuteAsync(IndexCommand[] commands,
+            CancellationToken ct = default)
+        {
+            var batch = IndexDocumentsBatch.Create<SearchDocument>();
+
+            commands.Foreach(x => CommandFactory.CreateCommands(x, batch.Actions));
+
+            await searchClient.IndexDocumentsAsync(batch, cancellationToken: ct);
+        }
+
+        public Task<List<DomainId>> SearchAsync(IAppEntity app, GeoQuery query, SearchScope scope,
+            CancellationToken ct = default)
+        {
+            Guard.NotNull(app, nameof(app));
+            Guard.NotNull(query, nameof(query));
+
+            return Task.FromResult<List<DomainId>>(null);
+        }
+
+        public async Task<List<DomainId>> SearchAsync(IAppEntity app, TextQuery query, SearchScope scope,
+            CancellationToken ct = default)
+        {
+            Guard.NotNull(app, nameof(app));
+            Guard.NotNull(query, nameof(query));
+
+            if (string.IsNullOrWhiteSpace(query.Text))
+            {
+                return null;
+            }
+
+            var searchOptions = new SearchOptions
+            {
+                Filter = BuildFilter(app, query, scope)
+            };
+
+            searchOptions.Select.Add("contentId");
+            searchOptions.Size = 2000;
+
+            var results = await searchClient.SearchAsync<SearchDocument>("*", searchOptions, ct);
+
+            var ids = new List<DomainId>();
+
+            await foreach (var item in results.Value.GetResultsAsync().WithCancellation(ct))
+            {
+                if (item != null)
+                {
+                    ids.Add(DomainId.Create(item.Document["contentId"].ToString()));
+                }
+            }
+
+            return ids;
+        }
+
+        private static string BuildFilter(IAppEntity app, TextQuery query, SearchScope scope)
+        {
+            var sb = new StringBuilder();
+
+            sb.Append($"appId eq '{app.Id}' and {GetServeField(scope)} eq true");
+
+            if (query.RequiredSchemaIds?.Count > 0)
+            {
+                var schemaIds = string.Join(" or ", query.RequiredSchemaIds.Select(x => $"schemaId eq '{x}'"));
+
+                sb.Append($" and ({schemaIds}) and search.ismatchscoring('{query.Text}')");
+            }
+            else if (query.PreferredSchemaId.HasValue)
+            {
+                sb.Append($" and ((search.ismatchscoring('{query.Text}') and search.ismatchscoring('{query.PreferredSchemaId}', 'schemaId')) or search.ismatchscoring('{query.Text}'))");
+            }
+            else
+            {
+                sb.Append($" and search.ismatchscoring('{query.Text}')");
+            }
+
+            return sb.ToString();
+        }
+
+        private static string GetServeField(SearchScope scope)
+        {
+            return scope == SearchScope.Published ?
+                "servePublished" :
+                "serveAll";
+        }
+    }
+}

backend/extensions/Squidex.Extensions/Text/Azure/AzureTextPlugin.cs

@@ -0,0 +1,62 @@
+// ==========================================================================
+//  Squidex Headless CMS
+// ==========================================================================
+//  Copyright (c) Squidex UG (haftungsbeschraenkt)
+//  All rights reserved. Licensed under the MIT license.
+// ==========================================================================
+
+using System;
+using Microsoft.Extensions.Configuration;
+using Microsoft.Extensions.DependencyInjection;
+using Squidex.Domain.Apps.Entities.Contents.Text;
+using Squidex.Hosting;
+using Squidex.Hosting.Configuration;
+using Squidex.Infrastructure.Plugins;
+
+namespace Squidex.Extensions.Text.Azure
+{
+    public sealed class AzureTextPlugin : IPlugin
+    {
+        public void ConfigureServices(IServiceCollection services, IConfiguration config)
+        {
+            var fullTextType = config.GetValue<string>("fullText:type");
+
+            if (string.Equals(fullTextType, "Azure", StringComparison.OrdinalIgnoreCase))
+            {
+                var serviceEndpoint = config.GetValue<string>("fullText:azure:serviceEndpoint");
+
+                if (string.IsNullOrWhiteSpace(serviceEndpoint))
+                {
+                    var error = new ConfigurationError("Value is required.", "fullText:azure:serviceEndpoint");
+
+                    throw new ConfigurationException(error);
+                }
+
+                var serviceApiKey = config.GetValue<string>("fullText:azure:apiKey");
+
+                if (string.IsNullOrWhiteSpace(serviceApiKey))
+                {
+                    var error = new ConfigurationError("Value is required.", "fullText:azure:apiKey");
+
+                    throw new ConfigurationException(error);
+                }
+
+                var indexName = config.GetValue<string>("fullText:azure:indexName");
+
+                if (string.IsNullOrWhiteSpace(indexName))
+                {
+                    indexName = "squidex-index";
+                }
+
+                services.AddSingleton(
+                    c => new AzureTextIndex(serviceEndpoint, serviceApiKey, indexName));
+
+                services.AddSingleton<ITextIndex>(
+                    c => c.GetRequiredService<AzureTextIndex>());
+
+                services.AddSingleton<IInitializable>(
+                    c => c.GetRequiredService<AzureTextIndex>());
+            }
+        }
+    }
+}

backend/extensions/Squidex.Extensions/Text/Azure/CommandFactory.cs

@@ -0,0 +1,81 @@
+// ==========================================================================
+//  Squidex Headless CMS
+// ==========================================================================
+//  Copyright (c) Squidex UG (haftungsbeschraenkt)
+//  All rights reserved. Licensed under the MIT license.
+// ==========================================================================
+
+using System;
+using System.Collections.Generic;
+using System.Text;
+using Azure.Search.Documents.Models;
+using Squidex.Domain.Apps.Entities.Contents.Text;
+
+namespace Squidex.Extensions.Text.Azure
+{
+    public static class CommandFactory
+    {
+        public static void CreateCommands(IndexCommand command, IList<IndexDocumentsAction<SearchDocument>> batch)
+        {
+            switch (command)
+            {
+                case UpsertIndexEntry upsert:
+                    batch.Add(UpsertEntry(upsert));
+                    break;
+                case UpdateIndexEntry update:
+                    batch.Add(UpdateEntry(update));
+                    break;
+                case DeleteIndexEntry delete:
+                    batch.Add(DeleteEntry(delete));
+                    break;
+            }
+        }
+
+        private static IndexDocumentsAction<SearchDocument> UpsertEntry(UpsertIndexEntry upsert)
+        {
+            var searchDocument = new SearchDocument
+            {
+                ["docId"] = upsert.DocId.ToBase64(),
+                ["appId"] = upsert.AppId.Id.ToString(),
+                ["appName"] = upsert.AppId.Name,
+                ["contentId"] = upsert.ContentId.ToString(),
+                ["schemaId"] = upsert.SchemaId.Id.ToString(),
+                ["schemaName"] = upsert.SchemaId.Name,
+                ["serveAll"] = upsert.ServeAll,
+                ["servePublished"] = upsert.ServePublished
+            };
+
+            if (upsert.Texts != null)
+            {
+                foreach (var (key, value) in upsert.Texts)
+                {
+                    searchDocument[AzureIndexDefinition.GetTextField(key)] = value;
+                }
+            }
+
+            return IndexDocumentsAction.MergeOrUpload(searchDocument);
+        }
+
+        private static IndexDocumentsAction<SearchDocument> UpdateEntry(UpdateIndexEntry update)
+        {
+            var searchDocument = new SearchDocument
+            {
+                ["docId"] = update.DocId.ToBase64(),
+                ["serveAll"] = update.ServeAll,
+                ["servePublished"] = update.ServePublished,
+            };
+
+            return IndexDocumentsAction.MergeOrUpload(searchDocument);
+        }
+
+        private static IndexDocumentsAction<SearchDocument> DeleteEntry(DeleteIndexEntry delete)
+        {
+            return IndexDocumentsAction.Delete("docId", delete.DocId.ToBase64());
+        }
+
+        private static string ToBase64(this string value)
+        {
+            return Convert.ToBase64String(Encoding.Default.GetBytes(value));
+        }
+    }
+}

backend/src/Squidex.Domain.Apps.Entities/Contents/Text/Elastic/CommandFactory.cs → backend/extensions/Squidex.Extensions/Text/ElasticSearch/CommandFactory.cs

@@ -6,8 +6,9 @@
 // ==========================================================================
 
 using System.Collections.Generic;
+using Squidex.Domain.Apps.Entities.Contents.Text;
 
-namespace Squidex.Domain.Apps.Entities.Contents.Text.Elastic
+namespace Squidex.Extensions.Text.ElasticSearch
 {
     public static class CommandFactory
     {

backend/src/Squidex.Domain.Apps.Entities/Contents/Text/Elastic/ElasticSearchMapping.cs → backend/extensions/Squidex.Extensions/Text/ElasticSearch/ElasticSearchMapping.cs

@@ -11,7 +11,7 @@ using System.Threading;
 using System.Threading.Tasks;
 using Elasticsearch.Net;
 
-namespace Squidex.Domain.Apps.Entities.Contents.Text.Elastic
+namespace Squidex.Extensions.Text.ElasticSearch
 {
     public static class ElasticSearchMapping
     {

backend/src/Squidex.Domain.Apps.Entities/Contents/Text/Elastic/ElasticSearchTextIndex.cs → backend/extensions/Squidex.Extensions/Text/ElasticSearch/ElasticSearchTextIndex.cs

@@ -7,18 +7,18 @@
 
 using System;
 using System.Collections.Generic;
-using System.Diagnostics.CodeAnalysis;
 using System.Linq;
 using System.Threading;
 using System.Threading.Tasks;
 using Elasticsearch.Net;
 using Squidex.Domain.Apps.Entities.Apps;
+using Squidex.Domain.Apps.Entities.Contents;
+using Squidex.Domain.Apps.Entities.Contents.Text;
 using Squidex.Hosting;
 using Squidex.Infrastructure;
 
-namespace Squidex.Domain.Apps.Entities.Contents.Text.Elastic
+namespace Squidex.Extensions.Text.ElasticSearch
 {
-    [ExcludeFromCodeCoverage]
     public sealed class ElasticSearchTextIndex : ITextIndex, IInitializable
     {
         private readonly ElasticLowLevelClient client;
@@ -74,13 +74,13 @@ namespace Squidex.Domain.Apps.Entities.Contents.Text.Elastic
             }
         }
 
-        public Task<List<DomainId>?> SearchAsync(IAppEntity app, GeoQuery query, SearchScope scope,
+        public Task<List<DomainId>> SearchAsync(IAppEntity app, GeoQuery query, SearchScope scope,
             CancellationToken ct = default)
         {
-            return Task.FromResult<List<DomainId>?>(null);
+            return Task.FromResult<List<DomainId>>(null);
         }
 
-        public async Task<List<DomainId>?> SearchAsync(IAppEntity app, TextQuery query, SearchScope scope,
+        public async Task<List<DomainId>> SearchAsync(IAppEntity app, TextQuery query, SearchScope scope,
             CancellationToken ct = default)
         {
             Guard.NotNull(app, nameof(app));
@@ -161,24 +161,29 @@ namespace Squidex.Domain.Apps.Entities.Contents.Text.Elastic
                 size = 2000
             };
 
-            if (query.Filter?.SchemaIds?.Length > 0)
+            if (query.RequiredSchemaIds?.Count > 0)
             {
                 var bySchema = new
                 {
                     terms = new Dictionary<string, object>
                     {
-                        ["schemaId.keyword"] = query.Filter.SchemaIds.Select(x => x.ToString()).ToArray()
+                        ["schemaId.keyword"] = query.RequiredSchemaIds.Select(x => x.ToString()).ToArray()
                     }
                 };
 
-                if (query.Filter.Must)
-                {
                 elasticQuery.query.@bool.must.Add(bySchema);
             }
-                else
+            else if (query.PreferredSchemaId.HasValue)
             {
-                    elasticQuery.query.@bool.should.Add(bySchema);
+                var bySchema = new
+                {
+                    terms = new Dictionary<string, object>
+                    {
+                        ["schemaId.keyword"] = query.PreferredSchemaId.ToString()
                     }
+                };
+
+                elasticQuery.query.@bool.should.Add(bySchema);
             }
 
             var result = await client.SearchAsync<DynamicResponse>(indexName, CreatePost(elasticQuery), ctx: ct);

backend/extensions/Squidex.Extensions/Text/ElasticSearch/ElasticSearchTextPlugin.cs

@@ -0,0 +1,53 @@
+// ==========================================================================
+//  Squidex Headless CMS
+// ==========================================================================
+//  Copyright (c) Squidex UG (haftungsbeschraenkt)
+//  All rights reserved. Licensed under the MIT license.
+// ==========================================================================
+
+using System;
+using Microsoft.Extensions.Configuration;
+using Microsoft.Extensions.DependencyInjection;
+using Squidex.Domain.Apps.Entities.Contents.Text;
+using Squidex.Hosting;
+using Squidex.Hosting.Configuration;
+using Squidex.Infrastructure.Plugins;
+
+namespace Squidex.Extensions.Text.ElasticSearch
+{
+    public sealed class ElasticSearchTextPlugin : IPlugin
+    {
+        public void ConfigureServices(IServiceCollection services, IConfiguration config)
+        {
+            var fullTextType = config.GetValue<string>("fullText:type");
+
+            if (string.Equals(fullTextType, "elastic", StringComparison.OrdinalIgnoreCase))
+            {
+                var elasticConfiguration = config.GetValue<string>("fullText:elastic:configuration");
+
+                if (string.IsNullOrWhiteSpace(elasticConfiguration))
+                {
+                    var error = new ConfigurationError("Value is required.", "fullText:elastic:configuration");
+
+                    throw new ConfigurationException(error);
+                }
+
+                var indexName = config.GetValue<string>("fullText:elastic:indexName");
+
+                if (string.IsNullOrWhiteSpace(indexName))
+                {
+                    indexName = "squidex-index";
+                }
+
+                services.AddSingleton(
+                    c => new ElasticSearchTextIndex(elasticConfiguration, indexName));
+
+                services.AddSingleton<ITextIndex>(
+                    c => c.GetRequiredService<ElasticSearchTextIndex>());
+
+                services.AddSingleton<IInitializable>(
+                    c => c.GetRequiredService<ElasticSearchTextIndex>());
+            }
+        }
+    }
+}

backend/src/Squidex.Domain.Apps.Entities.MongoDb/FullText/MongoTextIndex.cs

@@ -9,25 +9,44 @@ using System.Collections.Generic;
 using System.Linq;
 using System.Threading;
 using System.Threading.Tasks;
+using MongoDB.Bson;
+using MongoDB.Bson.Serialization.Attributes;
 using MongoDB.Driver;
 using Squidex.Domain.Apps.Entities.Apps;
 using Squidex.Domain.Apps.Entities.Contents;
 using Squidex.Domain.Apps.Entities.Contents.Text;
 using Squidex.Infrastructure;
 using Squidex.Infrastructure.MongoDb;
-using Squidex.Infrastructure.Tasks;
 
 namespace Squidex.Domain.Apps.Entities.MongoDb.FullText
 {
     public sealed class MongoTextIndex : MongoRepositoryBase<MongoTextIndexEntity>, ITextIndex, IDeleter
     {
-        private const int Limit = 2000;
+        private readonly ProjectionDefinition<MongoTextIndexEntity> searchTextProjection;
-        private const int LimitHalf = 1000;
+        private readonly ProjectionDefinition<MongoTextIndexEntity> searchGeoProjection;
-        private static readonly List<DomainId> EmptyResults = new List<DomainId>();
+
+        private sealed class MongoTextResult
+        {
+            [BsonId]
+            [BsonElement]
+            [BsonRepresentation(BsonType.String)]
+            public string Id { get; set; }
+
+            [BsonRequired]
+            [BsonElement("_ci")]
+            [BsonRepresentation(BsonType.String)]
+            public DomainId ContentId { get; set; }
+
+            [BsonIgnoreIfDefault]
+            [BsonElement("score")]
+            public double Score { get; set; }
+        }
 
         public MongoTextIndex(IMongoDatabase database, bool setup = false)
             : base(database, setup)
         {
+            searchGeoProjection = Projection.Include(x => x.ContentId);
+            searchTextProjection = Projection.Include(x => x.ContentId).MetaTextScore("score");
         }
 
         protected override Task SetupCollectionAsync(IMongoCollection<MongoTextIndexEntity> collection,
@@ -97,78 +116,80 @@ namespace Squidex.Domain.Apps.Entities.MongoDb.FullText
                     Filter.GeoWithinCenterSphere(x => x.GeoObject, query.Longitude, query.Latitude, query.Radius / 6378100));
 
             var byGeo =
-                await GetCollection(scope).Find(findFilter).Limit(Limit).Only(x => x.ContentId)
+                await GetCollection(scope).Find(findFilter).Limit(query.Take).Project<MongoTextResult>(searchGeoProjection)
                     .ToListAsync(ct);
 
-            var field = Field.Of<MongoTextIndexEntity>(x => nameof(x.ContentId));
+            return byGeo.Select(x => x.ContentId).ToList();
-
-            return byGeo.Select(x => DomainId.Create(x[field].AsString)).Distinct().ToList();
         }
 
         public async Task<List<DomainId>?> SearchAsync(IAppEntity app, TextQuery query, SearchScope scope,
             CancellationToken ct = default)
         {
-            var (queryText, filter) = query;
+            if (string.IsNullOrWhiteSpace(query.Text))
-
-            if (string.IsNullOrWhiteSpace(queryText))
             {
-                return EmptyResults;
+                return null;
             }
 
-            if (filter == null)
+            List<MongoTextResult> documents;
+
+            if (query.RequiredSchemaIds?.Count > 0)
             {
-                return await SearchByAppAsync(queryText, app, scope, Limit, ct);
+                documents = await SearchBySchemaAsync(query.Text, app, query.RequiredSchemaIds, scope, query.Take, ct);
             }
-            else if (filter.Must)
+            else if (query.PreferredSchemaId == null)
             {
-                return await SearchBySchemaAsync(queryText, app, filter, scope, Limit, ct);
+                documents = await SearchByAppAsync(query.Text, app, scope, query.Take, ct);
             }
             else
             {
-                var (bySchema, byApp) = await AsyncHelper.WhenAll(
+                var halfBucket = query.Take / 2;
-                    SearchBySchemaAsync(queryText, app, filter, scope, LimitHalf, ct),
-                    SearchByAppAsync(queryText, app, scope, LimitHalf, ct));
 
-                return bySchema.Union(byApp).Distinct().ToList();
+                var schemaIds = Enumerable.Repeat(query.PreferredSchemaId.Value, 1);
+
+                documents = new List<MongoTextResult>();
+                documents.AddRange(await SearchBySchemaAsync(query.Text, app, schemaIds, scope, halfBucket, ct));
+                documents.AddRange(await SearchByAppAsync(query.Text, app, scope, halfBucket, ct));
             }
+
+            return documents.OrderByDescending(x => x.Score).Select(x => x.ContentId).Distinct().ToList();
         }
 
-        private async Task<List<DomainId>> SearchBySchemaAsync(string queryText, IAppEntity app, TextFilter filter, SearchScope scope, int limit,
+        private Task<List<MongoTextResult>> SearchBySchemaAsync(string queryText, IAppEntity app, IEnumerable<DomainId> schemaIds, SearchScope scope, int limit,
             CancellationToken ct = default)
         {
-            var findFilter =
+            var filter =
                 Filter.And(
                     Filter.Eq(x => x.AppId, app.Id),
-                    Filter.In(x => x.SchemaId, filter.SchemaIds),
+                    Filter.In(x => x.SchemaId, schemaIds),
                     Filter_ByScope(scope),
                     Filter.Text(queryText, "none"));
 
-            var bySchema =
+            return SearchAsync(filter, scope, limit, ct);
-                await GetCollection(scope).Find(findFilter).Limit(limit).Only(x => x.ContentId)
-                    .ToListAsync(ct);
-
-            var field = Field.Of<MongoTextIndexEntity>(x => nameof(x.ContentId));
-
-            return bySchema.Select(x => DomainId.Create(x[field].AsString)).Distinct().ToList();
         }
 
-        private async Task<List<DomainId>> SearchByAppAsync(string queryText, IAppEntity app, SearchScope scope, int limit,
+        private Task<List<MongoTextResult>> SearchByAppAsync(string queryText, IAppEntity app, SearchScope scope, int limit,
             CancellationToken ct = default)
         {
-            var findFilter =
+            var filter =
                 Filter.And(
                     Filter.Eq(x => x.AppId, app.Id),
                     Filter.Exists(x => x.SchemaId),
                     Filter_ByScope(scope),
                     Filter.Text(queryText, "none"));
 
-            var bySchema =
+            return SearchAsync(filter, scope, limit, ct);
-                await GetCollection(scope).Find(findFilter).Limit(limit).Only(x => x.ContentId)
+        }
-                    .ToListAsync(ct);
+
+        private Task<List<MongoTextResult>> SearchAsync(FilterDefinition<MongoTextIndexEntity> filter, SearchScope scope, int limit,
+            CancellationToken ct = default)
+        {
+            var collection = GetCollection(scope);
 
-            var field = Field.Of<MongoTextIndexEntity>(x => nameof(x.ContentId));
+            var find =
+                collection.Find(filter).Limit(limit)
+                    .Project<MongoTextResult>(searchTextProjection).Sort(Sort.MetaTextScore("score"));
 
-            return bySchema.Select(x => DomainId.Create(x[field].AsString)).Distinct().ToList();
+            return find.ToListAsync(ct);
         }
 
         private static FilterDefinition<MongoTextIndexEntity> Filter_ByScope(SearchScope scope)

backend/src/Squidex.Domain.Apps.Entities/Contents/ContentEntity.cs

@@ -46,6 +46,8 @@ namespace Squidex.Domain.Apps.Entities.Contents
 
         public bool IsSingleton { get; set; }
 
+        public bool IsDeleted { get; set; }
+
         public string SchemaDisplayName { get; set; }
 
         public string StatusColor { get; set; }

backend/src/Squidex.Domain.Apps.Entities/Contents/ContentsSearchSource.cs

@@ -6,6 +6,7 @@
 // ==========================================================================
 
 using System.Collections.Generic;
+using System.Linq;
 using System.Text;
 using System.Threading;
 using System.Threading.Tasks;
@@ -46,14 +47,17 @@ namespace Squidex.Domain.Apps.Entities.Contents
         {
             var result = new SearchResults();
 
-            var searchFilter = await CreateSearchFilterAsync(context, ct);
+            var schemaIds = await GetSchemaIdsAsync(context, ct);
 
-            if (searchFilter == null)
+            if (schemaIds.Count == 0)
             {
                 return result;
             }
 
-            var textQuery = new TextQuery($"{query}~", searchFilter);
+            var textQuery = new TextQuery($"{query}~", 10)
+            {
+                RequiredSchemaIds = schemaIds
+            };
 
             var ids = await contentTextIndexer.SearchAsync(context.App, textQuery, context.Scope(), ct);
 
@@ -78,27 +82,12 @@ namespace Squidex.Domain.Apps.Entities.Contents
             return result;
         }
 
-        private async Task<TextFilter?> CreateSearchFilterAsync(Context context,
+        private async Task<List<DomainId>> GetSchemaIdsAsync(Context context,
             CancellationToken ct)
         {
-            var allowedSchemas = new List<DomainId>();
-
             var schemas = await appProvider.GetSchemasAsync(context.App.Id, ct);
 
-            foreach (var schema in schemas)
+            return schemas.Where(x => HasPermission(context, x.SchemaDef.Name)).Select(x => x.Id).ToList();
-            {
-                if (HasPermission(context, schema.SchemaDef.Name))
-                {
-                    allowedSchemas.Add(schema.Id);
-                }
-            }
-
-            if (allowedSchemas.Count == 0)
-            {
-                return null;
-            }
-
-            return TextFilter.MustHaveSchemas(allowedSchemas.ToArray());
         }
 
         private static bool HasPermission(Context context, string schemaName)

backend/src/Squidex.Domain.Apps.Entities/Contents/IContentEntity.cs

@@ -27,5 +27,7 @@ namespace Squidex.Domain.Apps.Entities.Contents
         ContentData Data { get; }
 
         ScheduleJob? ScheduleJob { get; }
+
+        bool IsDeleted { get; }
     }
 }

backend/src/Squidex.Domain.Apps.Entities/Contents/Queries/ContentQueryParser.cs

@@ -92,7 +92,10 @@ namespace Squidex.Domain.Apps.Entities.Contents.Queries
                     throw new InvalidOperationException();
                 }
 
-                var textQuery = new TextQuery(query.FullText, TextFilter.ShouldHaveSchemas(schema.Id));
+                var textQuery = new TextQuery(query.FullText, 1000)
+                {
+                    PreferredSchemaId = schema.Id
+                };
 
                 var fullTextIds = await textIndex.SearchAsync(context.App, textQuery, context.Scope());
                 var fullTextFilter = ClrFilter.Eq("id", "__notfound__");

backend/src/Squidex.Domain.Apps.Entities/Contents/Queries/GeoQueryTransformer.cs

@@ -38,7 +38,7 @@ namespace Squidex.Domain.Apps.Entities.Contents.Queries
             {
                 var field = string.Join(".", nodeIn.Path.Skip(1));
 
-                var searchQuery = new GeoQuery(args.Schema.Id, field, sphere.Latitude, sphere.Longitude, sphere.Radius);
+                var searchQuery = new GeoQuery(args.Schema.Id, field, sphere.Latitude, sphere.Longitude, sphere.Radius, 1000);
                 var searchScope = args.Context.Scope();
 
                 var ids = await args.TextIndex.SearchAsync(args.Context.App, searchQuery, searchScope);

backend/src/Squidex.Domain.Apps.Entities/Contents/Text/GeoQuery.cs

@@ -11,7 +11,7 @@ using Squidex.Infrastructure;
 
 namespace Squidex.Domain.Apps.Entities.Contents.Text
 {
-    public sealed record GeoQuery(DomainId SchemaId, string Field, double Latitude, double Longitude, double Radius)
+    public sealed record GeoQuery(DomainId SchemaId, string Field, double Latitude, double Longitude, double Radius, int Take)
     {
     }
 }

backend/src/Squidex.Domain.Apps.Entities/Contents/Text/TextSearch.cs

@@ -5,26 +5,17 @@
 //  All rights reserved. Licensed under the MIT license.
 // ==========================================================================
 
+using System.Collections.Generic;
 using Squidex.Infrastructure;
 
 #pragma warning disable SA1313 // Parameter names should begin with lower-case letter
 
 namespace Squidex.Domain.Apps.Entities.Contents.Text
 {
-    public sealed record TextQuery(string? Text, TextFilter? Filter)
+    public sealed record TextQuery(string Text, int Take)
     {
-    }
+        public IReadOnlyList<DomainId>? RequiredSchemaIds { get; init; }
 
-    public sealed record TextFilter(DomainId[]? SchemaIds, bool Must)
+        public DomainId? PreferredSchemaId { get; init; }
-    {
-        public static TextFilter MustHaveSchemas(params DomainId[] schemaIds)
-        {
-            return new TextFilter(schemaIds, true);
-        }
-
-        public static TextFilter ShouldHaveSchemas(params DomainId[] schemaIds)
-        {
-            return new TextFilter(schemaIds, false);
-        }
     }
 }

backend/src/Squidex.Domain.Apps.Entities/Squidex.Domain.Apps.Entities.csproj

@@ -18,7 +18,6 @@
   </ItemGroup>
   <ItemGroup>
     <PackageReference Include="CsvHelper" Version="27.1.1" />
-    <PackageReference Include="Elasticsearch.Net" Version="7.14.1" />
     <PackageReference Include="Equals.Fody" Version="4.0.1" PrivateAssets="all" />
     <PackageReference Include="Fody" Version="6.5.2">
       <PrivateAssets>all</PrivateAssets>

backend/src/Squidex.Infrastructure.MongoDb/EventSourcing/MongoEventStore.cs

@@ -5,7 +5,6 @@
 //  All rights reserved. Licensed under the MIT license.
 // ==========================================================================
 
-using System;
 using System.Threading;
 using System.Threading.Tasks;
 using MongoDB.Bson;

backend/src/Squidex/Areas/Api/Controllers/Contents/Models/ContentDto.cs

@@ -103,6 +103,11 @@ namespace Squidex.Areas.Api.Controllers.Contents.Models
         /// </summary>
         public FieldDto[]? ReferenceFields { get; set; }
 
+        /// <summary>
+        /// Indicates whether the content is deleted.
+        /// </summary>
+        public bool IsDeleted { get; set; }
+
         /// <summary>
         /// The version of the content.
         /// </summary>
@@ -140,6 +145,11 @@ namespace Squidex.Areas.Api.Controllers.Contents.Models
                 SimpleMapper.Map(content.ScheduleJob, response.ScheduleJob);
             }
 
+            if (response.IsDeleted)
+            {
+                return response;
+            }
+
             return response.CreateLinksAsync(content, resources, content.SchemaId.Name);
         }
 

backend/src/Squidex/Config/Domain/ContentsServices.cs

@@ -16,7 +16,6 @@ using Squidex.Domain.Apps.Entities.Contents.DomainObject;
 using Squidex.Domain.Apps.Entities.Contents.Queries;
 using Squidex.Domain.Apps.Entities.Contents.Queries.Steps;
 using Squidex.Domain.Apps.Entities.Contents.Text;
-using Squidex.Domain.Apps.Entities.Contents.Text.Elastic;
 using Squidex.Domain.Apps.Entities.Contents.Validation;
 using Squidex.Domain.Apps.Entities.History;
 using Squidex.Domain.Apps.Entities.Search;
@@ -97,19 +96,6 @@ namespace Squidex.Config.Domain
 
             services.AddSingletonAs<GrainBootstrap<IContentSchedulerGrain>>()
                 .AsSelf();
-
-            config.ConfigureByOption("fullText:type", new Alternatives
-            {
-                ["Elastic"] = () =>
-                {
-                    var elasticConfiguration = config.GetRequiredValue("fullText:elastic:configuration");
-                    var elasticIndexName = config.GetRequiredValue("fullText:elastic:indexName");
-
-                    services.AddSingletonAs(c => new ElasticSearchTextIndex(elasticConfiguration, elasticIndexName))
-                        .As<ITextIndex>();
-                },
-                ["Default"] = () => { }
-            });
         }
     }
 }

backend/src/Squidex/Config/Domain/EventSourcingServices.cs

@@ -7,7 +7,6 @@
 
 using System.Linq;
 using EventStore.Client;
-using EventStore.ClientAPI;
 using Microsoft.Extensions.Configuration;
 using Microsoft.Extensions.DependencyInjection;
 using MongoDB.Driver;
@@ -16,7 +15,6 @@ using Squidex.Infrastructure.Commands;
 using Squidex.Infrastructure.Diagnostics;
 using Squidex.Infrastructure.EventSourcing;
 using Squidex.Infrastructure.EventSourcing.Grains;
-using Squidex.Infrastructure.Json;
 using Squidex.Infrastructure.States;
 
 namespace Squidex.Config.Domain

backend/tests/Squidex.Domain.Apps.Entities.Tests/Contents/ContentsSearchSourceTests.cs

@@ -192,7 +192,7 @@ namespace Squidex.Domain.Apps.Entities.Contents
 
             var ids = new List<DomainId> { content.Id };
 
-            A.CallTo(() => contentIndex.SearchAsync(ctx.App, A<TextQuery>.That.Matches(x => x.Text == "query~" && x.Filter != null), ctx.Scope(), ct))
+            A.CallTo(() => contentIndex.SearchAsync(ctx.App, A<TextQuery>.That.Matches(x => x.Text == "query~"), ctx.Scope(), ct))
                 .Returns(ids);
 
             A.CallTo(() => contentQuery.QueryAsync(ctx, A<Q>.That.HasIds(ids), ct))

backend/tests/Squidex.Domain.Apps.Entities.Tests/Contents/Queries/ContentQueryParserTests.cs

@@ -194,7 +194,7 @@ namespace Squidex.Domain.Apps.Entities.Contents.Queries
         [Fact]
         public async Task Should_convert_geo_query_to_filter()
         {
-            A.CallTo(() => textIndex.SearchAsync(requestContext.App, new GeoQuery(schemaId.Id, "geo.iv", 10, 20, 30), requestContext.Scope(), default))
+            A.CallTo(() => textIndex.SearchAsync(requestContext.App, new GeoQuery(schemaId.Id, "geo.iv", 10, 20, 30, 1000), requestContext.Scope(), default))
                 .Returns(new List<DomainId> { DomainId.Create("1"), DomainId.Create("2") });
 
             var query = Q.Empty.WithODataQuery("$filter=geo.distance(data/geo/iv, geography'POINT(20 10)') lt 30.0");
@@ -207,7 +207,7 @@ namespace Squidex.Domain.Apps.Entities.Contents.Queries
         [Fact]
         public async Task Should_convert_geo_query_to_filter_if_single_id_found()
         {
-            A.CallTo(() => textIndex.SearchAsync(requestContext.App, new GeoQuery(schemaId.Id, "geo.iv", 10, 20, 30), requestContext.Scope(), default))
+            A.CallTo(() => textIndex.SearchAsync(requestContext.App, new GeoQuery(schemaId.Id, "geo.iv", 10, 20, 30, 1000), requestContext.Scope(), default))
                 .Returns(new List<DomainId> { DomainId.Create("1") });
 
             var query = Q.Empty.WithODataQuery("$filter=geo.distance(data/geo/iv, geography'POINT(20 10)') lt 30.0");
@@ -220,7 +220,7 @@ namespace Squidex.Domain.Apps.Entities.Contents.Queries
         [Fact]
         public async Task Should_convert_geo_query_to_filter_if_index_returns_null()
         {
-            A.CallTo(() => textIndex.SearchAsync(requestContext.App, new GeoQuery(schemaId.Id, "geo.iv", 10, 20, 30), requestContext.Scope(), default))
+            A.CallTo(() => textIndex.SearchAsync(requestContext.App, new GeoQuery(schemaId.Id, "geo.iv", 10, 20, 30, 1000), requestContext.Scope(), default))
                 .Returns(Task.FromResult<List<DomainId>?>(null));
 
             var query = Q.Empty.WithODataQuery("$filter=geo.distance(data/geo/iv, geography'POINT(20 10)') lt 30.0");
@@ -233,7 +233,7 @@ namespace Squidex.Domain.Apps.Entities.Contents.Queries
         [Fact]
         public async Task Should_convert_geo_query_to_filter_if_index_returns_empty()
         {
-            A.CallTo(() => textIndex.SearchAsync(requestContext.App, new GeoQuery(schemaId.Id, "geo.iv", 10, 20, 30), requestContext.Scope(), default))
+            A.CallTo(() => textIndex.SearchAsync(requestContext.App, new GeoQuery(schemaId.Id, "geo.iv", 10, 20, 30, 1000), requestContext.Scope(), default))
                 .Returns(new List<DomainId>());
 
             var query = Q.Empty.WithODataQuery("$filter=geo.distance(data/geo/iv, geography'POINT(20 10)') lt 30.0");

backend/tests/Squidex.Domain.Apps.Entities.Tests/Contents/Text/TextIndexerTestsBase.cs

@@ -392,7 +392,7 @@ namespace Squidex.Domain.Apps.Entities.Contents.Text
         {
             return async p =>
             {
-                var query = new GeoQuery(schemaId.Id, field, latitude, longitude, 1000);
+                var query = new GeoQuery(schemaId.Id, field, latitude, longitude, 1000, 1000);
 
                 var result = await p.TextIndex.SearchAsync(app, query, target);
 
@@ -411,7 +411,10 @@ namespace Squidex.Domain.Apps.Entities.Contents.Text
         {
             return async p =>
             {
-                var query = new TextQuery(text, TextFilter.ShouldHaveSchemas(schemaId.Id));
+                var query = new TextQuery(text, 1000)
+                {
+                    RequiredSchemaIds = new List<DomainId> { schemaId.Id }
+                };
 
                 var result = await p.TextIndex.SearchAsync(app, query, target);
 

backend/tests/Squidex.Domain.Apps.Entities.Tests/Contents/Text/TextIndexerTests_Elastic.cs

@@ -6,7 +6,7 @@
 // ==========================================================================
 
 using System.Threading.Tasks;
-using Squidex.Domain.Apps.Entities.Contents.Text.Elastic;
+using Squidex.Extensions.Text.ElasticSearch;
 using Squidex.Infrastructure;
 using Xunit;
 

backend/tests/Squidex.Domain.Apps.Entities.Tests/Squidex.Domain.Apps.Entities.Tests.csproj

@@ -8,6 +8,7 @@
     <Nullable>enable</Nullable>
   </PropertyGroup>
   <ItemGroup>
+    <ProjectReference Include="..\..\extensions\Squidex.Extensions\Squidex.Extensions.csproj" />
     <ProjectReference Include="..\..\src\Squidex.Domain.Apps.Core.Model\Squidex.Domain.Apps.Core.Model.csproj" />
     <ProjectReference Include="..\..\src\Squidex.Domain.Apps.Core.Operations\Squidex.Domain.Apps.Core.Operations.csproj" />
     <ProjectReference Include="..\..\src\Squidex.Domain.Apps.Entities.MongoDb\Squidex.Domain.Apps.Entities.MongoDb.csproj" />

frontend/app/framework/angular/forms/editors/autocomplete.component.html

@@ -15,8 +15,8 @@
         <i class="icon-{{icon}}" [class.icon-spinner2]="snapshot.isLoading" [class.spin2]="snapshot.isLoading"></i>
     </div>
 
-    <ng-container *sqxModal="snapshot.suggestedItems.length > 0" position="bottom-left">
+    <ng-container *sqxModal="snapshot.suggestedItems.length > 0">
-        <div class="control-dropdown" [sqxAnchoredTo]="input" [style.width]="dropdownWidth" position="bottom-left" #container @fade>
+        <div class="control-dropdown" [sqxAnchoredTo]="input" [style.width]="dropdownWidth" [position]="dropdownPosition" #container @fade>
             <div *ngFor="let item of snapshot.suggestedItems; let i = index" class="control-dropdown-item control-dropdown-item-selectable"
                 [class.active]="i === snapshot.suggestedIndex"
                 (mousedown)="selectItem(item)"

frontend/app/framework/angular/forms/editors/autocomplete.component.ts

@@ -74,6 +74,9 @@ export class AutocompleteComponent extends StatefulControlComponent<State, Reado
     @Input()
     public debounceTime = 300;
 
+    @Input()
+    public dropdownPosition = 'bottom-left';
+
     @Input()
     public dropdownWidth = '18rem';
 

frontend/app/framework/angular/resized.directive.ts

@@ -12,7 +12,7 @@ import { ResizeListener, ResizeService, ResourceOwner } from '@app/framework/int
     selector: '[sqxResized], [sqxResizeCondition]',
 })
 export class ResizedDirective extends ResourceOwner implements OnDestroy, OnChanges, ResizeListener {
-    private condition: ((rect: ClientRect) => boolean) | undefined;
+    private condition: ((rect: DOMRect) => boolean) | undefined;
     private conditionValue = false;
 
     @Input('sqxResizeMinWidth')
@@ -25,7 +25,7 @@ export class ResizedDirective extends ResourceOwner implements OnDestroy, OnChan
     public resizeCondition = new EventEmitter<boolean>();
 
     @Output('sqxResized')
-    public resize = new EventEmitter<ClientRect>();
+    public resize = new EventEmitter<DOMRect>();
 
     constructor(resizeService: ResizeService, element: ElementRef,
         private readonly zone: NgZone,
@@ -50,7 +50,7 @@ export class ResizedDirective extends ResourceOwner implements OnDestroy, OnChan
         }
     }
 
-    public onResize(rect: ClientRect) {
+    public onResize(rect: DOMRect) {
         if (this.condition) {
             const value = this.condition(rect);
 

frontend/app/framework/angular/sync-width.directive.ts

@@ -29,7 +29,7 @@ export class SyncWidthDirective extends ResourceOwner implements AfterViewInit,
         this.onReposition();
     }
 
-    public onResize(size: ClientRect) {
+    public onResize(size: DOMRect) {
         this.resize(size.width);
     }
 

frontend/app/framework/utils/modal-positioner.spec.ts

@@ -8,7 +8,7 @@
 import { positionModal } from './modal-positioner';
 
 describe('position', () => {
-    function buildRect(x: number, y: number, w: number, h: number): ClientRect {
+    function buildRect(x: number, y: number, w: number, h: number): any {
         return {
             top: y,
             left: x,

frontend/app/framework/utils/modal-positioner.ts

@@ -18,7 +18,7 @@ const POSITION_RIGHT_CENTER = 'right';
 const POSITION_RIGHT_TOP = 'right-top';
 const POSITION_RIGHT_BOTTOM = 'right-bottom';
 
-export function positionModal(targetRect: ClientRect, modalRect: ClientRect, relativePosition: string, offset: number, fix: boolean, viewportWidth: number, viewportHeight: number): { x: number; y: number } {
+export function positionModal(targetRect: DOMRect, modalRect: DOMRect, relativePosition: string, offset: number, fix: boolean, viewportWidth: number, viewportHeight: number): { x: number; y: number } {
     let y = 0;
     let x = 0;
 

frontend/app/shell/pages/internal/search-menu.component.html

@@ -2,6 +2,7 @@
     <div class="search-container ms-4">
         <sqx-autocomplete #searchControl
             dropdownWidth="30rem"
+            dropdownPosition="bottom-right"
             icon="search"
             inputName="searchMenu" 
             (ngModelChange)="selectResult($event)"