Improved indices for contents.

backend/src/Squidex.Domain.Apps.Entities.MongoDb/Contents/MongoContentCollection.cs

@@ -45,16 +45,29 @@ namespace Squidex.Domain.Apps.Entities.MongoDb.Contents
         {
             return collection.Indexes.CreateManyAsync(new[]
             {
+                new CreateIndexModel<MongoContentEntity>(Index
+                    .Ascending(x => x.IndexedAppId)
+                    .Ascending(x => x.IsDeleted)
+                    .Ascending(x => x.Status)
+                    .Descending(x => x.LastModified)),
                 new CreateIndexModel<MongoContentEntity>(Index
                     .Ascending(x => x.IndexedAppId)
                     .Ascending(x => x.IsDeleted)
                     .Ascending(x => x.Status)
                     .Ascending(x => x.Id)),
+                new CreateIndexModel<MongoContentEntity>(Index
+                    .Ascending(x => x.IndexedSchemaId)
+                    .Ascending(x => x.IsDeleted)
+                    .Ascending(x => x.Status)
+                    .Descending(x => x.LastModified)),
                 new CreateIndexModel<MongoContentEntity>(Index
                     .Ascending(x => x.IndexedSchemaId)
                     .Ascending(x => x.IsDeleted)
                     .Ascending(x => x.Status)
                     .Ascending(x => x.Id)),
+                new CreateIndexModel<MongoContentEntity>(Index
+                    .Ascending(x => x.IndexedAppId)
+                    .Ascending(x => x.Id)),
                 new CreateIndexModel<MongoContentEntity>(Index
                     .Ascending(x => x.ScheduledAt)
                     .Ascending(x => x.IsDeleted)),
@@ -149,11 +162,19 @@ namespace Squidex.Domain.Apps.Entities.MongoDb.Contents
 
         public async Task<IContentEntity?> FindContentAsync(ISchemaEntity schema, Guid id, Status[]? status, bool includeDraft)
         {
-            var find = Collection.Find(FilterFactory.Build(schema.Id, id, status));
+            var find = Collection.Find(x => x.Id == id);
 
             var contentEntity = await find.WithoutDraft(includeDraft).FirstOrDefaultAsync();
 
-            contentEntity?.ParseData(schema.SchemaDef, serializer);
+            if (contentEntity != null)
+            {
+                if (contentEntity.IndexedSchemaId != schema.Id || status?.Contains(contentEntity.Status) == false)
+                {
+                    return null;
+                }
+
+                contentEntity?.ParseData(schema.SchemaDef, serializer);
+            }
 
             return contentEntity;
         }
@@ -180,10 +201,10 @@ namespace Squidex.Domain.Apps.Entities.MongoDb.Contents
             return contentEntities.Select(x => (Guid.Parse(x["_si"].AsString), Guid.Parse(x["_id"].AsString))).ToList();
         }
 
-        public async Task<IReadOnlyList<(Guid SchemaId, Guid Id)>> QueryIdsAsync(HashSet<Guid> ids)
+        public async Task<IReadOnlyList<(Guid SchemaId, Guid Id)>> QueryIdsAsync(Guid appId, HashSet<Guid> ids)
         {
             var contentEntities =
-                await Collection.Find(Filter.In(x => x.Id, ids)).Only(x => x.Id, x => x.IndexedSchemaId)
+                await Collection.Find(Filter.And(Filter.Eq(x => x.IndexedAppId, appId), Filter.In(x => x.Id, ids))).Only(x => x.Id, x => x.IndexedSchemaId)
                     .ToListAsync();
 
             return contentEntities.Select(x => (Guid.Parse(x["_si"].AsString), Guid.Parse(x["_id"].AsString))).ToList();

backend/src/Squidex.Domain.Apps.Entities.MongoDb/Contents/MongoContentRepository.cs

@@ -129,7 +129,7 @@ namespace Squidex.Domain.Apps.Entities.MongoDb.Contents
         {
             using (Profiler.TraceMethod<MongoContentRepository>())
             {
-                return await contents.QueryIdsAsync(ids);
+                return await contents.QueryIdsAsync(appId, ids);
             }
         }
 

backend/src/Squidex.Domain.Apps.Entities.MongoDb/Contents/Visitors/FilterFactory.cs

@@ -62,11 +62,6 @@ namespace Squidex.Domain.Apps.Entities.MongoDb.Contents.Visitors
             return !includeDraft ? cursor.Not(x => x.DataDraftByIds, x => x.IsDeleted) : cursor;
         }
 
-        public static FilterDefinition<MongoContentEntity> Build(Guid schemaId, Guid id, Status[]? status)
-        {
-            return CreateFilter(null, schemaId, new List<Guid> { id }, status, null);
-        }
-
         public static FilterDefinition<MongoContentEntity> IdsByApp(Guid appId, ICollection<Guid> ids, Status[]? status)
         {
             return CreateFilter(appId, null, ids, status, null);

backend/src/Squidex.Domain.Apps.Entities/Contents/Text/Extensions.cs

@@ -111,7 +111,7 @@ namespace Squidex.Domain.Apps.Entities.Contents.Text
                 result = new BytesRef();
             }
 
-            if (reader == null)
+            if (reader == null || docId < 0)
             {
                 return result;
             }

backend/src/Squidex.Infrastructure.MongoDb/EventSourcing/MongoEventStore_Writer.cs

@@ -16,7 +16,7 @@ namespace Squidex.Infrastructure.EventSourcing
 {
     public partial class MongoEventStore
     {
-        private const int MaxCommitSize = 10;
+        private const int MaxCommitSize = 100;
         private const int MaxWriteAttempts = 20;
         private static readonly BsonTimestamp EmptyTimestamp = new BsonTimestamp(0);
 

backend/tools/LoadTest/LoadTest.csproj

@@ -4,9 +4,9 @@
     <TargetFramework>netcoreapp3.0</TargetFramework>
   </PropertyGroup>
   <ItemGroup>
-    <PackageReference Include="Microsoft.NET.Test.Sdk" Version="16.2.0" />
+    <PackageReference Include="Microsoft.NET.Test.Sdk" Version="16.4.0" />
     <PackageReference Include="RefactoringEssentials" Version="5.6.0" PrivateAssets="all" />
-    <PackageReference Include="Squidex.ClientLibrary" Version="3.9.0" />
+    <PackageReference Include="Squidex.ClientLibrary" Version="4.0.2" />
     <PackageReference Include="StyleCop.Analyzers" Version="1.1.118" PrivateAssets="all" />
     <PackageReference Include="xunit" Version="2.4.1" />
     <PackageReference Include="xunit.runner.visualstudio" Version="2.4.1">

backend/tools/LoadTest/ManyItemsTests.cs

@@ -0,0 +1,67 @@
+// ==========================================================================
+//  Squidex Headless CMS
+// ==========================================================================
+//  Copyright (c) Squidex UG (haftungsbeschraenkt)
+//  All rights reserved. Licensed under the MIT license.
+// ==========================================================================
+
+using System.Threading.Tasks;
+using LoadTest.Model;
+using LoadTest.Utils;
+using Squidex.ClientLibrary;
+using Xunit;
+
+namespace LoadTest
+{
+    public class ManyItemsTests
+    {
+        private readonly SquidexClient<TestEntity, TestEntityData> client;
+
+        public ManyItemsTests()
+        {
+            client = TestClient.BuildAsync("multiple").Result;
+        }
+
+        [Fact]
+        public async Task Should_read_many_async()
+        {
+            var contents = await client.GetAsync();
+
+            for (var i = contents.Total; i < 20000; i++)
+            {
+                await client.CreateAsync(new TestEntityData
+                {
+                    String = RandomString.Create(1000)
+                }, publish: true);
+            }
+
+            var found = await client.GetAll2Async();
+
+            Assert.Equal(20000, found.Items.Count);
+        }
+    }
+
+    public static class SquidexClientExtensions
+    {
+        public static async Task<SquidexEntities<TEntity, TData>> GetAll2Async<TEntity, TData>(this SquidexClient<TEntity, TData> client, int batchSize = 200)
+            where TEntity : SquidexEntityBase<TData>
+            where TData : class, new()
+        {
+            var query = new ODataQuery { Top = batchSize, Skip = 0 };
+
+            var entities = new SquidexEntities<TEntity, TData>();
+            do
+            {
+                var getResult = await client.GetAsync(query);
+
+                entities.Total = getResult.Total;
+                entities.Items.AddRange(getResult.Items);
+
+                query.Skip += getResult.Items.Count;
+            }
+            while (query.Skip < entities.Total);
+
+            return entities;
+        }
+    }
+}

backend/tools/LoadTest/Model/TestClient.cs

@@ -72,11 +72,18 @@ namespace LoadTest.Model
                     {
                         new UpsertSchemaFieldDto
                         {
-                            Name = "value",
+                            Name = "number",
                             Properties = new NumberFieldPropertiesDto
                             {
-                                IsRequired = true,
+                                IsRequired = true
-                                IsListField = true
+                            }
+                        },
+                        new UpsertSchemaFieldDto
+                        {
+                            Name = "string",
+                            Properties = new StringFieldPropertiesDto
+                            {
+                                IsRequired = false
                             }
                         }
                     },

backend/tools/LoadTest/Model/TestEntity.cs

@@ -17,6 +17,9 @@ namespace LoadTest.Model
     public sealed class TestEntityData
     {
         [JsonConverter(typeof(InvariantConverter))]
-        public int Value { get; set; }
+        public int Number { get; set; }
+
+        [JsonConverter(typeof(InvariantConverter))]
+        public string String { get; set; }
     }
 }

backend/tools/LoadTest/ReadingFixture.cs

@@ -36,7 +36,7 @@ namespace LoadTest
 
                     for (var i = 10; i > 0; i--)
                     {
-                        await Client.CreateAsync(new TestEntityData { Value = i }, true);
+                        await Client.CreateAsync(new TestEntityData { Number = i }, true);
                     }
                 }
 

backend/tools/LoadTest/Utils/RandomHash.cs

@@ -0,0 +1,48 @@
+// ==========================================================================
+//  Squidex Headless CMS
+// ==========================================================================
+//  Copyright (c) Squidex UG (haftungsbeschraenkt)
+//  All rights reserved. Licensed under the MIT license.
+// ==========================================================================
+
+using System;
+using System.Security.Cryptography;
+using System.Text;
+
+namespace LoadTest.Utils
+{
+    public static class RandomHash
+    {
+        public static string New()
+        {
+            return Guid.NewGuid()
+                .ToString().Sha256Base64()
+                .ToLowerInvariant()
+                .Replace("+", "x")
+                .Replace("=", "x")
+                .Replace("/", "x");
+        }
+
+        public static string Simple()
+        {
+            return Guid.NewGuid().ToString().Replace("-", string.Empty);
+        }
+
+        public static string Sha256Base64(this string value)
+        {
+            return Sha256Base64(Encoding.UTF8.GetBytes(value));
+        }
+
+        public static string Sha256Base64(this byte[] bytes)
+        {
+            using (var sha = SHA256.Create())
+            {
+                var bytesHash = sha.ComputeHash(bytes);
+
+                var result = Convert.ToBase64String(bytesHash);
+
+                return result;
+            }
+        }
+    }
+}

backend/tools/LoadTest/Utils/RandomString.cs

@@ -0,0 +1,28 @@
+// ==========================================================================
+//  Squidex Headless CMS
+// ==========================================================================
+//  Copyright (c) Squidex UG (haftungsbeschraenkt)
+//  All rights reserved. Licensed under the MIT license.
+// ==========================================================================
+
+using System;
+
+namespace LoadTest.Utils
+{
+    public static class RandomString
+    {
+        private static readonly Random Random = new Random();
+
+        public static string Create(int length)
+        {
+            var chars = new char[length];
+
+            for (var i = 0; i < length; i++)
+            {
+                chars[i] = (char)Random.Next(48, 122);
+            }
+
+            return new string(chars);
+        }
+    }
+}

backend/tools/LoadTest/WritingBenchmarks.cs

@@ -62,7 +62,7 @@ namespace LoadTest
 
             await Run.Parallel(numUsers, numIterationsPerUser, async () =>
             {
-                await Fixture.Client.CreateAsync(new TestEntityData { Value = random.Next() }, true);
+                await Fixture.Client.CreateAsync(new TestEntityData { Number = random.Next() }, true);
             });
         }
     }

backend/tools/LoadTest/WritingFixture.cs

@@ -20,7 +20,7 @@ namespace LoadTest
         {
             Task.Run(async () =>
             {
-                Client = await TestClient.BuildAsync("reading");
+                Client = await TestClient.BuildAsync("writing");
             }).Wait();
         }