From 1f13ed43af27f2b05d9257e3884559e3badade06 Mon Sep 17 00:00:00 2001 From: Sebastian Stehle Date: Fri, 6 Mar 2020 09:16:15 +0100 Subject: [PATCH] Index tests (#494) * Indexing tests * Benchmark indexing performance * Cleanup --- backend/Squidex.sln | 19 ++- .../FullText/MongoDirectory.cs | 150 ------------------ .../FullText/MongoIndexInput.cs | 105 ------------ .../FullText/MongoIndexOutput.cs | 114 ------------- .../FullText/MongoIndexStorage.cs | 92 +++++++++-- .../Text/Lucene/LuceneTextIndexGrain.cs | 11 +- .../Text/Lucene/Storage/AssetIndexStorage.cs | 27 +--- .../Text/Lucene/Storage/FileIndexStorage.cs | 4 +- .../Text/Lucene/Storage/IIndexStorage.cs | 4 +- .../Config/Domain/SubscriptionServices.cs | 2 +- .../Contents/Text/TextIndexerBenchmark.cs | 117 -------------- backend/tools/Benchmarks/Benchmarks.csproj | 24 +++ backend/tools/Benchmarks/IndexStorages.cs | 49 ++++++ .../tools/Benchmarks/IndexingBenchmarks.cs | 103 ++++++++++++ backend/tools/Benchmarks/Program.cs | 19 +++ backend/tools/Benchmarks/Utils/NoopLog.cs | 28 ++++ 16 files changed, 343 insertions(+), 525 deletions(-) delete mode 100644 backend/src/Squidex.Domain.Apps.Entities.MongoDb/FullText/MongoDirectory.cs delete mode 100644 backend/src/Squidex.Domain.Apps.Entities.MongoDb/FullText/MongoIndexInput.cs delete mode 100644 backend/src/Squidex.Domain.Apps.Entities.MongoDb/FullText/MongoIndexOutput.cs delete mode 100644 backend/tests/Squidex.Domain.Apps.Entities.Tests/Contents/Text/TextIndexerBenchmark.cs create mode 100644 backend/tools/Benchmarks/Benchmarks.csproj create mode 100644 backend/tools/Benchmarks/IndexStorages.cs create mode 100644 backend/tools/Benchmarks/IndexingBenchmarks.cs create mode 100644 backend/tools/Benchmarks/Program.cs create mode 100644 backend/tools/Benchmarks/Utils/NoopLog.cs diff --git a/backend/Squidex.sln b/backend/Squidex.sln index 88fd5bb3e..272b5a13f 100644 --- a/backend/Squidex.sln +++ b/backend/Squidex.sln @@ -22,7 +22,7 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Squidex.Infrastructure.Rabb EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Squidex.Infrastructure.GoogleCloud", "src\Squidex.Infrastructure.GoogleCloud\Squidex.Infrastructure.GoogleCloud.csproj", "{945871B1-77B8-43FB-B53C-27CF385AB756}" EndProject -Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "migrations", "migrations", "{94207AA6-4923-4183-A558-E0F8196B8CA3}" +Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "tools", "tools", "{94207AA6-4923-4183-A558-E0F8196B8CA3}" EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Migrate_00", "tools\Migrate_00\Migrate_00.csproj", "{B51126A8-0D75-4A79-867D-10724EC6AC84}" EndProject @@ -65,7 +65,9 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "shared", "shared", "{7EDE8C EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Squidex.Web", "src\Squidex.Web\Squidex.Web.csproj", "{5B2D251F-46E3-486A-AE16-E3FE06B559ED}" EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Squidex.Infrastructure.Amazon", "src\Squidex.Infrastructure.Amazon\Squidex.Infrastructure.Amazon.csproj", "{32DA4B56-7EFA-4E34-A29D-30E00579A894}" +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Squidex.Infrastructure.Amazon", "src\Squidex.Infrastructure.Amazon\Squidex.Infrastructure.Amazon.csproj", "{32DA4B56-7EFA-4E34-A29D-30E00579A894}" +EndProject +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Benchmarks", "tools\Benchmarks\Benchmarks.csproj", "{F36EF843-BDDD-45A8-B9C6-360001161AAA}" EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution @@ -353,6 +355,18 @@ Global {32DA4B56-7EFA-4E34-A29D-30E00579A894}.Release|x64.Build.0 = Release|Any CPU {32DA4B56-7EFA-4E34-A29D-30E00579A894}.Release|x86.ActiveCfg = Release|Any CPU {32DA4B56-7EFA-4E34-A29D-30E00579A894}.Release|x86.Build.0 = Release|Any CPU + {F36EF843-BDDD-45A8-B9C6-360001161AAA}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {F36EF843-BDDD-45A8-B9C6-360001161AAA}.Debug|Any CPU.Build.0 = Debug|Any CPU + {F36EF843-BDDD-45A8-B9C6-360001161AAA}.Debug|x64.ActiveCfg = Debug|Any CPU + {F36EF843-BDDD-45A8-B9C6-360001161AAA}.Debug|x64.Build.0 = Debug|Any CPU + {F36EF843-BDDD-45A8-B9C6-360001161AAA}.Debug|x86.ActiveCfg = Debug|Any CPU + {F36EF843-BDDD-45A8-B9C6-360001161AAA}.Debug|x86.Build.0 = Debug|Any CPU + {F36EF843-BDDD-45A8-B9C6-360001161AAA}.Release|Any CPU.ActiveCfg = Release|Any CPU + {F36EF843-BDDD-45A8-B9C6-360001161AAA}.Release|Any CPU.Build.0 = Release|Any CPU + {F36EF843-BDDD-45A8-B9C6-360001161AAA}.Release|x64.ActiveCfg = Release|Any CPU + {F36EF843-BDDD-45A8-B9C6-360001161AAA}.Release|x64.Build.0 = Release|Any CPU + {F36EF843-BDDD-45A8-B9C6-360001161AAA}.Release|x86.ActiveCfg = Release|Any CPU + {F36EF843-BDDD-45A8-B9C6-360001161AAA}.Release|x86.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE @@ -382,6 +396,7 @@ Global {F3C41B82-6A67-409A-B7FE-54543EE4F38B} = {FB8BC3A2-2010-4C3C-A87D-D4A98C05EE52} {5B2D251F-46E3-486A-AE16-E3FE06B559ED} = {7EDE8CF1-B1E4-4005-B154-834B944E0D7A} {32DA4B56-7EFA-4E34-A29D-30E00579A894} = {8CF53B92-5EB1-461D-98F8-70DA9B603FBF} + {F36EF843-BDDD-45A8-B9C6-360001161AAA} = {94207AA6-4923-4183-A558-E0F8196B8CA3} EndGlobalSection GlobalSection(ExtensibilityGlobals) = postSolution SolutionGuid = {02F2E872-3141-44F5-BD6A-33CD84E9FE08} diff --git a/backend/src/Squidex.Domain.Apps.Entities.MongoDb/FullText/MongoDirectory.cs b/backend/src/Squidex.Domain.Apps.Entities.MongoDb/FullText/MongoDirectory.cs deleted file mode 100644 index d6606f46f..000000000 --- a/backend/src/Squidex.Domain.Apps.Entities.MongoDb/FullText/MongoDirectory.cs +++ /dev/null @@ -1,150 +0,0 @@ -// ========================================================================== -// Squidex Headless CMS -// ========================================================================== -// Copyright (c) Squidex UG (haftungsbeschraenkt) -// All rights reserved. Licensed under the MIT license. -// ========================================================================== - -using System; -using System.Collections.Generic; -using System.IO; -using System.Linq; -using Lucene.Net.Store; -using MongoDB.Bson; -using MongoDB.Driver; -using MongoDB.Driver.GridFS; -using LuceneDirectory = Lucene.Net.Store.Directory; - -namespace Squidex.Domain.Apps.Entities.MongoDb.FullText -{ - public sealed class MongoDirectory : BaseDirectory - { - private readonly IGridFSBucket bucket; - private readonly string directory; - private readonly DirectoryInfo cacheDirectoryInfo; - private readonly LuceneDirectory cacheDirectory; - private bool isDisposed; - - public LuceneDirectory CacheDirectory - { - get { return cacheDirectory; } - } - - public DirectoryInfo CacheDirectoryInfo - { - get { return cacheDirectoryInfo; } - } - - public IGridFSBucket Bucket - { - get { return bucket; } - } - - public MongoDirectory(IGridFSBucket bucket, string directory, DirectoryInfo cacheDirectoryInfo) - { - this.bucket = bucket; - - this.directory = directory; - - this.cacheDirectoryInfo = cacheDirectoryInfo; - - cacheDirectoryInfo.Create(); - cacheDirectory = FSDirectory.Open(cacheDirectoryInfo); - - SetLockFactory(new NativeFSLockFactory(cacheDirectoryInfo)); - } - - protected override void Dispose(bool disposing) - { - if (disposing) - { - isDisposed = true; - - cacheDirectory.Dispose(); - } - } - - public override string GetLockID() - { - return cacheDirectory.GetLockID(); - } - - public override IndexOutput CreateOutput(string name, IOContext context) - { - return new MongoIndexOutput(this, context, name); - } - - public override IndexInput OpenInput(string name, IOContext context) - { - return new MongoIndexInput(this, context, name); - } - - public override void DeleteFile(string name) - { - EnsureNotDisposed(); - - var fullName = GetFullName(name); - - try - { - Bucket.Delete(fullName); - } - catch (GridFSFileNotFoundException) - { - } - } - - public override long FileLength(string name) - { - EnsureNotDisposed(); - - var file = FindFile(name) ?? throw new FileNotFoundException(null, GetFullName(name)); - - return file.Length; - } - - public override string[] ListAll() - { - EnsureNotDisposed(); - - var files = Bucket.Find(Builders>.Filter.Regex(x => x.Id, new BsonRegularExpression($"^{directory}/"))).ToList(); - - return files.Select(x => x.Filename).ToArray(); - } - - public GridFSFileInfo? FindFile(string name) - { - var fullName = GetFullName(name); - - return Bucket.Find(Builders>.Filter.Eq(x => x.Id, fullName)).FirstOrDefault(); - } - - public override void Sync(ICollection names) - { - } - - [Obsolete] - public override bool FileExists(string name) - { - throw new NotSupportedException(); - } - - public string GetFullName(string name) - { - return $"{directory}/{name}"; - } - - public string GetFullPath(string name) - { - return Path.Combine(cacheDirectoryInfo.FullName, name); - } - - private void EnsureNotDisposed() - { - if (isDisposed) - { - throw new ObjectDisposedException(GetType().FullName); - } - } - } -} diff --git a/backend/src/Squidex.Domain.Apps.Entities.MongoDb/FullText/MongoIndexInput.cs b/backend/src/Squidex.Domain.Apps.Entities.MongoDb/FullText/MongoIndexInput.cs deleted file mode 100644 index 18710ce67..000000000 --- a/backend/src/Squidex.Domain.Apps.Entities.MongoDb/FullText/MongoIndexInput.cs +++ /dev/null @@ -1,105 +0,0 @@ -// ========================================================================== -// Squidex Headless CMS -// ========================================================================== -// Copyright (c) Squidex UG (haftungsbeschraenkt) -// All rights reserved. Licensed under the MIT license. -// ========================================================================== - -using System.IO; -using Lucene.Net.Store; -using MongoDB.Driver.GridFS; - -namespace Squidex.Domain.Apps.Entities.MongoDb.FullText -{ - public sealed class MongoIndexInput : IndexInput - { - private readonly IndexInput cacheInput; - private readonly MongoDirectory indexDirectory; - private readonly IOContext context; - private readonly string indexFileName; - - public override long Length - { - get { return cacheInput.Length; } - } - - public MongoIndexInput(MongoDirectory indexDirectory, IOContext context, string indexFileName) - : base(indexDirectory.GetFullName(indexFileName)) - { - this.indexDirectory = indexDirectory; - this.indexFileName = indexFileName; - - this.context = context; - - try - { - var file = indexDirectory.FindFile(indexFileName); - - if (file != null) - { - var fileInfo = new FileInfo(indexDirectory.GetFullPath(indexFileName)); - - var writtenTime = file.Metadata["WrittenTime"].ToUniversalTime(); - - if (!fileInfo.Exists || fileInfo.LastWriteTimeUtc < writtenTime) - { - using (var fs = new FileStream(fileInfo.FullName, FileMode.Create, FileAccess.Write)) - { - var fullName = indexDirectory.GetFullName(indexFileName); - - indexDirectory.Bucket.DownloadToStream(fullName, fs); - } - } - } - } - catch (GridFSFileNotFoundException) - { - throw new FileNotFoundException(); - } - - cacheInput = indexDirectory.CacheDirectory.OpenInput(indexFileName, context); - } - - public MongoIndexInput(MongoIndexInput source) - : base("clone") - { - cacheInput = (IndexInput)source.cacheInput.Clone(); - context = source.context; - indexDirectory = source.indexDirectory; - indexFileName = source.indexFileName; - } - - protected override void Dispose(bool disposing) - { - if (disposing) - { - cacheInput.Dispose(); - } - } - - public override long GetFilePointer() - { - return cacheInput.GetFilePointer(); - } - - public override byte ReadByte() - { - return cacheInput.ReadByte(); - } - - public override void ReadBytes(byte[] b, int offset, int len) - { - cacheInput.ReadBytes(b, offset, len); - } - - public override void Seek(long pos) - { - cacheInput.Seek(pos); - } - - public override object Clone() - { - return new MongoIndexInput(indexDirectory, context, indexFileName); - } - } -} diff --git a/backend/src/Squidex.Domain.Apps.Entities.MongoDb/FullText/MongoIndexOutput.cs b/backend/src/Squidex.Domain.Apps.Entities.MongoDb/FullText/MongoIndexOutput.cs deleted file mode 100644 index 51c3fc88d..000000000 --- a/backend/src/Squidex.Domain.Apps.Entities.MongoDb/FullText/MongoIndexOutput.cs +++ /dev/null @@ -1,114 +0,0 @@ -// ========================================================================== -// Squidex Headless CMS -// ========================================================================== -// Copyright (c) Squidex UG (haftungsbeschraenkt) -// All rights reserved. Licensed under the MIT license. -// ========================================================================== - -using System; -using System.IO; -using System.Linq; -using Lucene.Net.Store; -using MongoDB.Bson; -using MongoDB.Driver; -using MongoDB.Driver.GridFS; - -namespace Squidex.Domain.Apps.Entities.MongoDb.FullText -{ - public sealed class MongoIndexOutput : IndexOutput - { - private readonly IndexOutput cacheOutput; - private readonly MongoDirectory indexDirectory; - private readonly string indexFileName; - private bool isFlushed; - private bool isWritten; - - public override long Length - { - get { return cacheOutput.Length; } - } - - public override long Checksum - { - get { return cacheOutput.Checksum; } - } - - public MongoIndexOutput(MongoDirectory indexDirectory, IOContext context, string indexFileName) - { - this.indexDirectory = indexDirectory; - this.indexFileName = indexFileName; - - cacheOutput = indexDirectory.CacheDirectory.CreateOutput(indexFileName, context); - } - - protected override void Dispose(bool disposing) - { - if (disposing) - { - Flush(); - - cacheOutput.Dispose(); - - if (isWritten && isFlushed) - { - var fileInfo = new FileInfo(indexDirectory.GetFullPath(indexFileName)); - - using (var fs = new FileStream(indexDirectory.GetFullPath(indexFileName), FileMode.Open, FileAccess.Read)) - { - var fullName = indexDirectory.GetFullName(indexFileName); - - var options = new GridFSUploadOptions - { - Metadata = new BsonDocument - { - ["WrittenTime"] = fileInfo.LastWriteTimeUtc - } - }; - - try - { - indexDirectory.Bucket.UploadFromStream(fullName, indexFileName, fs, options); - } - catch (MongoBulkWriteException ex) when (ex.WriteErrors.Any(x => x.Code == 11000)) - { - indexDirectory.Bucket.Delete(fullName); - indexDirectory.Bucket.UploadFromStream(fullName, indexFileName, fs, options); - } - } - } - } - } - - public override long GetFilePointer() - { - return cacheOutput.GetFilePointer(); - } - - public override void Flush() - { - cacheOutput.Flush(); - - isFlushed = true; - } - - public override void WriteByte(byte b) - { - cacheOutput.WriteByte(b); - - isWritten = true; - } - - public override void WriteBytes(byte[] b, int offset, int length) - { - cacheOutput.WriteBytes(b, offset, length); - - isWritten = true; - } - - [Obsolete] - public override void Seek(long pos) - { - cacheOutput.Seek(pos); - } - } -} diff --git a/backend/src/Squidex.Domain.Apps.Entities.MongoDb/FullText/MongoIndexStorage.cs b/backend/src/Squidex.Domain.Apps.Entities.MongoDb/FullText/MongoIndexStorage.cs index c5a81a42d..18e26a197 100644 --- a/backend/src/Squidex.Domain.Apps.Entities.MongoDb/FullText/MongoIndexStorage.cs +++ b/backend/src/Squidex.Domain.Apps.Entities.MongoDb/FullText/MongoIndexStorage.cs @@ -7,8 +7,10 @@ using System; using System.IO; +using System.IO.Compression; using System.Threading.Tasks; using Lucene.Net.Index; +using Lucene.Net.Store; using MongoDB.Driver.GridFS; using Squidex.Domain.Apps.Entities.Contents.Text.Lucene; using Squidex.Infrastructure; @@ -27,26 +29,96 @@ namespace Squidex.Domain.Apps.Entities.MongoDb.FullText this.bucket = bucket; } - public Task CreateDirectoryAsync(Guid ownerId) + public async Task CreateDirectoryAsync(Guid ownerId) { - var folderName = ownerId.ToString(); + var fileId = $"index_{ownerId}"; - var tempFolder = Path.Combine(Path.GetTempPath(), "Indices", folderName); - var tempDirectory = new DirectoryInfo(tempFolder); + var directoryInfo = new DirectoryInfo(Path.Combine(Path.GetTempPath(), fileId)); - var directory = new MongoDirectory(bucket, folderName, tempDirectory); + if (directoryInfo.Exists) + { + directoryInfo.Delete(true); + } - return Task.FromResult(directory); + directoryInfo.Create(); + + try + { + using (var stream = await bucket.OpenDownloadStreamAsync(fileId)) + { + using (var zipArchive = new ZipArchive(stream, ZipArchiveMode.Read, true)) + { + foreach (var entry in zipArchive.Entries) + { + var file = new FileInfo(Path.Combine(directoryInfo.FullName, entry.Name)); + + using (var entryStream = entry.Open()) + { + using (var fileStream = file.OpenWrite()) + { + await entryStream.CopyToAsync(fileStream); + } + } + } + } + } + } + catch (GridFSFileNotFoundException) + { + } + + var directory = FSDirectory.Open(directoryInfo); + + return directory; } - public Task ClearAsync() + public async Task WriteAsync(LuceneDirectory directory, SnapshotDeletionPolicy snapshotter) { - return bucket.DropAsync(); + var directoryInfo = ((FSDirectory)directory).Directory; + + var commit = snapshotter.Snapshot(); + try + { + var fileId = directoryInfo.Name; + + try + { + await bucket.DeleteAsync(fileId); + } + catch (GridFSFileNotFoundException) + { + } + + using (var stream = await bucket.OpenUploadStreamAsync(fileId, fileId)) + { + using (var zipArchive = new ZipArchive(stream, ZipArchiveMode.Create, true)) + { + foreach (var fileName in commit.FileNames) + { + var file = new FileInfo(Path.Combine(directoryInfo.FullName, fileName)); + + using (var fileStream = file.OpenRead()) + { + var entry = zipArchive.CreateEntry(fileStream.Name); + + using (var entryStream = entry.Open()) + { + await fileStream.CopyToAsync(entryStream); + } + } + } + } + } + } + finally + { + snapshotter.Release(commit); + } } - public Task WriteAsync(LuceneDirectory directory, SnapshotDeletionPolicy snapshotter) + public Task ClearAsync() { - return Task.CompletedTask; + return bucket.DropAsync(); } } } diff --git a/backend/src/Squidex.Domain.Apps.Entities/Contents/Text/Lucene/LuceneTextIndexGrain.cs b/backend/src/Squidex.Domain.Apps.Entities/Contents/Text/Lucene/LuceneTextIndexGrain.cs index 97f44b04d..ccc44d582 100644 --- a/backend/src/Squidex.Domain.Apps.Entities/Contents/Text/Lucene/LuceneTextIndexGrain.cs +++ b/backend/src/Squidex.Domain.Apps.Entities/Contents/Text/Lucene/LuceneTextIndexGrain.cs @@ -214,7 +214,16 @@ namespace Squidex.Domain.Apps.Entities.Contents.Text.Lucene index.Writer.DeleteDocuments(new Term(MetaId, delete.DocId)); break; case UpdateIndexEntry update: - index.Writer.UpdateBinaryDocValue(new Term(MetaId, update.DocId), MetaFor, GetValue(update.ServeAll, update.ServePublished)); + try + { + var values = GetValue(update.ServeAll, update.ServePublished); + + index.Writer.UpdateBinaryDocValue(new Term(MetaId, update.DocId), MetaFor, values); + } + catch (ArgumentException) + { + } + break; case UpsertIndexEntry upsert: { diff --git a/backend/src/Squidex.Domain.Apps.Entities/Contents/Text/Lucene/Storage/AssetIndexStorage.cs b/backend/src/Squidex.Domain.Apps.Entities/Contents/Text/Lucene/Storage/AssetIndexStorage.cs index ce4871a18..4ff2d87e2 100644 --- a/backend/src/Squidex.Domain.Apps.Entities/Contents/Text/Lucene/Storage/AssetIndexStorage.cs +++ b/backend/src/Squidex.Domain.Apps.Entities/Contents/Text/Lucene/Storage/AssetIndexStorage.cs @@ -20,7 +20,6 @@ namespace Squidex.Domain.Apps.Entities.Contents.Text.Lucene.Storage public sealed class AssetIndexStorage : IIndexStorage { private const string ArchiveFile = "Archive.zip"; - private const string LockFile = "write.lock"; private readonly IAssetStore assetStore; public AssetIndexStorage(IAssetStore assetStore) @@ -64,16 +63,8 @@ namespace Squidex.Domain.Apps.Entities.Contents.Text.Lucene.Storage return directory; } - public Task ClearAsync() - { - return Task.CompletedTask; - } - public async Task WriteAsync(LuceneDirectory directory, SnapshotDeletionPolicy snapshotter) { - Guard.NotNull(directory); - Guard.NotNull(snapshotter); - var directoryInfo = ((FSDirectory)directory).Directory; var commit = snapshotter.Snapshot(); @@ -87,18 +78,7 @@ namespace Squidex.Domain.Apps.Entities.Contents.Text.Lucene.Storage { var file = new FileInfo(Path.Combine(directoryInfo.FullName, fileName)); - try - { - if (!file.Name.Equals(ArchiveFile, StringComparison.OrdinalIgnoreCase) && - !file.Name.Equals(LockFile, StringComparison.OrdinalIgnoreCase)) - { - zipArchive.CreateEntryFromFile(file.FullName, file.Name); - } - } - catch (IOException) - { - continue; - } + zipArchive.CreateEntryFromFile(file.FullName, file.Name); } } @@ -125,5 +105,10 @@ namespace Squidex.Domain.Apps.Entities.Contents.Text.Lucene.Storage 4096, FileOptions.DeleteOnClose); } + + public Task ClearAsync() + { + return Task.CompletedTask; + } } } diff --git a/backend/src/Squidex.Domain.Apps.Entities/Contents/Text/Lucene/Storage/FileIndexStorage.cs b/backend/src/Squidex.Domain.Apps.Entities/Contents/Text/Lucene/Storage/FileIndexStorage.cs index 31482f852..b90926c32 100644 --- a/backend/src/Squidex.Domain.Apps.Entities/Contents/Text/Lucene/Storage/FileIndexStorage.cs +++ b/backend/src/Squidex.Domain.Apps.Entities/Contents/Text/Lucene/Storage/FileIndexStorage.cs @@ -24,12 +24,12 @@ namespace Squidex.Domain.Apps.Entities.Contents.Text.Lucene.Storage return Task.FromResult(FSDirectory.Open(folderPath)); } - public Task ClearAsync() + public Task WriteAsync(LuceneDirectory directory, SnapshotDeletionPolicy snapshotter) { return Task.CompletedTask; } - public Task WriteAsync(LuceneDirectory directory, SnapshotDeletionPolicy snapshotter) + public Task ClearAsync() { return Task.CompletedTask; } diff --git a/backend/src/Squidex.Domain.Apps.Entities/Contents/Text/Lucene/Storage/IIndexStorage.cs b/backend/src/Squidex.Domain.Apps.Entities/Contents/Text/Lucene/Storage/IIndexStorage.cs index edfab22bc..37036c469 100644 --- a/backend/src/Squidex.Domain.Apps.Entities/Contents/Text/Lucene/Storage/IIndexStorage.cs +++ b/backend/src/Squidex.Domain.Apps.Entities/Contents/Text/Lucene/Storage/IIndexStorage.cs @@ -16,8 +16,8 @@ namespace Squidex.Domain.Apps.Entities.Contents.Text.Lucene { Task CreateDirectoryAsync(Guid ownerId); - Task ClearAsync(); - Task WriteAsync(Directory directory, SnapshotDeletionPolicy snapshotter); + + Task ClearAsync(); } } diff --git a/backend/src/Squidex/Config/Domain/SubscriptionServices.cs b/backend/src/Squidex/Config/Domain/SubscriptionServices.cs index 949faf170..5f4a0717e 100644 --- a/backend/src/Squidex/Config/Domain/SubscriptionServices.cs +++ b/backend/src/Squidex/Config/Domain/SubscriptionServices.cs @@ -28,7 +28,7 @@ namespace Squidex.Config.Domain .AsOptional(); services.AddSingletonAs() - .AsOptional(); + .AsSelf(); services.AddSingletonAs() .AsOptional(); diff --git a/backend/tests/Squidex.Domain.Apps.Entities.Tests/Contents/Text/TextIndexerBenchmark.cs b/backend/tests/Squidex.Domain.Apps.Entities.Tests/Contents/Text/TextIndexerBenchmark.cs deleted file mode 100644 index 30f061912..000000000 --- a/backend/tests/Squidex.Domain.Apps.Entities.Tests/Contents/Text/TextIndexerBenchmark.cs +++ /dev/null @@ -1,117 +0,0 @@ -// ========================================================================== -// Squidex Headless CMS -// ========================================================================== -// Copyright (c) Squidex UG (haftungsbeschraenkt) -// All rights reserved. Licensed under the MIT license. -// ========================================================================== - -using System; -using System.Collections.Generic; -using System.Threading.Tasks; -using FakeItEasy; -using Orleans.Concurrency; -using Squidex.Domain.Apps.Entities.Contents.Text.Lucene; -using Squidex.Infrastructure; -using Squidex.Infrastructure.Log; -using Xunit; - -#pragma warning disable xUnit1004 // Test methods should not be skipped - -namespace Squidex.Domain.Apps.Entities.Contents.Text -{ - [Trait("Category", "Dependencies")] - public class TextIndexerBenchmark - { - private const int Size = 200; - private readonly NamedId appId = NamedId.Of(Guid.NewGuid(), "my-app"); - private readonly NamedId schemaId = NamedId.Of(Guid.NewGuid(), "my-schema"); - - [Fact] - public async Task Should_index_and_search_in_temp_folder() - { - await IndexAndSearchAsync(TestStorages.TempFolder()); - } - - [Fact] - public async Task Should_index_and_search_in_assets() - { - await IndexAndSearchAsync(TestStorages.Assets()); - } - - [Fact] - public async Task Should_index_and_search_in_mongoDB() - { - await IndexAndSearchAsync(TestStorages.MongoDB()); - } - - private async Task IndexAndSearchAsync(IIndexStorage storage) - { - var factory = new IndexManager(storage, A.Fake()); - - var grain = new LuceneTextIndexGrain(factory); - - await grain.ActivateAsync(appId.Id); - - var elapsed1 = await IndexAsync(grain); - var elapsed2 = await SearchAsync(grain); - var elapsed3 = await SearchAsync(grain); - - Assert.Equal(new long[0], new[] { elapsed1, elapsed2, elapsed3 }); - } - - private async Task IndexAsync(LuceneTextIndexGrain grain) - { - var text = new Dictionary - { - ["iv"] = "Hello World" - }; - - var ids = new Guid[Size]; - - for (var i = 0; i < ids.Length; i++) - { - ids[i] = Guid.NewGuid(); - } - - var watch = ValueStopwatch.StartNew(); - - foreach (var id in ids) - { - var commands = new IndexCommand[] - { - new UpsertIndexEntry - { - ContentId = id, - DocId = id.ToString(), - ServeAll = true, - ServePublished = true, - Texts = text - } - }; - - await grain.IndexAsync(schemaId, commands.AsImmutable()); - } - - return watch.Stop(); - } - - private async Task SearchAsync(LuceneTextIndexGrain grain) - { - var searchContext = new SearchContext - { - Languages = new HashSet() - }; - - var watch = ValueStopwatch.StartNew(); - - for (var i = 0; i < Size; i++) - { - var result = await grain.SearchAsync("Hello", default, searchContext); - - Assert.NotEmpty(result); - } - - return watch.Stop(); - } - } -} diff --git a/backend/tools/Benchmarks/Benchmarks.csproj b/backend/tools/Benchmarks/Benchmarks.csproj new file mode 100644 index 000000000..c973d6821 --- /dev/null +++ b/backend/tools/Benchmarks/Benchmarks.csproj @@ -0,0 +1,24 @@ + + + Exe + netcoreapp3.1 + + + + + + + + + + ..\..\Squidex.ruleset + + + + + + + + + + diff --git a/backend/tools/Benchmarks/IndexStorages.cs b/backend/tools/Benchmarks/IndexStorages.cs new file mode 100644 index 000000000..169ba1bad --- /dev/null +++ b/backend/tools/Benchmarks/IndexStorages.cs @@ -0,0 +1,49 @@ +// ========================================================================== +// Squidex Headless CMS +// ========================================================================== +// Copyright (c) Squidex UG (haftungsbeschraenkt) +// All rights reserved. Licensed under the MIT license. +// ========================================================================== + +using System; +using MongoDB.Driver; +using MongoDB.Driver.GridFS; +using Squidex.Domain.Apps.Entities.Contents.Text.Lucene; +using Squidex.Domain.Apps.Entities.Contents.Text.Lucene.Storage; +using Squidex.Domain.Apps.Entities.MongoDb.FullText; +using Squidex.Infrastructure.Assets; + +namespace Benchmarks +{ + public static class IndexStorages + { + public static IIndexStorage Assets() + { + var storage = new AssetIndexStorage(new MemoryAssetStore()); + + return storage; + } + + public static IIndexStorage TempFolder() + { + var storage = new FileIndexStorage(); + + return storage; + } + + public static IIndexStorage MongoDB() + { + var mongoClient = new MongoClient("mongodb://localhost"); + var mongoDatabase = mongoClient.GetDatabase("FullText"); + + var mongoBucket = new GridFSBucket(mongoDatabase, new GridFSBucketOptions + { + BucketName = $"bucket_{DateTime.UtcNow.Ticks}" + }); + + var storage = new MongoIndexStorage(mongoBucket); + + return storage; + } + } +} diff --git a/backend/tools/Benchmarks/IndexingBenchmarks.cs b/backend/tools/Benchmarks/IndexingBenchmarks.cs new file mode 100644 index 000000000..ed4289779 --- /dev/null +++ b/backend/tools/Benchmarks/IndexingBenchmarks.cs @@ -0,0 +1,103 @@ +// ========================================================================== +// Squidex Headless CMS +// ========================================================================== +// Copyright (c) Squidex UG (haftungsbeschraenkt) +// All rights reserved. Licensed under the MIT license. +// ========================================================================== + +using System; +using System.Collections.Generic; +using System.Threading.Tasks; +using BenchmarkDotNet.Attributes; +using Benchmarks.Utils; +using LoremNET; +using Orleans.Concurrency; +using Squidex.Domain.Apps.Entities.Contents.Text; +using Squidex.Domain.Apps.Entities.Contents.Text.Lucene; +using Squidex.Infrastructure; + +namespace Benchmarks +{ + [ShortRunJob] + [StopOnFirstError] + [RPlotExporter] + public class IndexingBenchmarks + { + private readonly IIndexStorage storageAssets = IndexStorages.Assets(); + private readonly IIndexStorage storageTempFolder = IndexStorages.TempFolder(); + private readonly IIndexStorage storageMongoDB = IndexStorages.MongoDB(); + private readonly Dictionary texts; + + public IndexingBenchmarks() + { + texts = new Dictionary + { + ["iv"] = Lorem.Paragraph(10, 10) + }; + } + + [Params(1000)] + public int N { get; set; } + + [Params(10)] + public int M { get; set; } + + [Benchmark(Baseline = true)] + public async Task Index_TempFolder() + { + await IndexAndSearchAsync(storageTempFolder); + } + + [Benchmark] + public async Task Index_Assets() + { + await IndexAndSearchAsync(storageAssets); + } + + [Benchmark] + public async Task Index_MongoDB() + { + await IndexAndSearchAsync(storageMongoDB); + } + + private async Task IndexAndSearchAsync(IIndexStorage storage) + { + var schemaId = NamedId.Of(Guid.NewGuid(), "my-schema"); + + var factory = new IndexManager(storage, new NoopLog()); + + var grain = new LuceneTextIndexGrain(factory); + + await grain.ActivateAsync(Guid.NewGuid()); + + for (var i = 0; i < M; i++) + { + var ids = new Guid[N]; + + for (var j = 0; j < ids.Length; j++) + { + ids[j] = Guid.NewGuid(); + } + + foreach (var id in ids) + { + var commands = new IndexCommand[] + { + new UpsertIndexEntry + { + ContentId = id, + DocId = id.ToString(), + ServeAll = true, + ServePublished = true, + Texts = texts + } + }; + + await grain.IndexAsync(schemaId, commands.AsImmutable()); + } + + await grain.CommitAsync(); + } + } + } +} diff --git a/backend/tools/Benchmarks/Program.cs b/backend/tools/Benchmarks/Program.cs new file mode 100644 index 000000000..9527b6673 --- /dev/null +++ b/backend/tools/Benchmarks/Program.cs @@ -0,0 +1,19 @@ +// ========================================================================== +// Squidex Headless CMS +// ========================================================================== +// Copyright (c) Squidex UG (haftungsbeschraenkt) +// All rights reserved. Licensed under the MIT license. +// ========================================================================== + +using BenchmarkDotNet.Running; + +namespace Benchmarks +{ + public static class Program + { + public static void Main() + { + BenchmarkRunner.Run(); + } + } +} diff --git a/backend/tools/Benchmarks/Utils/NoopLog.cs b/backend/tools/Benchmarks/Utils/NoopLog.cs new file mode 100644 index 000000000..2e59e1646 --- /dev/null +++ b/backend/tools/Benchmarks/Utils/NoopLog.cs @@ -0,0 +1,28 @@ +// ========================================================================== +// Squidex Headless CMS +// ========================================================================== +// Copyright (c) Squidex UG (haftungsbeschraenkt) +// All rights reserved. Licensed under the MIT license. +// ========================================================================== + +using System; +using Squidex.Infrastructure.Log; + +namespace Benchmarks.Utils +{ + public sealed class NoopLog : ISemanticLog + { + public ISemanticLog CreateScope(Action objectWriter) + { + return this; + } + + public void Log(SemanticLogLevel logLevel, T context, Exception exception, LogFormatter action) + { + } + + public void Log(SemanticLogLevel logLevel, Exception exception, LogFormatter action) + { + } + } +}