Browse Source

Index tests (#494)

* Indexing tests

* Benchmark indexing performance

* Cleanup
pull/495/head
Sebastian Stehle 6 years ago
committed by GitHub
parent
commit
1f13ed43af
No known key found for this signature in database GPG Key ID: 4AEE18F83AFDEB23
  1. 19
      backend/Squidex.sln
  2. 150
      backend/src/Squidex.Domain.Apps.Entities.MongoDb/FullText/MongoDirectory.cs
  3. 105
      backend/src/Squidex.Domain.Apps.Entities.MongoDb/FullText/MongoIndexInput.cs
  4. 114
      backend/src/Squidex.Domain.Apps.Entities.MongoDb/FullText/MongoIndexOutput.cs
  5. 92
      backend/src/Squidex.Domain.Apps.Entities.MongoDb/FullText/MongoIndexStorage.cs
  6. 11
      backend/src/Squidex.Domain.Apps.Entities/Contents/Text/Lucene/LuceneTextIndexGrain.cs
  7. 27
      backend/src/Squidex.Domain.Apps.Entities/Contents/Text/Lucene/Storage/AssetIndexStorage.cs
  8. 4
      backend/src/Squidex.Domain.Apps.Entities/Contents/Text/Lucene/Storage/FileIndexStorage.cs
  9. 4
      backend/src/Squidex.Domain.Apps.Entities/Contents/Text/Lucene/Storage/IIndexStorage.cs
  10. 2
      backend/src/Squidex/Config/Domain/SubscriptionServices.cs
  11. 117
      backend/tests/Squidex.Domain.Apps.Entities.Tests/Contents/Text/TextIndexerBenchmark.cs
  12. 24
      backend/tools/Benchmarks/Benchmarks.csproj
  13. 49
      backend/tools/Benchmarks/IndexStorages.cs
  14. 103
      backend/tools/Benchmarks/IndexingBenchmarks.cs
  15. 19
      backend/tools/Benchmarks/Program.cs
  16. 28
      backend/tools/Benchmarks/Utils/NoopLog.cs

19
backend/Squidex.sln

@ -22,7 +22,7 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Squidex.Infrastructure.Rabb
EndProject
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Squidex.Infrastructure.GoogleCloud", "src\Squidex.Infrastructure.GoogleCloud\Squidex.Infrastructure.GoogleCloud.csproj", "{945871B1-77B8-43FB-B53C-27CF385AB756}"
EndProject
Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "migrations", "migrations", "{94207AA6-4923-4183-A558-E0F8196B8CA3}"
Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "tools", "tools", "{94207AA6-4923-4183-A558-E0F8196B8CA3}"
EndProject
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Migrate_00", "tools\Migrate_00\Migrate_00.csproj", "{B51126A8-0D75-4A79-867D-10724EC6AC84}"
EndProject
@ -65,7 +65,9 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "shared", "shared", "{7EDE8C
EndProject
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Squidex.Web", "src\Squidex.Web\Squidex.Web.csproj", "{5B2D251F-46E3-486A-AE16-E3FE06B559ED}"
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Squidex.Infrastructure.Amazon", "src\Squidex.Infrastructure.Amazon\Squidex.Infrastructure.Amazon.csproj", "{32DA4B56-7EFA-4E34-A29D-30E00579A894}"
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Squidex.Infrastructure.Amazon", "src\Squidex.Infrastructure.Amazon\Squidex.Infrastructure.Amazon.csproj", "{32DA4B56-7EFA-4E34-A29D-30E00579A894}"
EndProject
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Benchmarks", "tools\Benchmarks\Benchmarks.csproj", "{F36EF843-BDDD-45A8-B9C6-360001161AAA}"
EndProject
Global
GlobalSection(SolutionConfigurationPlatforms) = preSolution
@ -353,6 +355,18 @@ Global
{32DA4B56-7EFA-4E34-A29D-30E00579A894}.Release|x64.Build.0 = Release|Any CPU
{32DA4B56-7EFA-4E34-A29D-30E00579A894}.Release|x86.ActiveCfg = Release|Any CPU
{32DA4B56-7EFA-4E34-A29D-30E00579A894}.Release|x86.Build.0 = Release|Any CPU
{F36EF843-BDDD-45A8-B9C6-360001161AAA}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{F36EF843-BDDD-45A8-B9C6-360001161AAA}.Debug|Any CPU.Build.0 = Debug|Any CPU
{F36EF843-BDDD-45A8-B9C6-360001161AAA}.Debug|x64.ActiveCfg = Debug|Any CPU
{F36EF843-BDDD-45A8-B9C6-360001161AAA}.Debug|x64.Build.0 = Debug|Any CPU
{F36EF843-BDDD-45A8-B9C6-360001161AAA}.Debug|x86.ActiveCfg = Debug|Any CPU
{F36EF843-BDDD-45A8-B9C6-360001161AAA}.Debug|x86.Build.0 = Debug|Any CPU
{F36EF843-BDDD-45A8-B9C6-360001161AAA}.Release|Any CPU.ActiveCfg = Release|Any CPU
{F36EF843-BDDD-45A8-B9C6-360001161AAA}.Release|Any CPU.Build.0 = Release|Any CPU
{F36EF843-BDDD-45A8-B9C6-360001161AAA}.Release|x64.ActiveCfg = Release|Any CPU
{F36EF843-BDDD-45A8-B9C6-360001161AAA}.Release|x64.Build.0 = Release|Any CPU
{F36EF843-BDDD-45A8-B9C6-360001161AAA}.Release|x86.ActiveCfg = Release|Any CPU
{F36EF843-BDDD-45A8-B9C6-360001161AAA}.Release|x86.Build.0 = Release|Any CPU
EndGlobalSection
GlobalSection(SolutionProperties) = preSolution
HideSolutionNode = FALSE
@ -382,6 +396,7 @@ Global
{F3C41B82-6A67-409A-B7FE-54543EE4F38B} = {FB8BC3A2-2010-4C3C-A87D-D4A98C05EE52}
{5B2D251F-46E3-486A-AE16-E3FE06B559ED} = {7EDE8CF1-B1E4-4005-B154-834B944E0D7A}
{32DA4B56-7EFA-4E34-A29D-30E00579A894} = {8CF53B92-5EB1-461D-98F8-70DA9B603FBF}
{F36EF843-BDDD-45A8-B9C6-360001161AAA} = {94207AA6-4923-4183-A558-E0F8196B8CA3}
EndGlobalSection
GlobalSection(ExtensibilityGlobals) = postSolution
SolutionGuid = {02F2E872-3141-44F5-BD6A-33CD84E9FE08}

150
backend/src/Squidex.Domain.Apps.Entities.MongoDb/FullText/MongoDirectory.cs

@ -1,150 +0,0 @@
// ==========================================================================
// Squidex Headless CMS
// ==========================================================================
// Copyright (c) Squidex UG (haftungsbeschraenkt)
// All rights reserved. Licensed under the MIT license.
// ==========================================================================
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using Lucene.Net.Store;
using MongoDB.Bson;
using MongoDB.Driver;
using MongoDB.Driver.GridFS;
using LuceneDirectory = Lucene.Net.Store.Directory;
namespace Squidex.Domain.Apps.Entities.MongoDb.FullText
{
public sealed class MongoDirectory : BaseDirectory
{
private readonly IGridFSBucket<string> bucket;
private readonly string directory;
private readonly DirectoryInfo cacheDirectoryInfo;
private readonly LuceneDirectory cacheDirectory;
private bool isDisposed;
public LuceneDirectory CacheDirectory
{
get { return cacheDirectory; }
}
public DirectoryInfo CacheDirectoryInfo
{
get { return cacheDirectoryInfo; }
}
public IGridFSBucket<string> Bucket
{
get { return bucket; }
}
public MongoDirectory(IGridFSBucket<string> bucket, string directory, DirectoryInfo cacheDirectoryInfo)
{
this.bucket = bucket;
this.directory = directory;
this.cacheDirectoryInfo = cacheDirectoryInfo;
cacheDirectoryInfo.Create();
cacheDirectory = FSDirectory.Open(cacheDirectoryInfo);
SetLockFactory(new NativeFSLockFactory(cacheDirectoryInfo));
}
protected override void Dispose(bool disposing)
{
if (disposing)
{
isDisposed = true;
cacheDirectory.Dispose();
}
}
public override string GetLockID()
{
return cacheDirectory.GetLockID();
}
public override IndexOutput CreateOutput(string name, IOContext context)
{
return new MongoIndexOutput(this, context, name);
}
public override IndexInput OpenInput(string name, IOContext context)
{
return new MongoIndexInput(this, context, name);
}
public override void DeleteFile(string name)
{
EnsureNotDisposed();
var fullName = GetFullName(name);
try
{
Bucket.Delete(fullName);
}
catch (GridFSFileNotFoundException)
{
}
}
public override long FileLength(string name)
{
EnsureNotDisposed();
var file = FindFile(name) ?? throw new FileNotFoundException(null, GetFullName(name));
return file.Length;
}
public override string[] ListAll()
{
EnsureNotDisposed();
var files = Bucket.Find(Builders<GridFSFileInfo<string>>.Filter.Regex(x => x.Id, new BsonRegularExpression($"^{directory}/"))).ToList();
return files.Select(x => x.Filename).ToArray();
}
public GridFSFileInfo<string>? FindFile(string name)
{
var fullName = GetFullName(name);
return Bucket.Find(Builders<GridFSFileInfo<string>>.Filter.Eq(x => x.Id, fullName)).FirstOrDefault();
}
public override void Sync(ICollection<string> names)
{
}
[Obsolete]
public override bool FileExists(string name)
{
throw new NotSupportedException();
}
public string GetFullName(string name)
{
return $"{directory}/{name}";
}
public string GetFullPath(string name)
{
return Path.Combine(cacheDirectoryInfo.FullName, name);
}
private void EnsureNotDisposed()
{
if (isDisposed)
{
throw new ObjectDisposedException(GetType().FullName);
}
}
}
}

105
backend/src/Squidex.Domain.Apps.Entities.MongoDb/FullText/MongoIndexInput.cs

@ -1,105 +0,0 @@
// ==========================================================================
// Squidex Headless CMS
// ==========================================================================
// Copyright (c) Squidex UG (haftungsbeschraenkt)
// All rights reserved. Licensed under the MIT license.
// ==========================================================================
using System.IO;
using Lucene.Net.Store;
using MongoDB.Driver.GridFS;
namespace Squidex.Domain.Apps.Entities.MongoDb.FullText
{
public sealed class MongoIndexInput : IndexInput
{
private readonly IndexInput cacheInput;
private readonly MongoDirectory indexDirectory;
private readonly IOContext context;
private readonly string indexFileName;
public override long Length
{
get { return cacheInput.Length; }
}
public MongoIndexInput(MongoDirectory indexDirectory, IOContext context, string indexFileName)
: base(indexDirectory.GetFullName(indexFileName))
{
this.indexDirectory = indexDirectory;
this.indexFileName = indexFileName;
this.context = context;
try
{
var file = indexDirectory.FindFile(indexFileName);
if (file != null)
{
var fileInfo = new FileInfo(indexDirectory.GetFullPath(indexFileName));
var writtenTime = file.Metadata["WrittenTime"].ToUniversalTime();
if (!fileInfo.Exists || fileInfo.LastWriteTimeUtc < writtenTime)
{
using (var fs = new FileStream(fileInfo.FullName, FileMode.Create, FileAccess.Write))
{
var fullName = indexDirectory.GetFullName(indexFileName);
indexDirectory.Bucket.DownloadToStream(fullName, fs);
}
}
}
}
catch (GridFSFileNotFoundException)
{
throw new FileNotFoundException();
}
cacheInput = indexDirectory.CacheDirectory.OpenInput(indexFileName, context);
}
public MongoIndexInput(MongoIndexInput source)
: base("clone")
{
cacheInput = (IndexInput)source.cacheInput.Clone();
context = source.context;
indexDirectory = source.indexDirectory;
indexFileName = source.indexFileName;
}
protected override void Dispose(bool disposing)
{
if (disposing)
{
cacheInput.Dispose();
}
}
public override long GetFilePointer()
{
return cacheInput.GetFilePointer();
}
public override byte ReadByte()
{
return cacheInput.ReadByte();
}
public override void ReadBytes(byte[] b, int offset, int len)
{
cacheInput.ReadBytes(b, offset, len);
}
public override void Seek(long pos)
{
cacheInput.Seek(pos);
}
public override object Clone()
{
return new MongoIndexInput(indexDirectory, context, indexFileName);
}
}
}

114
backend/src/Squidex.Domain.Apps.Entities.MongoDb/FullText/MongoIndexOutput.cs

@ -1,114 +0,0 @@
// ==========================================================================
// Squidex Headless CMS
// ==========================================================================
// Copyright (c) Squidex UG (haftungsbeschraenkt)
// All rights reserved. Licensed under the MIT license.
// ==========================================================================
using System;
using System.IO;
using System.Linq;
using Lucene.Net.Store;
using MongoDB.Bson;
using MongoDB.Driver;
using MongoDB.Driver.GridFS;
namespace Squidex.Domain.Apps.Entities.MongoDb.FullText
{
public sealed class MongoIndexOutput : IndexOutput
{
private readonly IndexOutput cacheOutput;
private readonly MongoDirectory indexDirectory;
private readonly string indexFileName;
private bool isFlushed;
private bool isWritten;
public override long Length
{
get { return cacheOutput.Length; }
}
public override long Checksum
{
get { return cacheOutput.Checksum; }
}
public MongoIndexOutput(MongoDirectory indexDirectory, IOContext context, string indexFileName)
{
this.indexDirectory = indexDirectory;
this.indexFileName = indexFileName;
cacheOutput = indexDirectory.CacheDirectory.CreateOutput(indexFileName, context);
}
protected override void Dispose(bool disposing)
{
if (disposing)
{
Flush();
cacheOutput.Dispose();
if (isWritten && isFlushed)
{
var fileInfo = new FileInfo(indexDirectory.GetFullPath(indexFileName));
using (var fs = new FileStream(indexDirectory.GetFullPath(indexFileName), FileMode.Open, FileAccess.Read))
{
var fullName = indexDirectory.GetFullName(indexFileName);
var options = new GridFSUploadOptions
{
Metadata = new BsonDocument
{
["WrittenTime"] = fileInfo.LastWriteTimeUtc
}
};
try
{
indexDirectory.Bucket.UploadFromStream(fullName, indexFileName, fs, options);
}
catch (MongoBulkWriteException ex) when (ex.WriteErrors.Any(x => x.Code == 11000))
{
indexDirectory.Bucket.Delete(fullName);
indexDirectory.Bucket.UploadFromStream(fullName, indexFileName, fs, options);
}
}
}
}
}
public override long GetFilePointer()
{
return cacheOutput.GetFilePointer();
}
public override void Flush()
{
cacheOutput.Flush();
isFlushed = true;
}
public override void WriteByte(byte b)
{
cacheOutput.WriteByte(b);
isWritten = true;
}
public override void WriteBytes(byte[] b, int offset, int length)
{
cacheOutput.WriteBytes(b, offset, length);
isWritten = true;
}
[Obsolete]
public override void Seek(long pos)
{
cacheOutput.Seek(pos);
}
}
}

92
backend/src/Squidex.Domain.Apps.Entities.MongoDb/FullText/MongoIndexStorage.cs

@ -7,8 +7,10 @@
using System;
using System.IO;
using System.IO.Compression;
using System.Threading.Tasks;
using Lucene.Net.Index;
using Lucene.Net.Store;
using MongoDB.Driver.GridFS;
using Squidex.Domain.Apps.Entities.Contents.Text.Lucene;
using Squidex.Infrastructure;
@ -27,26 +29,96 @@ namespace Squidex.Domain.Apps.Entities.MongoDb.FullText
this.bucket = bucket;
}
public Task<LuceneDirectory> CreateDirectoryAsync(Guid ownerId)
public async Task<LuceneDirectory> CreateDirectoryAsync(Guid ownerId)
{
var folderName = ownerId.ToString();
var fileId = $"index_{ownerId}";
var tempFolder = Path.Combine(Path.GetTempPath(), "Indices", folderName);
var tempDirectory = new DirectoryInfo(tempFolder);
var directoryInfo = new DirectoryInfo(Path.Combine(Path.GetTempPath(), fileId));
var directory = new MongoDirectory(bucket, folderName, tempDirectory);
if (directoryInfo.Exists)
{
directoryInfo.Delete(true);
}
return Task.FromResult<LuceneDirectory>(directory);
directoryInfo.Create();
try
{
using (var stream = await bucket.OpenDownloadStreamAsync(fileId))
{
using (var zipArchive = new ZipArchive(stream, ZipArchiveMode.Read, true))
{
foreach (var entry in zipArchive.Entries)
{
var file = new FileInfo(Path.Combine(directoryInfo.FullName, entry.Name));
using (var entryStream = entry.Open())
{
using (var fileStream = file.OpenWrite())
{
await entryStream.CopyToAsync(fileStream);
}
}
}
}
}
}
catch (GridFSFileNotFoundException)
{
}
var directory = FSDirectory.Open(directoryInfo);
return directory;
}
public Task ClearAsync()
public async Task WriteAsync(LuceneDirectory directory, SnapshotDeletionPolicy snapshotter)
{
return bucket.DropAsync();
var directoryInfo = ((FSDirectory)directory).Directory;
var commit = snapshotter.Snapshot();
try
{
var fileId = directoryInfo.Name;
try
{
await bucket.DeleteAsync(fileId);
}
catch (GridFSFileNotFoundException)
{
}
using (var stream = await bucket.OpenUploadStreamAsync(fileId, fileId))
{
using (var zipArchive = new ZipArchive(stream, ZipArchiveMode.Create, true))
{
foreach (var fileName in commit.FileNames)
{
var file = new FileInfo(Path.Combine(directoryInfo.FullName, fileName));
using (var fileStream = file.OpenRead())
{
var entry = zipArchive.CreateEntry(fileStream.Name);
using (var entryStream = entry.Open())
{
await fileStream.CopyToAsync(entryStream);
}
}
}
}
}
}
finally
{
snapshotter.Release(commit);
}
}
public Task WriteAsync(LuceneDirectory directory, SnapshotDeletionPolicy snapshotter)
public Task ClearAsync()
{
return Task.CompletedTask;
return bucket.DropAsync();
}
}
}

11
backend/src/Squidex.Domain.Apps.Entities/Contents/Text/Lucene/LuceneTextIndexGrain.cs

@ -214,7 +214,16 @@ namespace Squidex.Domain.Apps.Entities.Contents.Text.Lucene
index.Writer.DeleteDocuments(new Term(MetaId, delete.DocId));
break;
case UpdateIndexEntry update:
index.Writer.UpdateBinaryDocValue(new Term(MetaId, update.DocId), MetaFor, GetValue(update.ServeAll, update.ServePublished));
try
{
var values = GetValue(update.ServeAll, update.ServePublished);
index.Writer.UpdateBinaryDocValue(new Term(MetaId, update.DocId), MetaFor, values);
}
catch (ArgumentException)
{
}
break;
case UpsertIndexEntry upsert:
{

27
backend/src/Squidex.Domain.Apps.Entities/Contents/Text/Lucene/Storage/AssetIndexStorage.cs

@ -20,7 +20,6 @@ namespace Squidex.Domain.Apps.Entities.Contents.Text.Lucene.Storage
public sealed class AssetIndexStorage : IIndexStorage
{
private const string ArchiveFile = "Archive.zip";
private const string LockFile = "write.lock";
private readonly IAssetStore assetStore;
public AssetIndexStorage(IAssetStore assetStore)
@ -64,16 +63,8 @@ namespace Squidex.Domain.Apps.Entities.Contents.Text.Lucene.Storage
return directory;
}
public Task ClearAsync()
{
return Task.CompletedTask;
}
public async Task WriteAsync(LuceneDirectory directory, SnapshotDeletionPolicy snapshotter)
{
Guard.NotNull(directory);
Guard.NotNull(snapshotter);
var directoryInfo = ((FSDirectory)directory).Directory;
var commit = snapshotter.Snapshot();
@ -87,18 +78,7 @@ namespace Squidex.Domain.Apps.Entities.Contents.Text.Lucene.Storage
{
var file = new FileInfo(Path.Combine(directoryInfo.FullName, fileName));
try
{
if (!file.Name.Equals(ArchiveFile, StringComparison.OrdinalIgnoreCase) &&
!file.Name.Equals(LockFile, StringComparison.OrdinalIgnoreCase))
{
zipArchive.CreateEntryFromFile(file.FullName, file.Name);
}
}
catch (IOException)
{
continue;
}
zipArchive.CreateEntryFromFile(file.FullName, file.Name);
}
}
@ -125,5 +105,10 @@ namespace Squidex.Domain.Apps.Entities.Contents.Text.Lucene.Storage
4096,
FileOptions.DeleteOnClose);
}
public Task ClearAsync()
{
return Task.CompletedTask;
}
}
}

4
backend/src/Squidex.Domain.Apps.Entities/Contents/Text/Lucene/Storage/FileIndexStorage.cs

@ -24,12 +24,12 @@ namespace Squidex.Domain.Apps.Entities.Contents.Text.Lucene.Storage
return Task.FromResult<LuceneDirectory>(FSDirectory.Open(folderPath));
}
public Task ClearAsync()
public Task WriteAsync(LuceneDirectory directory, SnapshotDeletionPolicy snapshotter)
{
return Task.CompletedTask;
}
public Task WriteAsync(LuceneDirectory directory, SnapshotDeletionPolicy snapshotter)
public Task ClearAsync()
{
return Task.CompletedTask;
}

4
backend/src/Squidex.Domain.Apps.Entities/Contents/Text/Lucene/Storage/IIndexStorage.cs

@ -16,8 +16,8 @@ namespace Squidex.Domain.Apps.Entities.Contents.Text.Lucene
{
Task<Directory> CreateDirectoryAsync(Guid ownerId);
Task ClearAsync();
Task WriteAsync(Directory directory, SnapshotDeletionPolicy snapshotter);
Task ClearAsync();
}
}

2
backend/src/Squidex/Config/Domain/SubscriptionServices.cs

@ -28,7 +28,7 @@ namespace Squidex.Config.Domain
.AsOptional<IAppPlanBillingManager>();
services.AddSingletonAs<UsageGate>()
.AsOptional<IUserEvents>();
.AsSelf();
services.AddSingletonAs<NoopUserEvents>()
.AsOptional<IUserEvents>();

117
backend/tests/Squidex.Domain.Apps.Entities.Tests/Contents/Text/TextIndexerBenchmark.cs

@ -1,117 +0,0 @@
// ==========================================================================
// Squidex Headless CMS
// ==========================================================================
// Copyright (c) Squidex UG (haftungsbeschraenkt)
// All rights reserved. Licensed under the MIT license.
// ==========================================================================
using System;
using System.Collections.Generic;
using System.Threading.Tasks;
using FakeItEasy;
using Orleans.Concurrency;
using Squidex.Domain.Apps.Entities.Contents.Text.Lucene;
using Squidex.Infrastructure;
using Squidex.Infrastructure.Log;
using Xunit;
#pragma warning disable xUnit1004 // Test methods should not be skipped
namespace Squidex.Domain.Apps.Entities.Contents.Text
{
[Trait("Category", "Dependencies")]
public class TextIndexerBenchmark
{
private const int Size = 200;
private readonly NamedId<Guid> appId = NamedId.Of(Guid.NewGuid(), "my-app");
private readonly NamedId<Guid> schemaId = NamedId.Of(Guid.NewGuid(), "my-schema");
[Fact]
public async Task Should_index_and_search_in_temp_folder()
{
await IndexAndSearchAsync(TestStorages.TempFolder());
}
[Fact]
public async Task Should_index_and_search_in_assets()
{
await IndexAndSearchAsync(TestStorages.Assets());
}
[Fact]
public async Task Should_index_and_search_in_mongoDB()
{
await IndexAndSearchAsync(TestStorages.MongoDB());
}
private async Task IndexAndSearchAsync(IIndexStorage storage)
{
var factory = new IndexManager(storage, A.Fake<ISemanticLog>());
var grain = new LuceneTextIndexGrain(factory);
await grain.ActivateAsync(appId.Id);
var elapsed1 = await IndexAsync(grain);
var elapsed2 = await SearchAsync(grain);
var elapsed3 = await SearchAsync(grain);
Assert.Equal(new long[0], new[] { elapsed1, elapsed2, elapsed3 });
}
private async Task<long> IndexAsync(LuceneTextIndexGrain grain)
{
var text = new Dictionary<string, string>
{
["iv"] = "Hello World"
};
var ids = new Guid[Size];
for (var i = 0; i < ids.Length; i++)
{
ids[i] = Guid.NewGuid();
}
var watch = ValueStopwatch.StartNew();
foreach (var id in ids)
{
var commands = new IndexCommand[]
{
new UpsertIndexEntry
{
ContentId = id,
DocId = id.ToString(),
ServeAll = true,
ServePublished = true,
Texts = text
}
};
await grain.IndexAsync(schemaId, commands.AsImmutable());
}
return watch.Stop();
}
private async Task<long> SearchAsync(LuceneTextIndexGrain grain)
{
var searchContext = new SearchContext
{
Languages = new HashSet<string>()
};
var watch = ValueStopwatch.StartNew();
for (var i = 0; i < Size; i++)
{
var result = await grain.SearchAsync("Hello", default, searchContext);
Assert.NotEmpty(result);
}
return watch.Stop();
}
}
}

24
backend/tools/Benchmarks/Benchmarks.csproj

@ -0,0 +1,24 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<OutputType>Exe</OutputType>
<TargetFramework>netcoreapp3.1</TargetFramework>
</PropertyGroup>
<ItemGroup>
<PackageReference Include="BenchmarkDotNet" Version="0.12.0" />
<PackageReference Include="Lorem.Universal.Net" Version="3.0.64" />
<PackageReference Include="Microsoft.Orleans.Core.Abstractions" Version="3.1.2" />
<PackageReference Include="RefactoringEssentials" Version="5.6.0" PrivateAssets="all" />
<PackageReference Include="StyleCop.Analyzers" Version="1.1.118" PrivateAssets="all" />
</ItemGroup>
<PropertyGroup>
<CodeAnalysisRuleSet>..\..\Squidex.ruleset</CodeAnalysisRuleSet>
</PropertyGroup>
<ItemGroup>
<AdditionalFiles Include="..\..\stylecop.json" Link="stylecop.json" />
</ItemGroup>
<ItemGroup>
<ProjectReference Include="..\..\src\Squidex.Domain.Apps.Entities.MongoDb\Squidex.Domain.Apps.Entities.MongoDb.csproj" />
<ProjectReference Include="..\..\src\Squidex.Domain.Apps.Entities\Squidex.Domain.Apps.Entities.csproj" />
<ProjectReference Include="..\..\src\Squidex.Infrastructure\Squidex.Infrastructure.csproj" />
</ItemGroup>
</Project>

49
backend/tools/Benchmarks/IndexStorages.cs

@ -0,0 +1,49 @@
// ==========================================================================
// Squidex Headless CMS
// ==========================================================================
// Copyright (c) Squidex UG (haftungsbeschraenkt)
// All rights reserved. Licensed under the MIT license.
// ==========================================================================
using System;
using MongoDB.Driver;
using MongoDB.Driver.GridFS;
using Squidex.Domain.Apps.Entities.Contents.Text.Lucene;
using Squidex.Domain.Apps.Entities.Contents.Text.Lucene.Storage;
using Squidex.Domain.Apps.Entities.MongoDb.FullText;
using Squidex.Infrastructure.Assets;
namespace Benchmarks
{
public static class IndexStorages
{
public static IIndexStorage Assets()
{
var storage = new AssetIndexStorage(new MemoryAssetStore());
return storage;
}
public static IIndexStorage TempFolder()
{
var storage = new FileIndexStorage();
return storage;
}
public static IIndexStorage MongoDB()
{
var mongoClient = new MongoClient("mongodb://localhost");
var mongoDatabase = mongoClient.GetDatabase("FullText");
var mongoBucket = new GridFSBucket<string>(mongoDatabase, new GridFSBucketOptions
{
BucketName = $"bucket_{DateTime.UtcNow.Ticks}"
});
var storage = new MongoIndexStorage(mongoBucket);
return storage;
}
}
}

103
backend/tools/Benchmarks/IndexingBenchmarks.cs

@ -0,0 +1,103 @@
// ==========================================================================
// Squidex Headless CMS
// ==========================================================================
// Copyright (c) Squidex UG (haftungsbeschraenkt)
// All rights reserved. Licensed under the MIT license.
// ==========================================================================
using System;
using System.Collections.Generic;
using System.Threading.Tasks;
using BenchmarkDotNet.Attributes;
using Benchmarks.Utils;
using LoremNET;
using Orleans.Concurrency;
using Squidex.Domain.Apps.Entities.Contents.Text;
using Squidex.Domain.Apps.Entities.Contents.Text.Lucene;
using Squidex.Infrastructure;
namespace Benchmarks
{
[ShortRunJob]
[StopOnFirstError]
[RPlotExporter]
public class IndexingBenchmarks
{
private readonly IIndexStorage storageAssets = IndexStorages.Assets();
private readonly IIndexStorage storageTempFolder = IndexStorages.TempFolder();
private readonly IIndexStorage storageMongoDB = IndexStorages.MongoDB();
private readonly Dictionary<string, string> texts;
public IndexingBenchmarks()
{
texts = new Dictionary<string, string>
{
["iv"] = Lorem.Paragraph(10, 10)
};
}
[Params(1000)]
public int N { get; set; }
[Params(10)]
public int M { get; set; }
[Benchmark(Baseline = true)]
public async Task Index_TempFolder()
{
await IndexAndSearchAsync(storageTempFolder);
}
[Benchmark]
public async Task Index_Assets()
{
await IndexAndSearchAsync(storageAssets);
}
[Benchmark]
public async Task Index_MongoDB()
{
await IndexAndSearchAsync(storageMongoDB);
}
private async Task IndexAndSearchAsync(IIndexStorage storage)
{
var schemaId = NamedId.Of(Guid.NewGuid(), "my-schema");
var factory = new IndexManager(storage, new NoopLog());
var grain = new LuceneTextIndexGrain(factory);
await grain.ActivateAsync(Guid.NewGuid());
for (var i = 0; i < M; i++)
{
var ids = new Guid[N];
for (var j = 0; j < ids.Length; j++)
{
ids[j] = Guid.NewGuid();
}
foreach (var id in ids)
{
var commands = new IndexCommand[]
{
new UpsertIndexEntry
{
ContentId = id,
DocId = id.ToString(),
ServeAll = true,
ServePublished = true,
Texts = texts
}
};
await grain.IndexAsync(schemaId, commands.AsImmutable());
}
await grain.CommitAsync();
}
}
}
}

19
backend/tools/Benchmarks/Program.cs

@ -0,0 +1,19 @@
// ==========================================================================
// Squidex Headless CMS
// ==========================================================================
// Copyright (c) Squidex UG (haftungsbeschraenkt)
// All rights reserved. Licensed under the MIT license.
// ==========================================================================
using BenchmarkDotNet.Running;
namespace Benchmarks
{
public static class Program
{
public static void Main()
{
BenchmarkRunner.Run<IndexingBenchmarks>();
}
}
}

28
backend/tools/Benchmarks/Utils/NoopLog.cs

@ -0,0 +1,28 @@
// ==========================================================================
// Squidex Headless CMS
// ==========================================================================
// Copyright (c) Squidex UG (haftungsbeschraenkt)
// All rights reserved. Licensed under the MIT license.
// ==========================================================================
using System;
using Squidex.Infrastructure.Log;
namespace Benchmarks.Utils
{
public sealed class NoopLog : ISemanticLog
{
public ISemanticLog CreateScope(Action<IObjectWriter> objectWriter)
{
return this;
}
public void Log<T>(SemanticLogLevel logLevel, T context, Exception exception, LogFormatter<T> action)
{
}
public void Log(SemanticLogLevel logLevel, Exception exception, LogFormatter action)
{
}
}
}
Loading…
Cancel
Save