From 9e0838023eb26134fdb48ed6c68a804c60a27fd0 Mon Sep 17 00:00:00 2001 From: Sebastian Date: Wed, 4 Dec 2019 19:13:14 +0100 Subject: [PATCH] Support asset store. --- .../FullText/MongoDirectoryFactory.cs | 13 +- .../Text/AssetStoreDirectoryFactory.cs | 124 ++++++++++++++++++ .../Contents/Text/FSDirectoryFactory.cs | 12 +- .../Contents/Text/IDirectoryFactory.cs | 6 +- .../Contents/Text/IndexHolder.cs | 11 +- .../Contents/Text/IndexHolderFactory.cs | 33 ++++- .../Contents/Text/TextIndexerGrain.cs | 14 +- 7 files changed, 190 insertions(+), 23 deletions(-) create mode 100644 backend/src/Squidex.Domain.Apps.Entities/Contents/Text/AssetStoreDirectoryFactory.cs diff --git a/backend/src/Squidex.Domain.Apps.Entities.MongoDb/FullText/MongoDirectoryFactory.cs b/backend/src/Squidex.Domain.Apps.Entities.MongoDb/FullText/MongoDirectoryFactory.cs index adeb62ae3..1fd9e5806 100644 --- a/backend/src/Squidex.Domain.Apps.Entities.MongoDb/FullText/MongoDirectoryFactory.cs +++ b/backend/src/Squidex.Domain.Apps.Entities.MongoDb/FullText/MongoDirectoryFactory.cs @@ -7,6 +7,8 @@ using System; using System.IO; +using System.Threading.Tasks; +using Lucene.Net.Index; using MongoDB.Driver.GridFS; using Squidex.Domain.Apps.Entities.Contents.Text; using LuceneDirectory = Lucene.Net.Store.Directory; @@ -22,14 +24,21 @@ namespace Squidex.Domain.Apps.Entities.MongoDb.FullText this.bucket = bucket; } - public LuceneDirectory Create(Guid schemaId) + public Task CreateAsync(Guid schemaId) { var folderName = schemaId.ToString(); var tempFolder = Path.Combine(Path.GetTempPath(), "Indices", folderName); var tempDirectory = new DirectoryInfo(tempFolder); - return new MongoDirectory(bucket, folderName, tempDirectory); + var directory = new MongoDirectory(bucket, folderName, tempDirectory); + + return Task.FromResult(directory); + } + + public Task WriteAsync(IndexWriter writer, SnapshotDeletionPolicy snapshotter) + { + return Task.CompletedTask; } } } diff --git a/backend/src/Squidex.Domain.Apps.Entities/Contents/Text/AssetStoreDirectoryFactory.cs b/backend/src/Squidex.Domain.Apps.Entities/Contents/Text/AssetStoreDirectoryFactory.cs new file mode 100644 index 000000000..f0af46aa5 --- /dev/null +++ b/backend/src/Squidex.Domain.Apps.Entities/Contents/Text/AssetStoreDirectoryFactory.cs @@ -0,0 +1,124 @@ +// ========================================================================== +// Squidex Headless CMS +// ========================================================================== +// Copyright (c) Squidex UG (haftungsbeschraenkt) +// All rights reserved. Licensed under the MIT license. +// ========================================================================== + +using System; +using System.IO; +using System.IO.Compression; +using System.Threading.Tasks; +using Lucene.Net.Index; +using Lucene.Net.Store; +using Squidex.Infrastructure; +using Squidex.Infrastructure.Assets; +using LuceneDirectory = Lucene.Net.Store.Directory; + +namespace Squidex.Domain.Apps.Entities.Contents.Text +{ + public sealed class AssetStoreDirectoryFactory : IDirectoryFactory + { + private const string ArchiveFile = "Archive.zip"; + private const string LockFile = "write.lock"; + private readonly IAssetStore assetStore; + + public AssetStoreDirectoryFactory(IAssetStore assetStore) + { + Guard.NotNull(assetStore); + + this.assetStore = assetStore; + } + + public async Task CreateAsync(Guid schemaId) + { + var directoryInfo = new DirectoryInfo(Path.Combine(Path.GetTempPath(), "LocalIndices", schemaId.ToString())); + + if (directoryInfo.Exists) + { + directoryInfo.Delete(true); + } + + directoryInfo.Create(); + + using (var fileStream = GetArchiveStream(directoryInfo)) + { + try + { + await assetStore.DownloadAsync(directoryInfo.Name, 0, string.Empty, fileStream); + + fileStream.Position = 0; + + using (var zipArchive = new ZipArchive(fileStream, ZipArchiveMode.Read, true)) + { + zipArchive.ExtractToDirectory(directoryInfo.FullName); + } + } + catch (AssetNotFoundException) + { + } + } + + var directory = FSDirectory.Open(directoryInfo); + + return directory; + } + + public async Task WriteAsync(IndexWriter writer, SnapshotDeletionPolicy snapshotter) + { + Guard.NotNull(writer); + Guard.NotNull(writer); + + var directoryInfo = ((FSDirectory)writer.Directory).Directory; + + var commit = snapshotter.Snapshot(); + try + { + using (var fileStream = GetArchiveStream(directoryInfo)) + { + using (var zipArchive = new ZipArchive(fileStream, ZipArchiveMode.Create, true)) + { + foreach (var fileName in commit.FileNames) + { + var file = new FileInfo(Path.Combine(directoryInfo.FullName, fileName)); + + try + { + if (!file.Name.Equals(ArchiveFile, StringComparison.OrdinalIgnoreCase) && + !file.Name.Equals(LockFile, StringComparison.OrdinalIgnoreCase)) + { + zipArchive.CreateEntryFromFile(file.FullName, file.Name); + } + } + catch (IOException) + { + continue; + } + } + } + + fileStream.Position = 0; + + await assetStore.UploadAsync(directoryInfo.Name, 0, string.Empty, fileStream, true); + } + } + finally + { + snapshotter.Release(commit); + } + } + + private static FileStream GetArchiveStream(DirectoryInfo directoryInfo) + { + var path = Path.Combine(directoryInfo.FullName, ArchiveFile); + + return new FileStream( + path, + FileMode.Create, + FileAccess.ReadWrite, + FileShare.None, + 4096, + FileOptions.DeleteOnClose); + } + } +} diff --git a/backend/src/Squidex.Domain.Apps.Entities/Contents/Text/FSDirectoryFactory.cs b/backend/src/Squidex.Domain.Apps.Entities/Contents/Text/FSDirectoryFactory.cs index 63bc8a4e6..7b3706f24 100644 --- a/backend/src/Squidex.Domain.Apps.Entities/Contents/Text/FSDirectoryFactory.cs +++ b/backend/src/Squidex.Domain.Apps.Entities/Contents/Text/FSDirectoryFactory.cs @@ -7,6 +7,8 @@ using System; using System.IO; +using System.Threading.Tasks; +using Lucene.Net.Index; using Lucene.Net.Store; using LuceneDirectory = Lucene.Net.Store.Directory; @@ -14,13 +16,17 @@ namespace Squidex.Domain.Apps.Entities.Contents.Text { public sealed class FSDirectoryFactory : IDirectoryFactory { - public LuceneDirectory Create(Guid schemaId) + public Task CreateAsync(Guid schemaId) { var folderName = $"Indexes/{schemaId}"; + var folderPath = Path.Combine(Path.GetTempPath(), folderName); - var tempFolder = Path.Combine(Path.GetTempPath(), folderName); + return Task.FromResult(FSDirectory.Open(folderPath)); + } - return FSDirectory.Open(tempFolder); + public Task WriteAsync(IndexWriter writer, SnapshotDeletionPolicy snapshotter) + { + return Task.CompletedTask; } } } diff --git a/backend/src/Squidex.Domain.Apps.Entities/Contents/Text/IDirectoryFactory.cs b/backend/src/Squidex.Domain.Apps.Entities/Contents/Text/IDirectoryFactory.cs index 31fc6b213..7e49ba737 100644 --- a/backend/src/Squidex.Domain.Apps.Entities/Contents/Text/IDirectoryFactory.cs +++ b/backend/src/Squidex.Domain.Apps.Entities/Contents/Text/IDirectoryFactory.cs @@ -6,12 +6,16 @@ // ========================================================================== using System; +using System.Threading.Tasks; +using Lucene.Net.Index; using Lucene.Net.Store; namespace Squidex.Domain.Apps.Entities.Contents.Text { public interface IDirectoryFactory { - Directory Create(Guid schemaId); + Task CreateAsync(Guid schemaId); + + Task WriteAsync(IndexWriter writer, SnapshotDeletionPolicy snapshotter); } } diff --git a/backend/src/Squidex.Domain.Apps.Entities/Contents/Text/IndexHolder.cs b/backend/src/Squidex.Domain.Apps.Entities/Contents/Text/IndexHolder.cs index 478c6c6fc..3437502e4 100644 --- a/backend/src/Squidex.Domain.Apps.Entities/Contents/Text/IndexHolder.cs +++ b/backend/src/Squidex.Domain.Apps.Entities/Contents/Text/IndexHolder.cs @@ -6,6 +6,7 @@ // ========================================================================== using System; +using System.Threading.Tasks; using Lucene.Net.Analysis; using Lucene.Net.Index; using Lucene.Net.Search; @@ -22,6 +23,7 @@ namespace Squidex.Domain.Apps.Entities.Contents.Text private static readonly Analyzer SharedAnalyzer = new MultiLanguageAnalyzer(Version); private readonly SnapshotDeletionPolicy snapshotter = new SnapshotDeletionPolicy(new KeepOnlyLastCommitDeletionPolicy()); private readonly Directory directory; + private readonly IDirectoryFactory directoryFactory; private IndexWriter indexWriter; private IndexSearcher? indexSearcher; private DirectoryReader? indexReader; @@ -76,9 +78,10 @@ namespace Squidex.Domain.Apps.Entities.Contents.Text } } - public IndexHolder(IDirectoryFactory directoryFactory, Guid schemaId) + public IndexHolder(Directory directory, IDirectoryFactory directoryFactory) { - directory = directoryFactory.Create(schemaId); + this.directory = directory; + this.directoryFactory = directoryFactory; } public void Open() @@ -136,7 +139,7 @@ namespace Squidex.Domain.Apps.Entities.Contents.Text } } - public void Commit() + public async Task CommitAsync() { ThrowIfDisposed(); @@ -145,6 +148,8 @@ namespace Squidex.Domain.Apps.Entities.Contents.Text MarkStale(); indexWriter.Commit(); + + await directoryFactory.WriteAsync(indexWriter, snapshotter); } catch (OutOfMemoryException) { diff --git a/backend/src/Squidex.Domain.Apps.Entities/Contents/Text/IndexHolderFactory.cs b/backend/src/Squidex.Domain.Apps.Entities/Contents/Text/IndexHolderFactory.cs index 2ce30ffa3..ba66f1457 100644 --- a/backend/src/Squidex.Domain.Apps.Entities/Contents/Text/IndexHolderFactory.cs +++ b/backend/src/Squidex.Domain.Apps.Entities/Contents/Text/IndexHolderFactory.cs @@ -7,6 +7,8 @@ using System; using System.Collections.Generic; +using System.Threading; +using System.Threading.Tasks; using Squidex.Infrastructure; using Squidex.Infrastructure.Log; @@ -15,6 +17,7 @@ namespace Squidex.Domain.Apps.Entities.Contents.Text public sealed class IndexHolderFactory : DisposableObjectBase { private readonly Dictionary indices = new Dictionary(); + private readonly SemaphoreSlim lockObject = new SemaphoreSlim(1); private readonly IDirectoryFactory directoryFactory; private readonly ISemanticLog log; @@ -32,8 +35,10 @@ namespace Squidex.Domain.Apps.Entities.Contents.Text { if (disposing) { - lock (indices) + try { + lockObject.Wait(); + if (indices.Count > 0) { log.LogWarning(w => w @@ -48,15 +53,21 @@ namespace Squidex.Domain.Apps.Entities.Contents.Text indices.Clear(); } } + finally + { + lockObject.Release(); + } } } - public IndexHolder Acquire(Guid schemaId) + public async Task AcquireAsync(Guid schemaId) { IndexHolder? index; - lock (indices) + try { + await lockObject.WaitAsync(); + if (indices.TryGetValue(schemaId, out index)) { log.LogWarning(w => w @@ -66,10 +77,16 @@ namespace Squidex.Domain.Apps.Entities.Contents.Text index.Dispose(); } - index = new IndexHolder(directoryFactory, schemaId); + var directory = await directoryFactory.CreateAsync(schemaId); + + index = new IndexHolder(directory, directoryFactory); indices[schemaId] = index; } + finally + { + lockObject.Release(); + } index.Open(); @@ -78,10 +95,16 @@ namespace Squidex.Domain.Apps.Entities.Contents.Text public void Release(Guid id) { - lock (indices) + try { + lockObject.Wait(); + indices.Remove(id); } + finally + { + lockObject.Release(); + } } } } diff --git a/backend/src/Squidex.Domain.Apps.Entities/Contents/Text/TextIndexerGrain.cs b/backend/src/Squidex.Domain.Apps.Entities/Contents/Text/TextIndexerGrain.cs index 6ca66341e..ff1c22510 100644 --- a/backend/src/Squidex.Domain.Apps.Entities/Contents/Text/TextIndexerGrain.cs +++ b/backend/src/Squidex.Domain.Apps.Entities/Contents/Text/TextIndexerGrain.cs @@ -15,7 +15,6 @@ using Lucene.Net.Util; using Squidex.Domain.Apps.Core; using Squidex.Infrastructure; using Squidex.Infrastructure.Orleans; -using Squidex.Infrastructure.Tasks; using Squidex.Infrastructure.Validation; namespace Squidex.Domain.Apps.Entities.Contents.Text @@ -50,12 +49,11 @@ namespace Squidex.Domain.Apps.Entities.Contents.Text return Task.CompletedTask; } - protected override Task OnActivateAsync(Guid key) + protected override async Task OnActivateAsync(Guid key) { - index = indexHolderFactory.Acquire(key); - indexState = new IndexState(index); + index = await indexHolderFactory.AcquireAsync(key); - return TaskHelper.Done; + indexState = new IndexState(index); } public Task IndexAsync(Update update) @@ -170,16 +168,14 @@ namespace Squidex.Domain.Apps.Entities.Contents.Text return false; } - public Task FlushAsync() + public async Task FlushAsync() { if (updates > 0) { - index.Commit(); + await index.CommitAsync(); updates = 0; } - - return TaskHelper.Done; } } }