Support asset store.

backend/src/Squidex.Domain.Apps.Entities.MongoDb/FullText/MongoDirectoryFactory.cs

@@ -7,6 +7,8 @@
 
 using System;
 using System.IO;
+using System.Threading.Tasks;
+using Lucene.Net.Index;
 using MongoDB.Driver.GridFS;
 using Squidex.Domain.Apps.Entities.Contents.Text;
 using LuceneDirectory = Lucene.Net.Store.Directory;
@@ -22,14 +24,21 @@ namespace Squidex.Domain.Apps.Entities.MongoDb.FullText
             this.bucket = bucket;
         }
 
-        public LuceneDirectory Create(Guid schemaId)
+        public Task<LuceneDirectory> CreateAsync(Guid schemaId)
         {
             var folderName = schemaId.ToString();
 
             var tempFolder = Path.Combine(Path.GetTempPath(), "Indices", folderName);
             var tempDirectory = new DirectoryInfo(tempFolder);
 
-            return new MongoDirectory(bucket, folderName, tempDirectory);
+            var directory = new MongoDirectory(bucket, folderName, tempDirectory);
+
+            return Task.FromResult<LuceneDirectory>(directory);
+        }
+
+        public Task WriteAsync(IndexWriter writer, SnapshotDeletionPolicy snapshotter)
+        {
+            return Task.CompletedTask;
         }
     }
 }

backend/src/Squidex.Domain.Apps.Entities/Contents/Text/AssetStoreDirectoryFactory.cs

@@ -0,0 +1,124 @@
+// ==========================================================================
+//  Squidex Headless CMS
+// ==========================================================================
+//  Copyright (c) Squidex UG (haftungsbeschraenkt)
+//  All rights reserved. Licensed under the MIT license.
+// ==========================================================================
+
+using System;
+using System.IO;
+using System.IO.Compression;
+using System.Threading.Tasks;
+using Lucene.Net.Index;
+using Lucene.Net.Store;
+using Squidex.Infrastructure;
+using Squidex.Infrastructure.Assets;
+using LuceneDirectory = Lucene.Net.Store.Directory;
+
+namespace Squidex.Domain.Apps.Entities.Contents.Text
+{
+    public sealed class AssetStoreDirectoryFactory : IDirectoryFactory
+    {
+        private const string ArchiveFile = "Archive.zip";
+        private const string LockFile = "write.lock";
+        private readonly IAssetStore assetStore;
+
+        public AssetStoreDirectoryFactory(IAssetStore assetStore)
+        {
+            Guard.NotNull(assetStore);
+
+            this.assetStore = assetStore;
+        }
+
+        public async Task<LuceneDirectory> CreateAsync(Guid schemaId)
+        {
+            var directoryInfo = new DirectoryInfo(Path.Combine(Path.GetTempPath(), "LocalIndices", schemaId.ToString()));
+
+            if (directoryInfo.Exists)
+            {
+                directoryInfo.Delete(true);
+            }
+
+            directoryInfo.Create();
+
+            using (var fileStream = GetArchiveStream(directoryInfo))
+            {
+                try
+                {
+                    await assetStore.DownloadAsync(directoryInfo.Name, 0, string.Empty, fileStream);
+
+                    fileStream.Position = 0;
+
+                    using (var zipArchive = new ZipArchive(fileStream, ZipArchiveMode.Read, true))
+                    {
+                        zipArchive.ExtractToDirectory(directoryInfo.FullName);
+                    }
+                }
+                catch (AssetNotFoundException)
+                {
+                }
+            }
+
+            var directory = FSDirectory.Open(directoryInfo);
+
+            return directory;
+        }
+
+        public async Task WriteAsync(IndexWriter writer, SnapshotDeletionPolicy snapshotter)
+        {
+            Guard.NotNull(writer);
+            Guard.NotNull(writer);
+
+            var directoryInfo = ((FSDirectory)writer.Directory).Directory;
+
+            var commit = snapshotter.Snapshot();
+            try
+            {
+                using (var fileStream = GetArchiveStream(directoryInfo))
+                {
+                    using (var zipArchive = new ZipArchive(fileStream, ZipArchiveMode.Create, true))
+                    {
+                        foreach (var fileName in commit.FileNames)
+                        {
+                            var file = new FileInfo(Path.Combine(directoryInfo.FullName, fileName));
+
+                            try
+                            {
+                                if (!file.Name.Equals(ArchiveFile, StringComparison.OrdinalIgnoreCase) &&
+                                    !file.Name.Equals(LockFile, StringComparison.OrdinalIgnoreCase))
+                                {
+                                    zipArchive.CreateEntryFromFile(file.FullName, file.Name);
+                                }
+                            }
+                            catch (IOException)
+                            {
+                                continue;
+                            }
+                        }
+                    }
+
+                    fileStream.Position = 0;
+
+                    await assetStore.UploadAsync(directoryInfo.Name, 0, string.Empty, fileStream, true);
+                }
+            }
+            finally
+            {
+                snapshotter.Release(commit);
+            }
+        }
+
+        private static FileStream GetArchiveStream(DirectoryInfo directoryInfo)
+        {
+            var path = Path.Combine(directoryInfo.FullName, ArchiveFile);
+
+            return new FileStream(
+                path,
+                FileMode.Create,
+                FileAccess.ReadWrite,
+                FileShare.None,
+                4096,
+                FileOptions.DeleteOnClose);
+        }
+    }
+}

backend/src/Squidex.Domain.Apps.Entities/Contents/Text/FSDirectoryFactory.cs

@@ -7,6 +7,8 @@
 
 using System;
 using System.IO;
+using System.Threading.Tasks;
+using Lucene.Net.Index;
 using Lucene.Net.Store;
 using LuceneDirectory = Lucene.Net.Store.Directory;
 
@@ -14,13 +16,17 @@ namespace Squidex.Domain.Apps.Entities.Contents.Text
 {
     public sealed class FSDirectoryFactory : IDirectoryFactory
     {
-        public LuceneDirectory Create(Guid schemaId)
+        public Task<LuceneDirectory> CreateAsync(Guid schemaId)
         {
             var folderName = $"Indexes/{schemaId}";
+            var folderPath = Path.Combine(Path.GetTempPath(), folderName);
 
-            var tempFolder = Path.Combine(Path.GetTempPath(), folderName);
+            return Task.FromResult<LuceneDirectory>(FSDirectory.Open(folderPath));
+        }
 
-            return FSDirectory.Open(tempFolder);
+        public Task WriteAsync(IndexWriter writer, SnapshotDeletionPolicy snapshotter)
+        {
+            return Task.CompletedTask;
         }
     }
 }

backend/src/Squidex.Domain.Apps.Entities/Contents/Text/IDirectoryFactory.cs

@@ -6,12 +6,16 @@
 // ==========================================================================
 
 using System;
+using System.Threading.Tasks;
+using Lucene.Net.Index;
 using Lucene.Net.Store;
 
 namespace Squidex.Domain.Apps.Entities.Contents.Text
 {
     public interface IDirectoryFactory
     {
-        Directory Create(Guid schemaId);
+        Task<Directory> CreateAsync(Guid schemaId);
+
+        Task WriteAsync(IndexWriter writer, SnapshotDeletionPolicy snapshotter);
     }
 }

backend/src/Squidex.Domain.Apps.Entities/Contents/Text/IndexHolder.cs

@@ -6,6 +6,7 @@
 // ==========================================================================
 
 using System;
+using System.Threading.Tasks;
 using Lucene.Net.Analysis;
 using Lucene.Net.Index;
 using Lucene.Net.Search;
@@ -22,6 +23,7 @@ namespace Squidex.Domain.Apps.Entities.Contents.Text
         private static readonly Analyzer SharedAnalyzer = new MultiLanguageAnalyzer(Version);
         private readonly SnapshotDeletionPolicy snapshotter = new SnapshotDeletionPolicy(new KeepOnlyLastCommitDeletionPolicy());
         private readonly Directory directory;
+        private readonly IDirectoryFactory directoryFactory;
         private IndexWriter indexWriter;
         private IndexSearcher? indexSearcher;
         private DirectoryReader? indexReader;
@@ -76,9 +78,10 @@ namespace Squidex.Domain.Apps.Entities.Contents.Text
             }
         }
 
-        public IndexHolder(IDirectoryFactory directoryFactory, Guid schemaId)
+        public IndexHolder(Directory directory, IDirectoryFactory directoryFactory)
         {
-            directory = directoryFactory.Create(schemaId);
+            this.directory = directory;
+            this.directoryFactory = directoryFactory;
         }
 
         public void Open()
@@ -136,7 +139,7 @@ namespace Squidex.Domain.Apps.Entities.Contents.Text
             }
         }
 
-        public void Commit()
+        public async Task CommitAsync()
         {
             ThrowIfDisposed();
 
@@ -145,6 +148,8 @@ namespace Squidex.Domain.Apps.Entities.Contents.Text
                 MarkStale();
 
                 indexWriter.Commit();
+
+                await directoryFactory.WriteAsync(indexWriter, snapshotter);
             }
             catch (OutOfMemoryException)
             {

backend/src/Squidex.Domain.Apps.Entities/Contents/Text/IndexHolderFactory.cs

@@ -7,6 +7,8 @@
 
 using System;
 using System.Collections.Generic;
+using System.Threading;
+using System.Threading.Tasks;
 using Squidex.Infrastructure;
 using Squidex.Infrastructure.Log;
 
@@ -15,6 +17,7 @@ namespace Squidex.Domain.Apps.Entities.Contents.Text
     public sealed class IndexHolderFactory : DisposableObjectBase
     {
         private readonly Dictionary<Guid, IndexHolder> indices = new Dictionary<Guid, IndexHolder>();
+        private readonly SemaphoreSlim lockObject = new SemaphoreSlim(1);
         private readonly IDirectoryFactory directoryFactory;
         private readonly ISemanticLog log;
 
@@ -32,8 +35,10 @@ namespace Squidex.Domain.Apps.Entities.Contents.Text
         {
             if (disposing)
             {
-                lock (indices)
+                try
                 {
+                    lockObject.Wait();
+
                     if (indices.Count > 0)
                     {
                         log.LogWarning(w => w
@@ -48,15 +53,21 @@ namespace Squidex.Domain.Apps.Entities.Contents.Text
                         indices.Clear();
                     }
                 }
+                finally
+                {
+                    lockObject.Release();
+                }
             }
         }
 
-        public IndexHolder Acquire(Guid schemaId)
+        public async Task<IndexHolder> AcquireAsync(Guid schemaId)
         {
             IndexHolder? index;
 
-            lock (indices)
+            try
             {
+                await lockObject.WaitAsync();
+
                 if (indices.TryGetValue(schemaId, out index))
                 {
                     log.LogWarning(w => w
@@ -66,10 +77,16 @@ namespace Squidex.Domain.Apps.Entities.Contents.Text
                     index.Dispose();
                 }
 
-                index = new IndexHolder(directoryFactory, schemaId);
+                var directory = await directoryFactory.CreateAsync(schemaId);
+
+                index = new IndexHolder(directory, directoryFactory);
 
                 indices[schemaId] = index;
             }
+            finally
+            {
+                lockObject.Release();
+            }
 
             index.Open();
 
@@ -78,10 +95,16 @@ namespace Squidex.Domain.Apps.Entities.Contents.Text
 
         public void Release(Guid id)
         {
-            lock (indices)
+            try
             {
+                lockObject.Wait();
+
                 indices.Remove(id);
             }
+            finally
+            {
+                lockObject.Release();
+            }
         }
     }
 }

backend/src/Squidex.Domain.Apps.Entities/Contents/Text/TextIndexerGrain.cs

@@ -15,7 +15,6 @@ using Lucene.Net.Util;
 using Squidex.Domain.Apps.Core;
 using Squidex.Infrastructure;
 using Squidex.Infrastructure.Orleans;
-using Squidex.Infrastructure.Tasks;
 using Squidex.Infrastructure.Validation;
 
 namespace Squidex.Domain.Apps.Entities.Contents.Text
@@ -50,12 +49,11 @@ namespace Squidex.Domain.Apps.Entities.Contents.Text
             return Task.CompletedTask;
         }
 
-        protected override Task OnActivateAsync(Guid key)
+        protected override async Task OnActivateAsync(Guid key)
         {
-            index = indexHolderFactory.Acquire(key);
-            indexState = new IndexState(index);
+            index = await indexHolderFactory.AcquireAsync(key);
 
-            return TaskHelper.Done;
+            indexState = new IndexState(index);
         }
 
         public Task<bool> IndexAsync(Update update)
@@ -170,16 +168,14 @@ namespace Squidex.Domain.Apps.Entities.Contents.Text
             return false;
         }
 
-        public Task FlushAsync()
+        public async Task FlushAsync()
         {
             if (updates > 0)
             {
-                index.Commit();
+                await index.CommitAsync();
 
                 updates = 0;
             }
-
-            return TaskHelper.Done;
         }
     }
 }