mirror of https://github.com/Squidex/squidex.git
19 changed files with 431 additions and 328 deletions
@ -0,0 +1,28 @@ |
|||||
|
// ==========================================================================
|
||||
|
// Squidex Headless CMS
|
||||
|
// ==========================================================================
|
||||
|
// Copyright (c) Squidex UG (haftungsbeschraenkt)
|
||||
|
// All rights reserved. Licensed under the MIT license.
|
||||
|
// ==========================================================================
|
||||
|
|
||||
|
using Lucene.Net.Analysis; |
||||
|
using Lucene.Net.Index; |
||||
|
using Lucene.Net.Search; |
||||
|
|
||||
|
namespace Squidex.Domain.Apps.Entities.Contents.Text |
||||
|
{ |
||||
|
public interface IIndex |
||||
|
{ |
||||
|
Analyzer? Analyzer { get; } |
||||
|
|
||||
|
IndexReader? Reader { get; } |
||||
|
|
||||
|
IndexSearcher? Searcher { get; } |
||||
|
|
||||
|
IndexWriter Writer { get; } |
||||
|
|
||||
|
void EnsureReader(); |
||||
|
|
||||
|
void MarkStale(); |
||||
|
} |
||||
|
} |
||||
@ -1,162 +0,0 @@ |
|||||
// ==========================================================================
|
|
||||
// Squidex Headless CMS
|
|
||||
// ==========================================================================
|
|
||||
// Copyright (c) Squidex UG (haftungsbeschraenkt)
|
|
||||
// All rights reserved. Licensed under the MIT license.
|
|
||||
// ==========================================================================
|
|
||||
|
|
||||
using System; |
|
||||
using System.Threading.Tasks; |
|
||||
using Lucene.Net.Analysis; |
|
||||
using Lucene.Net.Index; |
|
||||
using Lucene.Net.Search; |
|
||||
using Lucene.Net.Store; |
|
||||
using Lucene.Net.Util; |
|
||||
using Squidex.Infrastructure; |
|
||||
|
|
||||
namespace Squidex.Domain.Apps.Entities.Contents.Text |
|
||||
{ |
|
||||
public sealed class IndexHolder : DisposableObjectBase |
|
||||
{ |
|
||||
private const LuceneVersion Version = LuceneVersion.LUCENE_48; |
|
||||
private static readonly MergeScheduler MergeScheduler = new ConcurrentMergeScheduler(); |
|
||||
private static readonly Analyzer SharedAnalyzer = new MultiLanguageAnalyzer(Version); |
|
||||
private readonly SnapshotDeletionPolicy snapshotter = new SnapshotDeletionPolicy(new KeepOnlyLastCommitDeletionPolicy()); |
|
||||
private readonly Directory directory; |
|
||||
private readonly IDirectoryFactory directoryFactory; |
|
||||
private IndexWriter indexWriter; |
|
||||
private IndexSearcher? indexSearcher; |
|
||||
private DirectoryReader? indexReader; |
|
||||
|
|
||||
public Analyzer Analyzer |
|
||||
{ |
|
||||
get |
|
||||
{ |
|
||||
ThrowIfDisposed(); |
|
||||
|
|
||||
return SharedAnalyzer; |
|
||||
} |
|
||||
} |
|
||||
|
|
||||
public SnapshotDeletionPolicy Snapshotter |
|
||||
{ |
|
||||
get |
|
||||
{ |
|
||||
ThrowIfDisposed(); |
|
||||
|
|
||||
return snapshotter; |
|
||||
} |
|
||||
} |
|
||||
|
|
||||
public IndexWriter Writer |
|
||||
{ |
|
||||
get |
|
||||
{ |
|
||||
ThrowIfDisposed(); |
|
||||
|
|
||||
return indexWriter; |
|
||||
} |
|
||||
} |
|
||||
|
|
||||
public IndexReader? Reader |
|
||||
{ |
|
||||
get |
|
||||
{ |
|
||||
ThrowIfDisposed(); |
|
||||
|
|
||||
return indexReader; |
|
||||
} |
|
||||
} |
|
||||
|
|
||||
public IndexSearcher? Searcher |
|
||||
{ |
|
||||
get |
|
||||
{ |
|
||||
ThrowIfDisposed(); |
|
||||
|
|
||||
return indexSearcher; |
|
||||
} |
|
||||
} |
|
||||
|
|
||||
public IndexHolder(Directory directory, IDirectoryFactory directoryFactory) |
|
||||
{ |
|
||||
this.directory = directory; |
|
||||
this.directoryFactory = directoryFactory; |
|
||||
} |
|
||||
|
|
||||
public void Open() |
|
||||
{ |
|
||||
RecreateIndexWriter(); |
|
||||
|
|
||||
if (indexWriter.NumDocs > 0) |
|
||||
{ |
|
||||
EnsureReader(); |
|
||||
} |
|
||||
} |
|
||||
|
|
||||
protected override void DisposeObject(bool disposing) |
|
||||
{ |
|
||||
if (disposing) |
|
||||
{ |
|
||||
indexWriter?.Dispose(); |
|
||||
} |
|
||||
} |
|
||||
|
|
||||
private void RecreateIndexWriter() |
|
||||
{ |
|
||||
var config = new IndexWriterConfig(Version, Analyzer) |
|
||||
{ |
|
||||
IndexDeletionPolicy = snapshotter, |
|
||||
MergePolicy = new TieredMergePolicy(), |
|
||||
MergeScheduler = MergeScheduler |
|
||||
}; |
|
||||
|
|
||||
indexWriter = new IndexWriter(directory, config); |
|
||||
|
|
||||
MarkStale(); |
|
||||
} |
|
||||
|
|
||||
public void EnsureReader() |
|
||||
{ |
|
||||
ThrowIfDisposed(); |
|
||||
|
|
||||
if (indexReader == null) |
|
||||
{ |
|
||||
indexReader = indexWriter.GetReader(true); |
|
||||
indexSearcher = new IndexSearcher(indexReader); |
|
||||
} |
|
||||
} |
|
||||
|
|
||||
public void MarkStale() |
|
||||
{ |
|
||||
ThrowIfDisposed(); |
|
||||
|
|
||||
if (indexReader != null) |
|
||||
{ |
|
||||
indexReader.Dispose(); |
|
||||
indexReader = null; |
|
||||
indexSearcher = null; |
|
||||
} |
|
||||
} |
|
||||
|
|
||||
public async Task CommitAsync() |
|
||||
{ |
|
||||
ThrowIfDisposed(); |
|
||||
|
|
||||
try |
|
||||
{ |
|
||||
MarkStale(); |
|
||||
|
|
||||
indexWriter.Commit(); |
|
||||
|
|
||||
await directoryFactory.WriteAsync(indexWriter, snapshotter); |
|
||||
} |
|
||||
catch (OutOfMemoryException) |
|
||||
{ |
|
||||
RecreateIndexWriter(); |
|
||||
|
|
||||
throw; |
|
||||
} |
|
||||
} |
|
||||
} |
|
||||
} |
|
||||
@ -1,110 +0,0 @@ |
|||||
// ==========================================================================
|
|
||||
// Squidex Headless CMS
|
|
||||
// ==========================================================================
|
|
||||
// Copyright (c) Squidex UG (haftungsbeschraenkt)
|
|
||||
// All rights reserved. Licensed under the MIT license.
|
|
||||
// ==========================================================================
|
|
||||
|
|
||||
using System; |
|
||||
using System.Collections.Generic; |
|
||||
using System.Threading; |
|
||||
using System.Threading.Tasks; |
|
||||
using Squidex.Infrastructure; |
|
||||
using Squidex.Infrastructure.Log; |
|
||||
|
|
||||
namespace Squidex.Domain.Apps.Entities.Contents.Text |
|
||||
{ |
|
||||
public sealed class IndexHolderFactory : DisposableObjectBase |
|
||||
{ |
|
||||
private readonly Dictionary<Guid, IndexHolder> indices = new Dictionary<Guid, IndexHolder>(); |
|
||||
private readonly SemaphoreSlim lockObject = new SemaphoreSlim(1); |
|
||||
private readonly IDirectoryFactory directoryFactory; |
|
||||
private readonly ISemanticLog log; |
|
||||
|
|
||||
public IndexHolderFactory(IDirectoryFactory directoryFactory, ISemanticLog log) |
|
||||
{ |
|
||||
Guard.NotNull(directoryFactory); |
|
||||
Guard.NotNull(log); |
|
||||
|
|
||||
this.directoryFactory = directoryFactory; |
|
||||
|
|
||||
this.log = log; |
|
||||
} |
|
||||
|
|
||||
protected override void DisposeObject(bool disposing) |
|
||||
{ |
|
||||
if (disposing) |
|
||||
{ |
|
||||
try |
|
||||
{ |
|
||||
lockObject.Wait(); |
|
||||
|
|
||||
if (indices.Count > 0) |
|
||||
{ |
|
||||
log.LogWarning(w => w |
|
||||
.WriteProperty("message", "Unreleased indices found.") |
|
||||
.WriteProperty("count", indices.Count)); |
|
||||
|
|
||||
foreach (var index in indices) |
|
||||
{ |
|
||||
index.Value.Dispose(); |
|
||||
} |
|
||||
|
|
||||
indices.Clear(); |
|
||||
} |
|
||||
} |
|
||||
finally |
|
||||
{ |
|
||||
lockObject.Release(); |
|
||||
} |
|
||||
} |
|
||||
} |
|
||||
|
|
||||
public async Task<IndexHolder> AcquireAsync(Guid schemaId) |
|
||||
{ |
|
||||
IndexHolder? index; |
|
||||
|
|
||||
try |
|
||||
{ |
|
||||
await lockObject.WaitAsync(); |
|
||||
|
|
||||
if (indices.TryGetValue(schemaId, out index)) |
|
||||
{ |
|
||||
log.LogWarning(w => w |
|
||||
.WriteProperty("message", "Unreleased index found.") |
|
||||
.WriteProperty("schemaId", schemaId.ToString())); |
|
||||
|
|
||||
index.Dispose(); |
|
||||
} |
|
||||
|
|
||||
var directory = await directoryFactory.CreateAsync(schemaId); |
|
||||
|
|
||||
index = new IndexHolder(directory, directoryFactory); |
|
||||
|
|
||||
indices[schemaId] = index; |
|
||||
} |
|
||||
finally |
|
||||
{ |
|
||||
lockObject.Release(); |
|
||||
} |
|
||||
|
|
||||
index.Open(); |
|
||||
|
|
||||
return index; |
|
||||
} |
|
||||
|
|
||||
public void Release(Guid id) |
|
||||
{ |
|
||||
try |
|
||||
{ |
|
||||
lockObject.Wait(); |
|
||||
|
|
||||
indices.Remove(id); |
|
||||
} |
|
||||
finally |
|
||||
{ |
|
||||
lockObject.Release(); |
|
||||
} |
|
||||
} |
|
||||
} |
|
||||
} |
|
||||
@ -0,0 +1,148 @@ |
|||||
|
// ==========================================================================
|
||||
|
// Squidex Headless CMS
|
||||
|
// ==========================================================================
|
||||
|
// Copyright (c) Squidex UG (haftungsbeschraenkt)
|
||||
|
// All rights reserved. Licensed under the MIT license.
|
||||
|
// ==========================================================================
|
||||
|
|
||||
|
using System; |
||||
|
using System.Collections.Generic; |
||||
|
using System.Linq; |
||||
|
using System.Threading; |
||||
|
using System.Threading.Tasks; |
||||
|
using Squidex.Infrastructure; |
||||
|
using Squidex.Infrastructure.Log; |
||||
|
|
||||
|
namespace Squidex.Domain.Apps.Entities.Contents.Text |
||||
|
{ |
||||
|
public sealed partial class IndexManager : DisposableObjectBase |
||||
|
{ |
||||
|
private readonly Dictionary<Guid, IndexHolder> indices = new Dictionary<Guid, IndexHolder>(); |
||||
|
private readonly SemaphoreSlim lockObject = new SemaphoreSlim(1); |
||||
|
private readonly IIndexStorage directoryFactory; |
||||
|
private readonly ISemanticLog log; |
||||
|
|
||||
|
public IndexManager(IIndexStorage directoryFactory, ISemanticLog log) |
||||
|
{ |
||||
|
Guard.NotNull(directoryFactory); |
||||
|
Guard.NotNull(log); |
||||
|
|
||||
|
this.directoryFactory = directoryFactory; |
||||
|
|
||||
|
this.log = log; |
||||
|
} |
||||
|
|
||||
|
protected override void DisposeObject(bool disposing) |
||||
|
{ |
||||
|
if (disposing) |
||||
|
{ |
||||
|
ReleaseAllAsync().Wait(); |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
public async Task<IIndex> AcquireAsync(Guid schemaId) |
||||
|
{ |
||||
|
IndexHolder? indexHolder; |
||||
|
|
||||
|
try |
||||
|
{ |
||||
|
await lockObject.WaitAsync(); |
||||
|
|
||||
|
if (indices.TryGetValue(schemaId, out indexHolder)) |
||||
|
{ |
||||
|
log.LogWarning(w => w |
||||
|
.WriteProperty("message", "Unreleased index found.") |
||||
|
.WriteProperty("schemaId", schemaId.ToString())); |
||||
|
|
||||
|
await CommitInternalAsync(indexHolder, true); |
||||
|
} |
||||
|
|
||||
|
indexHolder = new IndexHolder(schemaId); |
||||
|
indices[schemaId] = indexHolder; |
||||
|
} |
||||
|
finally |
||||
|
{ |
||||
|
lockObject.Release(); |
||||
|
} |
||||
|
|
||||
|
var directory = await directoryFactory.CreateDirectoryAsync(schemaId); |
||||
|
|
||||
|
indexHolder.Open(directory); |
||||
|
|
||||
|
return indexHolder; |
||||
|
} |
||||
|
|
||||
|
public async Task ReleaseAsync(IIndex index) |
||||
|
{ |
||||
|
Guard.NotNull(index); |
||||
|
|
||||
|
var indexHolder = (IndexHolder)index; |
||||
|
|
||||
|
try |
||||
|
{ |
||||
|
lockObject.Wait(); |
||||
|
|
||||
|
indexHolder.Release(); |
||||
|
indices.Remove(indexHolder.Id); |
||||
|
} |
||||
|
finally |
||||
|
{ |
||||
|
lockObject.Release(); |
||||
|
} |
||||
|
|
||||
|
await CommitInternalAsync(indexHolder, true); |
||||
|
} |
||||
|
|
||||
|
public Task CommitAsync(IIndex index) |
||||
|
{ |
||||
|
Guard.NotNull(index); |
||||
|
|
||||
|
return CommitInternalAsync(index, false); |
||||
|
} |
||||
|
|
||||
|
private async Task CommitInternalAsync(IIndex index, bool dispose) |
||||
|
{ |
||||
|
if (index is IndexHolder holder) |
||||
|
{ |
||||
|
if (dispose) |
||||
|
{ |
||||
|
holder.Dispose(); |
||||
|
} |
||||
|
else |
||||
|
{ |
||||
|
holder.Commit(); |
||||
|
} |
||||
|
|
||||
|
await directoryFactory.WriteAsync(holder.GetUnsafeWriter().Directory, holder.Snapshotter); |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
private async Task ReleaseAllAsync() |
||||
|
{ |
||||
|
var current = indices.Values.ToList(); |
||||
|
|
||||
|
try |
||||
|
{ |
||||
|
lockObject.Wait(); |
||||
|
|
||||
|
indices.Clear(); |
||||
|
} |
||||
|
finally |
||||
|
{ |
||||
|
lockObject.Release(); |
||||
|
} |
||||
|
|
||||
|
if (current.Count > 0) |
||||
|
{ |
||||
|
log.LogWarning(w => w |
||||
|
.WriteProperty("message", "Unreleased indices found.") |
||||
|
.WriteProperty("count", indices.Count)); |
||||
|
|
||||
|
foreach (var index in current) |
||||
|
{ |
||||
|
await CommitInternalAsync(index, true); |
||||
|
} |
||||
|
} |
||||
|
} |
||||
|
} |
||||
|
} |
||||
@ -0,0 +1,176 @@ |
|||||
|
// ==========================================================================
|
||||
|
// Squidex Headless CMS
|
||||
|
// ==========================================================================
|
||||
|
// Copyright (c) Squidex UG (haftungsbeschraenkt)
|
||||
|
// All rights reserved. Licensed under the MIT license.
|
||||
|
// ==========================================================================
|
||||
|
|
||||
|
using System; |
||||
|
using Lucene.Net.Analysis; |
||||
|
using Lucene.Net.Index; |
||||
|
using Lucene.Net.Search; |
||||
|
using Lucene.Net.Store; |
||||
|
using Lucene.Net.Util; |
||||
|
using Squidex.Infrastructure; |
||||
|
|
||||
|
namespace Squidex.Domain.Apps.Entities.Contents.Text |
||||
|
{ |
||||
|
public sealed partial class IndexManager |
||||
|
{ |
||||
|
private sealed class IndexHolder : IDisposable, IIndex |
||||
|
{ |
||||
|
private const LuceneVersion Version = LuceneVersion.LUCENE_48; |
||||
|
private static readonly MergeScheduler MergeScheduler = new ConcurrentMergeScheduler(); |
||||
|
private static readonly Analyzer SharedAnalyzer = new MultiLanguageAnalyzer(Version); |
||||
|
private readonly SnapshotDeletionPolicy snapshotter = new SnapshotDeletionPolicy(new KeepOnlyLastCommitDeletionPolicy()); |
||||
|
private Directory directory; |
||||
|
private IndexWriter indexWriter; |
||||
|
private IndexSearcher? indexSearcher; |
||||
|
private DirectoryReader? indexReader; |
||||
|
private bool isReleased; |
||||
|
|
||||
|
public Analyzer Analyzer |
||||
|
{ |
||||
|
get { return SharedAnalyzer; } |
||||
|
} |
||||
|
|
||||
|
public SnapshotDeletionPolicy Snapshotter |
||||
|
{ |
||||
|
get { return snapshotter; } |
||||
|
} |
||||
|
|
||||
|
public IndexWriter Writer |
||||
|
{ |
||||
|
get |
||||
|
{ |
||||
|
ThrowIfReleased(); |
||||
|
|
||||
|
return indexWriter; |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
public IndexReader? Reader |
||||
|
{ |
||||
|
get |
||||
|
{ |
||||
|
ThrowIfReleased(); |
||||
|
|
||||
|
return indexReader; |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
public IndexSearcher? Searcher |
||||
|
{ |
||||
|
get |
||||
|
{ |
||||
|
ThrowIfReleased(); |
||||
|
|
||||
|
return indexSearcher; |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
public Guid Id { get; } |
||||
|
|
||||
|
public IndexHolder(Guid id) |
||||
|
{ |
||||
|
Id = id; |
||||
|
} |
||||
|
|
||||
|
public void Dispose() |
||||
|
{ |
||||
|
indexReader?.Dispose(); |
||||
|
indexWriter?.Dispose(); |
||||
|
} |
||||
|
|
||||
|
public void Open(Directory directory) |
||||
|
{ |
||||
|
Guard.NotNull(directory); |
||||
|
|
||||
|
this.directory = directory; |
||||
|
|
||||
|
RecreateIndexWriter(); |
||||
|
|
||||
|
if (indexWriter.NumDocs > 0) |
||||
|
{ |
||||
|
EnsureReader(); |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
private void RecreateIndexWriter() |
||||
|
{ |
||||
|
var config = new IndexWriterConfig(Version, Analyzer) |
||||
|
{ |
||||
|
IndexDeletionPolicy = snapshotter, |
||||
|
MergePolicy = new TieredMergePolicy(), |
||||
|
MergeScheduler = MergeScheduler |
||||
|
}; |
||||
|
|
||||
|
indexWriter = new IndexWriter(directory, config); |
||||
|
|
||||
|
MarkStale(); |
||||
|
} |
||||
|
|
||||
|
public void EnsureReader() |
||||
|
{ |
||||
|
ThrowIfReleased(); |
||||
|
|
||||
|
if (indexReader == null) |
||||
|
{ |
||||
|
indexReader = indexWriter.GetReader(true); |
||||
|
indexSearcher = new IndexSearcher(indexReader); |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
public void MarkStale() |
||||
|
{ |
||||
|
ThrowIfReleased(); |
||||
|
|
||||
|
MarkStaleInternal(); |
||||
|
} |
||||
|
|
||||
|
private void MarkStaleInternal() |
||||
|
{ |
||||
|
if (indexReader != null) |
||||
|
{ |
||||
|
indexReader.Dispose(); |
||||
|
indexReader = null; |
||||
|
indexSearcher = null; |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
internal void Commit() |
||||
|
{ |
||||
|
try |
||||
|
{ |
||||
|
MarkStaleInternal(); |
||||
|
|
||||
|
indexWriter.Commit(); |
||||
|
} |
||||
|
catch (OutOfMemoryException) |
||||
|
{ |
||||
|
RecreateIndexWriter(); |
||||
|
|
||||
|
throw; |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
internal void ThrowIfReleased() |
||||
|
{ |
||||
|
if (isReleased) |
||||
|
{ |
||||
|
throw new InvalidOperationException("Index is already released."); |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
internal void Release() |
||||
|
{ |
||||
|
isReleased = true; |
||||
|
} |
||||
|
|
||||
|
internal IndexWriter GetUnsafeWriter() |
||||
|
{ |
||||
|
return indexWriter; |
||||
|
} |
||||
|
} |
||||
|
} |
||||
|
} |
||||
@ -0,0 +1,23 @@ |
|||||
|
// ==========================================================================
|
||||
|
// Squidex Headless CMS
|
||||
|
// ==========================================================================
|
||||
|
// Copyright (c) Squidex UG (haftungsbeschraenkt)
|
||||
|
// All rights reserved. Licensed under the MIT license.
|
||||
|
// ==========================================================================
|
||||
|
|
||||
|
using Squidex.Infrastructure.Assets; |
||||
|
|
||||
|
namespace Squidex.Domain.Apps.Entities.Contents.Text |
||||
|
{ |
||||
|
public class TextIndexerGrainTests_Assets : TextIndexerGrainTestsBase |
||||
|
{ |
||||
|
public override IIndexStorage Storage { get; } = CreateStorage(); |
||||
|
|
||||
|
private static IIndexStorage CreateStorage() |
||||
|
{ |
||||
|
var storage = new AssetIndexStorage(new MemoryAssetStore()); |
||||
|
|
||||
|
return storage; |
||||
|
} |
||||
|
} |
||||
|
} |
||||
Loading…
Reference in new issue