mirror of https://github.com/Squidex/squidex.git
Sebastian
6 years ago
19 changed files with 431 additions and 328 deletions
@ -0,0 +1,28 @@ |
|||
// ==========================================================================
|
|||
// Squidex Headless CMS
|
|||
// ==========================================================================
|
|||
// Copyright (c) Squidex UG (haftungsbeschraenkt)
|
|||
// All rights reserved. Licensed under the MIT license.
|
|||
// ==========================================================================
|
|||
|
|||
using Lucene.Net.Analysis; |
|||
using Lucene.Net.Index; |
|||
using Lucene.Net.Search; |
|||
|
|||
namespace Squidex.Domain.Apps.Entities.Contents.Text |
|||
{ |
|||
public interface IIndex |
|||
{ |
|||
Analyzer? Analyzer { get; } |
|||
|
|||
IndexReader? Reader { get; } |
|||
|
|||
IndexSearcher? Searcher { get; } |
|||
|
|||
IndexWriter Writer { get; } |
|||
|
|||
void EnsureReader(); |
|||
|
|||
void MarkStale(); |
|||
} |
|||
} |
|||
@ -1,162 +0,0 @@ |
|||
// ==========================================================================
|
|||
// Squidex Headless CMS
|
|||
// ==========================================================================
|
|||
// Copyright (c) Squidex UG (haftungsbeschraenkt)
|
|||
// All rights reserved. Licensed under the MIT license.
|
|||
// ==========================================================================
|
|||
|
|||
using System; |
|||
using System.Threading.Tasks; |
|||
using Lucene.Net.Analysis; |
|||
using Lucene.Net.Index; |
|||
using Lucene.Net.Search; |
|||
using Lucene.Net.Store; |
|||
using Lucene.Net.Util; |
|||
using Squidex.Infrastructure; |
|||
|
|||
namespace Squidex.Domain.Apps.Entities.Contents.Text |
|||
{ |
|||
public sealed class IndexHolder : DisposableObjectBase |
|||
{ |
|||
private const LuceneVersion Version = LuceneVersion.LUCENE_48; |
|||
private static readonly MergeScheduler MergeScheduler = new ConcurrentMergeScheduler(); |
|||
private static readonly Analyzer SharedAnalyzer = new MultiLanguageAnalyzer(Version); |
|||
private readonly SnapshotDeletionPolicy snapshotter = new SnapshotDeletionPolicy(new KeepOnlyLastCommitDeletionPolicy()); |
|||
private readonly Directory directory; |
|||
private readonly IDirectoryFactory directoryFactory; |
|||
private IndexWriter indexWriter; |
|||
private IndexSearcher? indexSearcher; |
|||
private DirectoryReader? indexReader; |
|||
|
|||
public Analyzer Analyzer |
|||
{ |
|||
get |
|||
{ |
|||
ThrowIfDisposed(); |
|||
|
|||
return SharedAnalyzer; |
|||
} |
|||
} |
|||
|
|||
public SnapshotDeletionPolicy Snapshotter |
|||
{ |
|||
get |
|||
{ |
|||
ThrowIfDisposed(); |
|||
|
|||
return snapshotter; |
|||
} |
|||
} |
|||
|
|||
public IndexWriter Writer |
|||
{ |
|||
get |
|||
{ |
|||
ThrowIfDisposed(); |
|||
|
|||
return indexWriter; |
|||
} |
|||
} |
|||
|
|||
public IndexReader? Reader |
|||
{ |
|||
get |
|||
{ |
|||
ThrowIfDisposed(); |
|||
|
|||
return indexReader; |
|||
} |
|||
} |
|||
|
|||
public IndexSearcher? Searcher |
|||
{ |
|||
get |
|||
{ |
|||
ThrowIfDisposed(); |
|||
|
|||
return indexSearcher; |
|||
} |
|||
} |
|||
|
|||
public IndexHolder(Directory directory, IDirectoryFactory directoryFactory) |
|||
{ |
|||
this.directory = directory; |
|||
this.directoryFactory = directoryFactory; |
|||
} |
|||
|
|||
public void Open() |
|||
{ |
|||
RecreateIndexWriter(); |
|||
|
|||
if (indexWriter.NumDocs > 0) |
|||
{ |
|||
EnsureReader(); |
|||
} |
|||
} |
|||
|
|||
protected override void DisposeObject(bool disposing) |
|||
{ |
|||
if (disposing) |
|||
{ |
|||
indexWriter?.Dispose(); |
|||
} |
|||
} |
|||
|
|||
private void RecreateIndexWriter() |
|||
{ |
|||
var config = new IndexWriterConfig(Version, Analyzer) |
|||
{ |
|||
IndexDeletionPolicy = snapshotter, |
|||
MergePolicy = new TieredMergePolicy(), |
|||
MergeScheduler = MergeScheduler |
|||
}; |
|||
|
|||
indexWriter = new IndexWriter(directory, config); |
|||
|
|||
MarkStale(); |
|||
} |
|||
|
|||
public void EnsureReader() |
|||
{ |
|||
ThrowIfDisposed(); |
|||
|
|||
if (indexReader == null) |
|||
{ |
|||
indexReader = indexWriter.GetReader(true); |
|||
indexSearcher = new IndexSearcher(indexReader); |
|||
} |
|||
} |
|||
|
|||
public void MarkStale() |
|||
{ |
|||
ThrowIfDisposed(); |
|||
|
|||
if (indexReader != null) |
|||
{ |
|||
indexReader.Dispose(); |
|||
indexReader = null; |
|||
indexSearcher = null; |
|||
} |
|||
} |
|||
|
|||
public async Task CommitAsync() |
|||
{ |
|||
ThrowIfDisposed(); |
|||
|
|||
try |
|||
{ |
|||
MarkStale(); |
|||
|
|||
indexWriter.Commit(); |
|||
|
|||
await directoryFactory.WriteAsync(indexWriter, snapshotter); |
|||
} |
|||
catch (OutOfMemoryException) |
|||
{ |
|||
RecreateIndexWriter(); |
|||
|
|||
throw; |
|||
} |
|||
} |
|||
} |
|||
} |
|||
@ -1,110 +0,0 @@ |
|||
// ==========================================================================
|
|||
// Squidex Headless CMS
|
|||
// ==========================================================================
|
|||
// Copyright (c) Squidex UG (haftungsbeschraenkt)
|
|||
// All rights reserved. Licensed under the MIT license.
|
|||
// ==========================================================================
|
|||
|
|||
using System; |
|||
using System.Collections.Generic; |
|||
using System.Threading; |
|||
using System.Threading.Tasks; |
|||
using Squidex.Infrastructure; |
|||
using Squidex.Infrastructure.Log; |
|||
|
|||
namespace Squidex.Domain.Apps.Entities.Contents.Text |
|||
{ |
|||
public sealed class IndexHolderFactory : DisposableObjectBase |
|||
{ |
|||
private readonly Dictionary<Guid, IndexHolder> indices = new Dictionary<Guid, IndexHolder>(); |
|||
private readonly SemaphoreSlim lockObject = new SemaphoreSlim(1); |
|||
private readonly IDirectoryFactory directoryFactory; |
|||
private readonly ISemanticLog log; |
|||
|
|||
public IndexHolderFactory(IDirectoryFactory directoryFactory, ISemanticLog log) |
|||
{ |
|||
Guard.NotNull(directoryFactory); |
|||
Guard.NotNull(log); |
|||
|
|||
this.directoryFactory = directoryFactory; |
|||
|
|||
this.log = log; |
|||
} |
|||
|
|||
protected override void DisposeObject(bool disposing) |
|||
{ |
|||
if (disposing) |
|||
{ |
|||
try |
|||
{ |
|||
lockObject.Wait(); |
|||
|
|||
if (indices.Count > 0) |
|||
{ |
|||
log.LogWarning(w => w |
|||
.WriteProperty("message", "Unreleased indices found.") |
|||
.WriteProperty("count", indices.Count)); |
|||
|
|||
foreach (var index in indices) |
|||
{ |
|||
index.Value.Dispose(); |
|||
} |
|||
|
|||
indices.Clear(); |
|||
} |
|||
} |
|||
finally |
|||
{ |
|||
lockObject.Release(); |
|||
} |
|||
} |
|||
} |
|||
|
|||
public async Task<IndexHolder> AcquireAsync(Guid schemaId) |
|||
{ |
|||
IndexHolder? index; |
|||
|
|||
try |
|||
{ |
|||
await lockObject.WaitAsync(); |
|||
|
|||
if (indices.TryGetValue(schemaId, out index)) |
|||
{ |
|||
log.LogWarning(w => w |
|||
.WriteProperty("message", "Unreleased index found.") |
|||
.WriteProperty("schemaId", schemaId.ToString())); |
|||
|
|||
index.Dispose(); |
|||
} |
|||
|
|||
var directory = await directoryFactory.CreateAsync(schemaId); |
|||
|
|||
index = new IndexHolder(directory, directoryFactory); |
|||
|
|||
indices[schemaId] = index; |
|||
} |
|||
finally |
|||
{ |
|||
lockObject.Release(); |
|||
} |
|||
|
|||
index.Open(); |
|||
|
|||
return index; |
|||
} |
|||
|
|||
public void Release(Guid id) |
|||
{ |
|||
try |
|||
{ |
|||
lockObject.Wait(); |
|||
|
|||
indices.Remove(id); |
|||
} |
|||
finally |
|||
{ |
|||
lockObject.Release(); |
|||
} |
|||
} |
|||
} |
|||
} |
|||
@ -0,0 +1,148 @@ |
|||
// ==========================================================================
|
|||
// Squidex Headless CMS
|
|||
// ==========================================================================
|
|||
// Copyright (c) Squidex UG (haftungsbeschraenkt)
|
|||
// All rights reserved. Licensed under the MIT license.
|
|||
// ==========================================================================
|
|||
|
|||
using System; |
|||
using System.Collections.Generic; |
|||
using System.Linq; |
|||
using System.Threading; |
|||
using System.Threading.Tasks; |
|||
using Squidex.Infrastructure; |
|||
using Squidex.Infrastructure.Log; |
|||
|
|||
namespace Squidex.Domain.Apps.Entities.Contents.Text |
|||
{ |
|||
public sealed partial class IndexManager : DisposableObjectBase |
|||
{ |
|||
private readonly Dictionary<Guid, IndexHolder> indices = new Dictionary<Guid, IndexHolder>(); |
|||
private readonly SemaphoreSlim lockObject = new SemaphoreSlim(1); |
|||
private readonly IIndexStorage directoryFactory; |
|||
private readonly ISemanticLog log; |
|||
|
|||
public IndexManager(IIndexStorage directoryFactory, ISemanticLog log) |
|||
{ |
|||
Guard.NotNull(directoryFactory); |
|||
Guard.NotNull(log); |
|||
|
|||
this.directoryFactory = directoryFactory; |
|||
|
|||
this.log = log; |
|||
} |
|||
|
|||
protected override void DisposeObject(bool disposing) |
|||
{ |
|||
if (disposing) |
|||
{ |
|||
ReleaseAllAsync().Wait(); |
|||
} |
|||
} |
|||
|
|||
public async Task<IIndex> AcquireAsync(Guid schemaId) |
|||
{ |
|||
IndexHolder? indexHolder; |
|||
|
|||
try |
|||
{ |
|||
await lockObject.WaitAsync(); |
|||
|
|||
if (indices.TryGetValue(schemaId, out indexHolder)) |
|||
{ |
|||
log.LogWarning(w => w |
|||
.WriteProperty("message", "Unreleased index found.") |
|||
.WriteProperty("schemaId", schemaId.ToString())); |
|||
|
|||
await CommitInternalAsync(indexHolder, true); |
|||
} |
|||
|
|||
indexHolder = new IndexHolder(schemaId); |
|||
indices[schemaId] = indexHolder; |
|||
} |
|||
finally |
|||
{ |
|||
lockObject.Release(); |
|||
} |
|||
|
|||
var directory = await directoryFactory.CreateDirectoryAsync(schemaId); |
|||
|
|||
indexHolder.Open(directory); |
|||
|
|||
return indexHolder; |
|||
} |
|||
|
|||
public async Task ReleaseAsync(IIndex index) |
|||
{ |
|||
Guard.NotNull(index); |
|||
|
|||
var indexHolder = (IndexHolder)index; |
|||
|
|||
try |
|||
{ |
|||
lockObject.Wait(); |
|||
|
|||
indexHolder.Release(); |
|||
indices.Remove(indexHolder.Id); |
|||
} |
|||
finally |
|||
{ |
|||
lockObject.Release(); |
|||
} |
|||
|
|||
await CommitInternalAsync(indexHolder, true); |
|||
} |
|||
|
|||
public Task CommitAsync(IIndex index) |
|||
{ |
|||
Guard.NotNull(index); |
|||
|
|||
return CommitInternalAsync(index, false); |
|||
} |
|||
|
|||
private async Task CommitInternalAsync(IIndex index, bool dispose) |
|||
{ |
|||
if (index is IndexHolder holder) |
|||
{ |
|||
if (dispose) |
|||
{ |
|||
holder.Dispose(); |
|||
} |
|||
else |
|||
{ |
|||
holder.Commit(); |
|||
} |
|||
|
|||
await directoryFactory.WriteAsync(holder.GetUnsafeWriter().Directory, holder.Snapshotter); |
|||
} |
|||
} |
|||
|
|||
private async Task ReleaseAllAsync() |
|||
{ |
|||
var current = indices.Values.ToList(); |
|||
|
|||
try |
|||
{ |
|||
lockObject.Wait(); |
|||
|
|||
indices.Clear(); |
|||
} |
|||
finally |
|||
{ |
|||
lockObject.Release(); |
|||
} |
|||
|
|||
if (current.Count > 0) |
|||
{ |
|||
log.LogWarning(w => w |
|||
.WriteProperty("message", "Unreleased indices found.") |
|||
.WriteProperty("count", indices.Count)); |
|||
|
|||
foreach (var index in current) |
|||
{ |
|||
await CommitInternalAsync(index, true); |
|||
} |
|||
} |
|||
} |
|||
} |
|||
} |
|||
@ -0,0 +1,176 @@ |
|||
// ==========================================================================
|
|||
// Squidex Headless CMS
|
|||
// ==========================================================================
|
|||
// Copyright (c) Squidex UG (haftungsbeschraenkt)
|
|||
// All rights reserved. Licensed under the MIT license.
|
|||
// ==========================================================================
|
|||
|
|||
using System; |
|||
using Lucene.Net.Analysis; |
|||
using Lucene.Net.Index; |
|||
using Lucene.Net.Search; |
|||
using Lucene.Net.Store; |
|||
using Lucene.Net.Util; |
|||
using Squidex.Infrastructure; |
|||
|
|||
namespace Squidex.Domain.Apps.Entities.Contents.Text |
|||
{ |
|||
public sealed partial class IndexManager |
|||
{ |
|||
private sealed class IndexHolder : IDisposable, IIndex |
|||
{ |
|||
private const LuceneVersion Version = LuceneVersion.LUCENE_48; |
|||
private static readonly MergeScheduler MergeScheduler = new ConcurrentMergeScheduler(); |
|||
private static readonly Analyzer SharedAnalyzer = new MultiLanguageAnalyzer(Version); |
|||
private readonly SnapshotDeletionPolicy snapshotter = new SnapshotDeletionPolicy(new KeepOnlyLastCommitDeletionPolicy()); |
|||
private Directory directory; |
|||
private IndexWriter indexWriter; |
|||
private IndexSearcher? indexSearcher; |
|||
private DirectoryReader? indexReader; |
|||
private bool isReleased; |
|||
|
|||
public Analyzer Analyzer |
|||
{ |
|||
get { return SharedAnalyzer; } |
|||
} |
|||
|
|||
public SnapshotDeletionPolicy Snapshotter |
|||
{ |
|||
get { return snapshotter; } |
|||
} |
|||
|
|||
public IndexWriter Writer |
|||
{ |
|||
get |
|||
{ |
|||
ThrowIfReleased(); |
|||
|
|||
return indexWriter; |
|||
} |
|||
} |
|||
|
|||
public IndexReader? Reader |
|||
{ |
|||
get |
|||
{ |
|||
ThrowIfReleased(); |
|||
|
|||
return indexReader; |
|||
} |
|||
} |
|||
|
|||
public IndexSearcher? Searcher |
|||
{ |
|||
get |
|||
{ |
|||
ThrowIfReleased(); |
|||
|
|||
return indexSearcher; |
|||
} |
|||
} |
|||
|
|||
public Guid Id { get; } |
|||
|
|||
public IndexHolder(Guid id) |
|||
{ |
|||
Id = id; |
|||
} |
|||
|
|||
public void Dispose() |
|||
{ |
|||
indexReader?.Dispose(); |
|||
indexWriter?.Dispose(); |
|||
} |
|||
|
|||
public void Open(Directory directory) |
|||
{ |
|||
Guard.NotNull(directory); |
|||
|
|||
this.directory = directory; |
|||
|
|||
RecreateIndexWriter(); |
|||
|
|||
if (indexWriter.NumDocs > 0) |
|||
{ |
|||
EnsureReader(); |
|||
} |
|||
} |
|||
|
|||
private void RecreateIndexWriter() |
|||
{ |
|||
var config = new IndexWriterConfig(Version, Analyzer) |
|||
{ |
|||
IndexDeletionPolicy = snapshotter, |
|||
MergePolicy = new TieredMergePolicy(), |
|||
MergeScheduler = MergeScheduler |
|||
}; |
|||
|
|||
indexWriter = new IndexWriter(directory, config); |
|||
|
|||
MarkStale(); |
|||
} |
|||
|
|||
public void EnsureReader() |
|||
{ |
|||
ThrowIfReleased(); |
|||
|
|||
if (indexReader == null) |
|||
{ |
|||
indexReader = indexWriter.GetReader(true); |
|||
indexSearcher = new IndexSearcher(indexReader); |
|||
} |
|||
} |
|||
|
|||
public void MarkStale() |
|||
{ |
|||
ThrowIfReleased(); |
|||
|
|||
MarkStaleInternal(); |
|||
} |
|||
|
|||
private void MarkStaleInternal() |
|||
{ |
|||
if (indexReader != null) |
|||
{ |
|||
indexReader.Dispose(); |
|||
indexReader = null; |
|||
indexSearcher = null; |
|||
} |
|||
} |
|||
|
|||
internal void Commit() |
|||
{ |
|||
try |
|||
{ |
|||
MarkStaleInternal(); |
|||
|
|||
indexWriter.Commit(); |
|||
} |
|||
catch (OutOfMemoryException) |
|||
{ |
|||
RecreateIndexWriter(); |
|||
|
|||
throw; |
|||
} |
|||
} |
|||
|
|||
internal void ThrowIfReleased() |
|||
{ |
|||
if (isReleased) |
|||
{ |
|||
throw new InvalidOperationException("Index is already released."); |
|||
} |
|||
} |
|||
|
|||
internal void Release() |
|||
{ |
|||
isReleased = true; |
|||
} |
|||
|
|||
internal IndexWriter GetUnsafeWriter() |
|||
{ |
|||
return indexWriter; |
|||
} |
|||
} |
|||
} |
|||
} |
|||
@ -0,0 +1,23 @@ |
|||
// ==========================================================================
|
|||
// Squidex Headless CMS
|
|||
// ==========================================================================
|
|||
// Copyright (c) Squidex UG (haftungsbeschraenkt)
|
|||
// All rights reserved. Licensed under the MIT license.
|
|||
// ==========================================================================
|
|||
|
|||
using Squidex.Infrastructure.Assets; |
|||
|
|||
namespace Squidex.Domain.Apps.Entities.Contents.Text |
|||
{ |
|||
public class TextIndexerGrainTests_Assets : TextIndexerGrainTestsBase |
|||
{ |
|||
public override IIndexStorage Storage { get; } = CreateStorage(); |
|||
|
|||
private static IIndexStorage CreateStorage() |
|||
{ |
|||
var storage = new AssetIndexStorage(new MemoryAssetStore()); |
|||
|
|||
return storage; |
|||
} |
|||
} |
|||
} |
|||
Loading…
Reference in new issue