mirror of https://github.com/Squidex/squidex.git
Sebastian Stehle
6 years ago
committed by
GitHub
GitHub
25 changed files with 1135 additions and 454 deletions
@ -0,0 +1,150 @@ |
|||
// ==========================================================================
|
|||
// Squidex Headless CMS
|
|||
// ==========================================================================
|
|||
// Copyright (c) Squidex UG (haftungsbeschraenkt)
|
|||
// All rights reserved. Licensed under the MIT license.
|
|||
// ==========================================================================
|
|||
|
|||
using System; |
|||
using System.Collections.Generic; |
|||
using System.IO; |
|||
using System.Linq; |
|||
using Lucene.Net.Store; |
|||
using MongoDB.Bson; |
|||
using MongoDB.Driver; |
|||
using MongoDB.Driver.GridFS; |
|||
using LuceneDirectory = Lucene.Net.Store.Directory; |
|||
|
|||
namespace Squidex.Domain.Apps.Entities.MongoDb.FullText |
|||
{ |
|||
public sealed class MongoDirectory : BaseDirectory |
|||
{ |
|||
private readonly IGridFSBucket<string> bucket; |
|||
private readonly string directory; |
|||
private readonly DirectoryInfo cacheDirectoryInfo; |
|||
private readonly LuceneDirectory cacheDirectory; |
|||
private bool isDisposed; |
|||
|
|||
public LuceneDirectory CacheDirectory |
|||
{ |
|||
get { return cacheDirectory; } |
|||
} |
|||
|
|||
public DirectoryInfo CacheDirectoryInfo |
|||
{ |
|||
get { return cacheDirectoryInfo; } |
|||
} |
|||
|
|||
public IGridFSBucket<string> Bucket |
|||
{ |
|||
get { return bucket; } |
|||
} |
|||
|
|||
public MongoDirectory(IGridFSBucket<string> bucket, string directory, DirectoryInfo cacheDirectoryInfo) |
|||
{ |
|||
this.bucket = bucket; |
|||
|
|||
this.directory = directory; |
|||
|
|||
this.cacheDirectoryInfo = cacheDirectoryInfo; |
|||
|
|||
cacheDirectoryInfo.Create(); |
|||
cacheDirectory = FSDirectory.Open(cacheDirectoryInfo); |
|||
|
|||
SetLockFactory(new NativeFSLockFactory(cacheDirectoryInfo)); |
|||
} |
|||
|
|||
protected override void Dispose(bool disposing) |
|||
{ |
|||
if (disposing) |
|||
{ |
|||
isDisposed = true; |
|||
|
|||
cacheDirectory.Dispose(); |
|||
} |
|||
} |
|||
|
|||
public override string GetLockID() |
|||
{ |
|||
return cacheDirectory.GetLockID(); |
|||
} |
|||
|
|||
public override IndexOutput CreateOutput(string name, IOContext context) |
|||
{ |
|||
return new MongoIndexOutput(this, context, name); |
|||
} |
|||
|
|||
public override IndexInput OpenInput(string name, IOContext context) |
|||
{ |
|||
return new MongoIndexInput(this, context, name); |
|||
} |
|||
|
|||
public override void DeleteFile(string name) |
|||
{ |
|||
EnsureNotDisposed(); |
|||
|
|||
var fullName = GetFullName(name); |
|||
|
|||
try |
|||
{ |
|||
Bucket.Delete(fullName); |
|||
} |
|||
catch (GridFSFileNotFoundException) |
|||
{ |
|||
} |
|||
} |
|||
|
|||
public override long FileLength(string name) |
|||
{ |
|||
EnsureNotDisposed(); |
|||
|
|||
var file = FindFile(name) ?? throw new FileNotFoundException(); |
|||
|
|||
return file.Length; |
|||
} |
|||
|
|||
public override string[] ListAll() |
|||
{ |
|||
EnsureNotDisposed(); |
|||
|
|||
var files = Bucket.Find(Builders<GridFSFileInfo<string>>.Filter.Regex(x => x.Id, new BsonRegularExpression($"^{directory}/"))).ToList(); |
|||
|
|||
return files.Select(x => x.Filename).ToArray(); |
|||
} |
|||
|
|||
public GridFSFileInfo<string>? FindFile(string name) |
|||
{ |
|||
var fullName = GetFullName(name); |
|||
|
|||
return Bucket.Find(Builders<GridFSFileInfo<string>>.Filter.Eq(x => x.Id, fullName)).FirstOrDefault(); |
|||
} |
|||
|
|||
public override void Sync(ICollection<string> names) |
|||
{ |
|||
} |
|||
|
|||
[Obsolete] |
|||
public override bool FileExists(string name) |
|||
{ |
|||
throw new NotSupportedException(); |
|||
} |
|||
|
|||
public string GetFullName(string name) |
|||
{ |
|||
return $"{directory}/{name}"; |
|||
} |
|||
|
|||
public string GetFullPath(string name) |
|||
{ |
|||
return Path.Combine(cacheDirectoryInfo.FullName, name); |
|||
} |
|||
|
|||
private void EnsureNotDisposed() |
|||
{ |
|||
if (isDisposed) |
|||
{ |
|||
throw new ObjectDisposedException(GetType().FullName); |
|||
} |
|||
} |
|||
} |
|||
} |
|||
@ -0,0 +1,35 @@ |
|||
// ==========================================================================
|
|||
// Squidex Headless CMS
|
|||
// ==========================================================================
|
|||
// Copyright (c) Squidex UG (haftungsbeschraenkt)
|
|||
// All rights reserved. Licensed under the MIT license.
|
|||
// ==========================================================================
|
|||
|
|||
using System; |
|||
using System.IO; |
|||
using MongoDB.Driver.GridFS; |
|||
using Squidex.Domain.Apps.Entities.Contents.Text; |
|||
using LuceneDirectory = Lucene.Net.Store.Directory; |
|||
|
|||
namespace Squidex.Domain.Apps.Entities.MongoDb.FullText |
|||
{ |
|||
public sealed class MongoDirectoryFactory : IDirectoryFactory |
|||
{ |
|||
private readonly IGridFSBucket<string> bucket; |
|||
|
|||
public MongoDirectoryFactory(IGridFSBucket<string> bucket) |
|||
{ |
|||
this.bucket = bucket; |
|||
} |
|||
|
|||
public LuceneDirectory Create(Guid schemaId) |
|||
{ |
|||
var folderName = schemaId.ToString(); |
|||
|
|||
var tempFolder = Path.Combine(Path.GetTempPath(), folderName); |
|||
var tempDirectory = new DirectoryInfo(tempFolder); |
|||
|
|||
return new MongoDirectory(bucket, folderName, tempDirectory); |
|||
} |
|||
} |
|||
} |
|||
@ -0,0 +1,105 @@ |
|||
// ==========================================================================
|
|||
// Squidex Headless CMS
|
|||
// ==========================================================================
|
|||
// Copyright (c) Squidex UG (haftungsbeschraenkt)
|
|||
// All rights reserved. Licensed under the MIT license.
|
|||
// ==========================================================================
|
|||
|
|||
using System.IO; |
|||
using Lucene.Net.Store; |
|||
using MongoDB.Driver.GridFS; |
|||
|
|||
namespace Squidex.Domain.Apps.Entities.MongoDb.FullText |
|||
{ |
|||
public sealed class MongoIndexInput : IndexInput |
|||
{ |
|||
private readonly IndexInput cacheInput; |
|||
private readonly MongoDirectory indexDirectory; |
|||
private readonly IOContext context; |
|||
private readonly string indexFileName; |
|||
|
|||
public override long Length |
|||
{ |
|||
get { return cacheInput.Length; } |
|||
} |
|||
|
|||
public MongoIndexInput(MongoDirectory indexDirectory, IOContext context, string indexFileName) |
|||
: base(indexDirectory.GetFullName(indexFileName)) |
|||
{ |
|||
this.indexDirectory = indexDirectory; |
|||
this.indexFileName = indexFileName; |
|||
|
|||
this.context = context; |
|||
|
|||
try |
|||
{ |
|||
var file = indexDirectory.FindFile(indexFileName); |
|||
|
|||
if (file != null) |
|||
{ |
|||
var fileInfo = new FileInfo(indexDirectory.GetFullPath(indexFileName)); |
|||
|
|||
var writtenTime = file.Metadata["WrittenTime"].ToUniversalTime(); |
|||
|
|||
if (!fileInfo.Exists || fileInfo.LastWriteTimeUtc < writtenTime) |
|||
{ |
|||
using (var fs = new FileStream(fileInfo.FullName, FileMode.Create, FileAccess.Write)) |
|||
{ |
|||
var fullName = indexDirectory.GetFullName(indexFileName); |
|||
|
|||
indexDirectory.Bucket.DownloadToStream(fullName, fs); |
|||
} |
|||
} |
|||
} |
|||
} |
|||
catch (GridFSFileNotFoundException) |
|||
{ |
|||
throw new FileNotFoundException(); |
|||
} |
|||
|
|||
cacheInput = indexDirectory.CacheDirectory.OpenInput(indexFileName, context); |
|||
} |
|||
|
|||
public MongoIndexInput(MongoIndexInput source) |
|||
: base("clone") |
|||
{ |
|||
cacheInput = (IndexInput)source.cacheInput.Clone(); |
|||
context = source.context; |
|||
indexDirectory = source.indexDirectory; |
|||
indexFileName = source.indexFileName; |
|||
} |
|||
|
|||
protected override void Dispose(bool disposing) |
|||
{ |
|||
if (disposing) |
|||
{ |
|||
cacheInput.Dispose(); |
|||
} |
|||
} |
|||
|
|||
public override long GetFilePointer() |
|||
{ |
|||
return cacheInput.GetFilePointer(); |
|||
} |
|||
|
|||
public override byte ReadByte() |
|||
{ |
|||
return cacheInput.ReadByte(); |
|||
} |
|||
|
|||
public override void ReadBytes(byte[] b, int offset, int len) |
|||
{ |
|||
cacheInput.ReadBytes(b, offset, len); |
|||
} |
|||
|
|||
public override void Seek(long pos) |
|||
{ |
|||
cacheInput.Seek(pos); |
|||
} |
|||
|
|||
public override object Clone() |
|||
{ |
|||
return new MongoIndexInput(indexDirectory, context, indexFileName); |
|||
} |
|||
} |
|||
} |
|||
@ -0,0 +1,114 @@ |
|||
// ==========================================================================
|
|||
// Squidex Headless CMS
|
|||
// ==========================================================================
|
|||
// Copyright (c) Squidex UG (haftungsbeschraenkt)
|
|||
// All rights reserved. Licensed under the MIT license.
|
|||
// ==========================================================================
|
|||
|
|||
using System; |
|||
using System.IO; |
|||
using System.Linq; |
|||
using Lucene.Net.Store; |
|||
using MongoDB.Bson; |
|||
using MongoDB.Driver; |
|||
using MongoDB.Driver.GridFS; |
|||
|
|||
namespace Squidex.Domain.Apps.Entities.MongoDb.FullText |
|||
{ |
|||
public sealed class MongoIndexOutput : IndexOutput |
|||
{ |
|||
private readonly IndexOutput cacheOutput; |
|||
private readonly MongoDirectory indexDirectory; |
|||
private readonly string indexFileName; |
|||
private bool isFlushed; |
|||
private bool isWritten; |
|||
|
|||
public override long Length |
|||
{ |
|||
get { return cacheOutput.Length; } |
|||
} |
|||
|
|||
public override long Checksum |
|||
{ |
|||
get { return cacheOutput.Checksum; } |
|||
} |
|||
|
|||
public MongoIndexOutput(MongoDirectory indexDirectory, IOContext context, string indexFileName) |
|||
{ |
|||
this.indexDirectory = indexDirectory; |
|||
this.indexFileName = indexFileName; |
|||
|
|||
cacheOutput = indexDirectory.CacheDirectory.CreateOutput(indexFileName, context); |
|||
} |
|||
|
|||
protected override void Dispose(bool disposing) |
|||
{ |
|||
if (disposing) |
|||
{ |
|||
Flush(); |
|||
|
|||
cacheOutput.Dispose(); |
|||
|
|||
if (isWritten && isFlushed) |
|||
{ |
|||
var fileInfo = new FileInfo(indexDirectory.GetFullPath(indexFileName)); |
|||
|
|||
using (var fs = new FileStream(indexDirectory.GetFullPath(indexFileName), FileMode.Open, FileAccess.Read)) |
|||
{ |
|||
var fullName = indexDirectory.GetFullName(indexFileName); |
|||
|
|||
var options = new GridFSUploadOptions |
|||
{ |
|||
Metadata = new BsonDocument |
|||
{ |
|||
["WrittenTime"] = fileInfo.LastWriteTimeUtc |
|||
} |
|||
}; |
|||
|
|||
try |
|||
{ |
|||
indexDirectory.Bucket.UploadFromStream(fullName, indexFileName, fs, options); |
|||
} |
|||
catch (MongoBulkWriteException ex) when (ex.WriteErrors.Any(x => x.Code == 11000)) |
|||
{ |
|||
indexDirectory.Bucket.Delete(fullName); |
|||
indexDirectory.Bucket.UploadFromStream(fullName, indexFileName, fs, options); |
|||
} |
|||
} |
|||
} |
|||
} |
|||
} |
|||
|
|||
public override long GetFilePointer() |
|||
{ |
|||
return cacheOutput.GetFilePointer(); |
|||
} |
|||
|
|||
public override void Flush() |
|||
{ |
|||
cacheOutput.Flush(); |
|||
|
|||
isFlushed = true; |
|||
} |
|||
|
|||
public override void WriteByte(byte b) |
|||
{ |
|||
cacheOutput.WriteByte(b); |
|||
|
|||
isWritten = true; |
|||
} |
|||
|
|||
public override void WriteBytes(byte[] b, int offset, int length) |
|||
{ |
|||
cacheOutput.WriteBytes(b, offset, length); |
|||
|
|||
isWritten = true; |
|||
} |
|||
|
|||
[Obsolete] |
|||
public override void Seek(long pos) |
|||
{ |
|||
cacheOutput.Seek(pos); |
|||
} |
|||
} |
|||
} |
|||
@ -0,0 +1,26 @@ |
|||
// ==========================================================================
|
|||
// Squidex Headless CMS
|
|||
// ==========================================================================
|
|||
// Copyright (c) Squidex UG (haftungsbeschraenkt)
|
|||
// All rights reserved. Licensed under the MIT license.
|
|||
// ==========================================================================
|
|||
|
|||
using System; |
|||
using System.IO; |
|||
using Lucene.Net.Store; |
|||
using LuceneDirectory = Lucene.Net.Store.Directory; |
|||
|
|||
namespace Squidex.Domain.Apps.Entities.Contents.Text |
|||
{ |
|||
public sealed class FSDirectoryFactory : IDirectoryFactory |
|||
{ |
|||
public LuceneDirectory Create(Guid schemaId) |
|||
{ |
|||
var folderName = $"Indexes/{schemaId}"; |
|||
|
|||
var tempFolder = Path.Combine(Path.GetTempPath(), folderName); |
|||
|
|||
return FSDirectory.Open(tempFolder); |
|||
} |
|||
} |
|||
} |
|||
@ -0,0 +1,17 @@ |
|||
// ==========================================================================
|
|||
// Squidex Headless CMS
|
|||
// ==========================================================================
|
|||
// Copyright (c) Squidex UG (haftungsbeschraenkt)
|
|||
// All rights reserved. Licensed under the MIT license.
|
|||
// ==========================================================================
|
|||
|
|||
using System; |
|||
using Lucene.Net.Store; |
|||
|
|||
namespace Squidex.Domain.Apps.Entities.Contents.Text |
|||
{ |
|||
public interface IDirectoryFactory |
|||
{ |
|||
Directory Create(Guid schemaId); |
|||
} |
|||
} |
|||
@ -0,0 +1,157 @@ |
|||
// ==========================================================================
|
|||
// Squidex Headless CMS
|
|||
// ==========================================================================
|
|||
// Copyright (c) Squidex UG (haftungsbeschraenkt)
|
|||
// All rights reserved. Licensed under the MIT license.
|
|||
// ==========================================================================
|
|||
|
|||
using System; |
|||
using Lucene.Net.Analysis; |
|||
using Lucene.Net.Index; |
|||
using Lucene.Net.Search; |
|||
using Lucene.Net.Store; |
|||
using Lucene.Net.Util; |
|||
using Squidex.Infrastructure; |
|||
|
|||
namespace Squidex.Domain.Apps.Entities.Contents.Text |
|||
{ |
|||
public sealed class IndexHolder : DisposableObjectBase |
|||
{ |
|||
private const LuceneVersion Version = LuceneVersion.LUCENE_48; |
|||
private static readonly MergeScheduler MergeScheduler = new ConcurrentMergeScheduler(); |
|||
private static readonly Analyzer SharedAnalyzer = new MultiLanguageAnalyzer(Version); |
|||
private readonly SnapshotDeletionPolicy snapshotter = new SnapshotDeletionPolicy(new KeepOnlyLastCommitDeletionPolicy()); |
|||
private readonly Directory directory; |
|||
private IndexWriter indexWriter; |
|||
private IndexSearcher? indexSearcher; |
|||
private DirectoryReader? indexReader; |
|||
|
|||
public Analyzer Analyzer |
|||
{ |
|||
get |
|||
{ |
|||
ThrowIfDisposed(); |
|||
|
|||
return SharedAnalyzer; |
|||
} |
|||
} |
|||
|
|||
public SnapshotDeletionPolicy Snapshotter |
|||
{ |
|||
get |
|||
{ |
|||
ThrowIfDisposed(); |
|||
|
|||
return snapshotter; |
|||
} |
|||
} |
|||
|
|||
public IndexWriter Writer |
|||
{ |
|||
get |
|||
{ |
|||
ThrowIfDisposed(); |
|||
|
|||
return indexWriter; |
|||
} |
|||
} |
|||
|
|||
public IndexReader? Reader |
|||
{ |
|||
get |
|||
{ |
|||
ThrowIfDisposed(); |
|||
|
|||
return indexReader; |
|||
} |
|||
} |
|||
|
|||
public IndexSearcher? Searcher |
|||
{ |
|||
get |
|||
{ |
|||
ThrowIfDisposed(); |
|||
|
|||
return indexSearcher; |
|||
} |
|||
} |
|||
|
|||
public IndexHolder(IDirectoryFactory directoryFactory, Guid schemaId) |
|||
{ |
|||
directory = directoryFactory.Create(schemaId); |
|||
} |
|||
|
|||
public void Open() |
|||
{ |
|||
RecreateIndexWriter(); |
|||
|
|||
if (indexWriter.NumDocs > 0) |
|||
{ |
|||
EnsureReader(); |
|||
} |
|||
} |
|||
|
|||
protected override void DisposeObject(bool disposing) |
|||
{ |
|||
if (disposing) |
|||
{ |
|||
indexWriter.Dispose(); |
|||
} |
|||
} |
|||
|
|||
private void RecreateIndexWriter() |
|||
{ |
|||
var config = new IndexWriterConfig(Version, Analyzer) |
|||
{ |
|||
IndexDeletionPolicy = snapshotter, |
|||
MergePolicy = new TieredMergePolicy(), |
|||
MergeScheduler = MergeScheduler |
|||
}; |
|||
|
|||
indexWriter = new IndexWriter(directory, config); |
|||
|
|||
MarkStale(); |
|||
} |
|||
|
|||
public void EnsureReader() |
|||
{ |
|||
ThrowIfDisposed(); |
|||
|
|||
if (indexReader == null) |
|||
{ |
|||
indexReader = indexWriter.GetReader(true); |
|||
indexSearcher = new IndexSearcher(indexReader); |
|||
} |
|||
} |
|||
|
|||
public void MarkStale() |
|||
{ |
|||
ThrowIfDisposed(); |
|||
|
|||
if (indexReader != null) |
|||
{ |
|||
indexReader.Dispose(); |
|||
indexReader = null; |
|||
indexSearcher = null; |
|||
} |
|||
} |
|||
|
|||
public void Commit() |
|||
{ |
|||
ThrowIfDisposed(); |
|||
|
|||
try |
|||
{ |
|||
MarkStale(); |
|||
|
|||
indexWriter.Commit(); |
|||
} |
|||
catch (OutOfMemoryException) |
|||
{ |
|||
RecreateIndexWriter(); |
|||
|
|||
throw; |
|||
} |
|||
} |
|||
} |
|||
} |
|||
@ -0,0 +1,87 @@ |
|||
// ==========================================================================
|
|||
// Squidex Headless CMS
|
|||
// ==========================================================================
|
|||
// Copyright (c) Squidex UG (haftungsbeschraenkt)
|
|||
// All rights reserved. Licensed under the MIT license.
|
|||
// ==========================================================================
|
|||
|
|||
using System; |
|||
using System.Collections.Generic; |
|||
using Squidex.Infrastructure; |
|||
using Squidex.Infrastructure.Log; |
|||
|
|||
namespace Squidex.Domain.Apps.Entities.Contents.Text |
|||
{ |
|||
public sealed class IndexHolderFactory : DisposableObjectBase |
|||
{ |
|||
private readonly Dictionary<Guid, IndexHolder> indices = new Dictionary<Guid, IndexHolder>(); |
|||
private readonly IDirectoryFactory directoryFactory; |
|||
private readonly ISemanticLog log; |
|||
|
|||
public IndexHolderFactory(IDirectoryFactory directoryFactory, ISemanticLog log) |
|||
{ |
|||
Guard.NotNull(directoryFactory); |
|||
Guard.NotNull(log); |
|||
|
|||
this.directoryFactory = directoryFactory; |
|||
|
|||
this.log = log; |
|||
} |
|||
|
|||
protected override void DisposeObject(bool disposing) |
|||
{ |
|||
if (disposing) |
|||
{ |
|||
lock (indices) |
|||
{ |
|||
if (indices.Count > 0) |
|||
{ |
|||
log.LogWarning(w => w |
|||
.WriteProperty("message", "Unreleased indices found.") |
|||
.WriteProperty("count", indices.Count)); |
|||
|
|||
foreach (var index in indices) |
|||
{ |
|||
index.Value.Dispose(); |
|||
} |
|||
|
|||
indices.Clear(); |
|||
} |
|||
} |
|||
} |
|||
} |
|||
|
|||
public IndexHolder Acquire(Guid schemaId) |
|||
{ |
|||
IndexHolder? index; |
|||
|
|||
lock (indices) |
|||
{ |
|||
if (indices.TryGetValue(schemaId, out index)) |
|||
{ |
|||
log.LogWarning(w => w |
|||
.WriteProperty("message", "Unreleased index found.") |
|||
.WriteProperty("schemaId", schemaId.ToString())); |
|||
|
|||
index.Dispose(); |
|||
} |
|||
|
|||
index = new IndexHolder(directoryFactory, schemaId); |
|||
|
|||
indices[schemaId] = index; |
|||
} |
|||
|
|||
index.Open(); |
|||
|
|||
return index; |
|||
} |
|||
|
|||
public void Release(Guid id) |
|||
{ |
|||
lock (indices) |
|||
{ |
|||
indices.Remove(id); |
|||
} |
|||
} |
|||
} |
|||
} |
|||
@ -1,94 +0,0 @@ |
|||
// ==========================================================================
|
|||
// Squidex Headless CMS
|
|||
// ==========================================================================
|
|||
// Copyright (c) Squidex UG (haftungsbeschraenkt)
|
|||
// All rights reserved. Licensed under the MIT license.
|
|||
// ==========================================================================
|
|||
|
|||
using System; |
|||
using System.IO; |
|||
using System.IO.Compression; |
|||
using System.Threading.Tasks; |
|||
using Lucene.Net.Index; |
|||
using Squidex.Infrastructure.Assets; |
|||
|
|||
namespace Squidex.Domain.Apps.Entities.Contents.Text |
|||
{ |
|||
public static class PersistenceHelper |
|||
{ |
|||
private const string ArchiveFile = "Archive.zip"; |
|||
private const string LockFile = "write.lock"; |
|||
|
|||
public static async Task UploadDirectoryAsync(this IAssetStore assetStore, DirectoryInfo directory, IndexCommit commit) |
|||
{ |
|||
using (var fileStream = new FileStream( |
|||
Path.Combine(directory.FullName, ArchiveFile), |
|||
FileMode.Create, |
|||
FileAccess.ReadWrite, |
|||
FileShare.None, |
|||
4096, |
|||
FileOptions.DeleteOnClose)) |
|||
{ |
|||
using (var zipArchive = new ZipArchive(fileStream, ZipArchiveMode.Create, true)) |
|||
{ |
|||
foreach (var fileName in commit.FileNames) |
|||
{ |
|||
var file = new FileInfo(Path.Combine(directory.FullName, fileName)); |
|||
|
|||
try |
|||
{ |
|||
if (!file.Name.Equals(ArchiveFile, StringComparison.OrdinalIgnoreCase) && |
|||
!file.Name.Equals(LockFile, StringComparison.OrdinalIgnoreCase)) |
|||
{ |
|||
zipArchive.CreateEntryFromFile(file.FullName, file.Name); |
|||
} |
|||
} |
|||
catch (IOException) |
|||
{ |
|||
continue; |
|||
} |
|||
} |
|||
} |
|||
|
|||
fileStream.Position = 0; |
|||
|
|||
await assetStore.UploadAsync(directory.Name, 0, string.Empty, fileStream, true); |
|||
} |
|||
} |
|||
|
|||
public static async Task DownloadAsync(this IAssetStore assetStore, DirectoryInfo directory) |
|||
{ |
|||
if (directory.Exists) |
|||
{ |
|||
directory.Delete(true); |
|||
} |
|||
|
|||
directory.Create(); |
|||
|
|||
using (var fileStream = new FileStream( |
|||
Path.Combine(directory.FullName, ArchiveFile), |
|||
FileMode.Create, |
|||
FileAccess.ReadWrite, |
|||
FileShare.None, |
|||
4096, |
|||
FileOptions.DeleteOnClose)) |
|||
{ |
|||
try |
|||
{ |
|||
await assetStore.DownloadAsync(directory.Name, 0, string.Empty, fileStream); |
|||
|
|||
fileStream.Position = 0; |
|||
|
|||
using (var zipArchive = new ZipArchive(fileStream, ZipArchiveMode.Read, true)) |
|||
{ |
|||
zipArchive.ExtractToDirectory(directory.FullName); |
|||
} |
|||
} |
|||
catch (AssetNotFoundException) |
|||
{ |
|||
return; |
|||
} |
|||
} |
|||
} |
|||
} |
|||
} |
|||
@ -0,0 +1,91 @@ |
|||
// ==========================================================================
|
|||
// Squidex Headless CMS
|
|||
// ==========================================================================
|
|||
// Copyright (c) Squidex UG (haftungsbeschraenkt)
|
|||
// All rights reserved. Licensed under the MIT license.
|
|||
// ==========================================================================
|
|||
|
|||
using System; |
|||
using System.Collections.Generic; |
|||
using System.Text; |
|||
using Squidex.Domain.Apps.Core.Contents; |
|||
using Squidex.Infrastructure; |
|||
using Squidex.Infrastructure.Json.Objects; |
|||
|
|||
#pragma warning disable ORL1001
|
|||
|
|||
namespace Squidex.Domain.Apps.Entities.Contents.Text |
|||
{ |
|||
[Serializable] |
|||
public sealed class TextContent : Dictionary<string, string> |
|||
{ |
|||
public TextContent() |
|||
{ |
|||
} |
|||
|
|||
public TextContent(NamedContentData data) |
|||
{ |
|||
if (data == null) |
|||
{ |
|||
return; |
|||
} |
|||
|
|||
var languages = new Dictionary<string, StringBuilder>(); |
|||
|
|||
void AppendText(string language, string text) |
|||
{ |
|||
if (!string.IsNullOrWhiteSpace(text)) |
|||
{ |
|||
var sb = languages.GetOrAddNew(language); |
|||
|
|||
if (sb.Length > 0) |
|||
{ |
|||
sb.Append(" "); |
|||
} |
|||
|
|||
sb.Append(text); |
|||
} |
|||
} |
|||
|
|||
foreach (var field in data) |
|||
{ |
|||
if (field.Value != null) |
|||
{ |
|||
foreach (var fieldValue in field.Value) |
|||
{ |
|||
var appendText = new Action<string>(text => AppendText(fieldValue.Key, text)); |
|||
|
|||
AppendJsonText(fieldValue.Value, appendText); |
|||
} |
|||
} |
|||
} |
|||
|
|||
foreach (var kvp in languages) |
|||
{ |
|||
this[kvp.Key] = kvp.Value.ToString(); |
|||
} |
|||
} |
|||
|
|||
private static void AppendJsonText(IJsonValue value, Action<string> appendText) |
|||
{ |
|||
if (value.Type == JsonValueType.String) |
|||
{ |
|||
appendText(value.ToString()); |
|||
} |
|||
else if (value is JsonArray array) |
|||
{ |
|||
foreach (var item in array) |
|||
{ |
|||
AppendJsonText(item, appendText); |
|||
} |
|||
} |
|||
else if (value is JsonObject obj) |
|||
{ |
|||
foreach (var item in obj.Values) |
|||
{ |
|||
AppendJsonText(item, appendText); |
|||
} |
|||
} |
|||
} |
|||
} |
|||
} |
|||
@ -0,0 +1,59 @@ |
|||
// ==========================================================================
|
|||
// Squidex Headless CMS
|
|||
// ==========================================================================
|
|||
// Copyright (c) Squidex UG (haftungsbeschraenkt)
|
|||
// All rights reserved. Licensed under the MIT license.
|
|||
// ==========================================================================
|
|||
|
|||
using System; |
|||
using System.Threading.Tasks; |
|||
using FakeItEasy; |
|||
using Squidex.Infrastructure; |
|||
using Squidex.Infrastructure.Log; |
|||
using Xunit; |
|||
|
|||
namespace Squidex.Domain.Apps.Entities.Contents.Text |
|||
{ |
|||
public class TextIndexerBenchmark |
|||
{ |
|||
private readonly Guid schemaId = Guid.NewGuid(); |
|||
private readonly TextIndexerGrain sut; |
|||
|
|||
public TextIndexerBenchmark() |
|||
{ |
|||
var factory = new IndexHolderFactory(new FSDirectoryFactory(), A.Fake<ISemanticLog>()); |
|||
|
|||
sut = new TextIndexerGrain(factory); |
|||
sut.ActivateAsync(schemaId).Wait(); |
|||
} |
|||
|
|||
[Fact(Skip = "Only used for benchmarks")] |
|||
public async Task Should_index_many_documents() |
|||
{ |
|||
var text = new TextContent |
|||
{ |
|||
["iv"] = "Hallo Welt" |
|||
}; |
|||
|
|||
var ids = new Guid[10000]; |
|||
|
|||
for (var i = 0; i < ids.Length; i++) |
|||
{ |
|||
ids[i] = Guid.NewGuid(); |
|||
} |
|||
|
|||
var watch = ValueStopwatch.StartNew(); |
|||
|
|||
foreach (var id in ids) |
|||
{ |
|||
await sut.IndexAsync(new Update { Text = text, Id = id }); |
|||
} |
|||
|
|||
sut.OnDeactivateAsync().Wait(); |
|||
|
|||
var elapsed = watch.Stop(); |
|||
|
|||
Assert.InRange(elapsed, 0, 1); |
|||
} |
|||
} |
|||
} |
|||
@ -0,0 +1,21 @@ |
|||
// ==========================================================================
|
|||
// Squidex Headless CMS
|
|||
// ==========================================================================
|
|||
// Copyright (c) Squidex UG (haftungsbeschraenkt)
|
|||
// All rights reserved. Licensed under the MIT license.
|
|||
// ==========================================================================
|
|||
|
|||
namespace Squidex.Domain.Apps.Entities.Contents.Text |
|||
{ |
|||
public class TextIndexerGrainTests_FS : TextIndexerGrainTestsBase |
|||
{ |
|||
public override IDirectoryFactory DirectoryFactory => CreateFactory(); |
|||
|
|||
private static IDirectoryFactory CreateFactory() |
|||
{ |
|||
var directoryFactory = new FSDirectoryFactory(); |
|||
|
|||
return directoryFactory; |
|||
} |
|||
} |
|||
} |
|||
@ -0,0 +1,33 @@ |
|||
// ==========================================================================
|
|||
// Squidex Headless CMS
|
|||
// ==========================================================================
|
|||
// Copyright (c) Squidex UG (haftungsbeschraenkt)
|
|||
// All rights reserved. Licensed under the MIT license.
|
|||
// ==========================================================================
|
|||
|
|||
using MongoDB.Driver; |
|||
using MongoDB.Driver.GridFS; |
|||
using Squidex.Domain.Apps.Entities.MongoDb.FullText; |
|||
|
|||
namespace Squidex.Domain.Apps.Entities.Contents.Text |
|||
{ |
|||
internal class TextIndexerGrainTests_Mongo : TextIndexerGrainTestsBase |
|||
{ |
|||
public override IDirectoryFactory DirectoryFactory => CreateFactory(); |
|||
|
|||
private static IDirectoryFactory CreateFactory() |
|||
{ |
|||
var mongoClient = new MongoClient("mongodb://localhost"); |
|||
var mongoDatabase = mongoClient.GetDatabase("FullText"); |
|||
|
|||
var mongoBucket = new GridFSBucket<string>(mongoDatabase, new GridFSBucketOptions |
|||
{ |
|||
BucketName = "fs" |
|||
}); |
|||
|
|||
var directoryFactory = new MongoDirectoryFactory(mongoBucket); |
|||
|
|||
return directoryFactory; |
|||
} |
|||
} |
|||
} |
|||
Loading…
Reference in new issue