diff --git a/src/Squidex.Domain.Apps.Entities/Backup/BackupGrain.cs b/src/Squidex.Domain.Apps.Entities/Backup/BackupGrain.cs index f1bff2c44..fe6a484ec 100644 --- a/src/Squidex.Domain.Apps.Entities/Backup/BackupGrain.cs +++ b/src/Squidex.Domain.Apps.Entities/Backup/BackupGrain.cs @@ -164,7 +164,7 @@ namespace Squidex.Domain.Apps.Entities.Backup currentTask.Token.ThrowIfCancellationRequested(); - await assetStore.UploadAsync(job.Id.ToString(), 0, null, stream, currentTask.Token); + await assetStore.UploadAsync(job.Id.ToString(), 0, null, stream, false, currentTask.Token); } job.Status = JobStatus.Completed; diff --git a/src/Squidex.Domain.Apps.Entities/Contents/Text/MultiLanguageAnalyzer.cs b/src/Squidex.Domain.Apps.Entities/Contents/Text/MultiLanguageAnalyzer.cs new file mode 100644 index 000000000..8bd41df9c --- /dev/null +++ b/src/Squidex.Domain.Apps.Entities/Contents/Text/MultiLanguageAnalyzer.cs @@ -0,0 +1,65 @@ +// ========================================================================== +// Squidex Headless CMS +// ========================================================================== +// Copyright (c) Squidex UG (haftungsbeschraenkt) +// All rights reserved. Licensed under the MIT license. +// ========================================================================== + +using System; +using System.Collections.Generic; +using System.Linq; +using Lucene.Net.Analysis; +using Lucene.Net.Analysis.Standard; +using Lucene.Net.Util; +using Squidex.Infrastructure; + +namespace Squidex.Domain.Apps.Entities.Contents.Text +{ + public sealed class MultiLanguageAnalyzer : AnalyzerWrapper + { + private readonly StandardAnalyzer fallbackAnalyzer; + private readonly Dictionary analyzers = new Dictionary(StringComparer.OrdinalIgnoreCase); + + public MultiLanguageAnalyzer(LuceneVersion version) + : base(PER_FIELD_REUSE_STRATEGY) + { + fallbackAnalyzer = new StandardAnalyzer(version); + + foreach (var type in typeof(StandardAnalyzer).Assembly.GetTypes()) + { + if (typeof(Analyzer).IsAssignableFrom(type)) + { + var language = type.Namespace.Split('.').Last(); + + if (language.Length == 2) + { + try + { + var analyzer = Activator.CreateInstance(type, version); + + analyzers[language] = (Analyzer)analyzer; + } + catch (MissingMethodException) + { + continue; + } + } + } + } + } + + protected override Analyzer GetWrappedAnalyzer(string fieldName) + { + if (fieldName.Length > 0) + { + var analyzer = analyzers.GetOrDefault(fieldName.Substring(0, 2)) ?? fallbackAnalyzer; + + return analyzer; + } + else + { + return fallbackAnalyzer; + } + } + } +} diff --git a/src/Squidex.Domain.Apps.Entities/Contents/Text/PersistenceHelper.cs b/src/Squidex.Domain.Apps.Entities/Contents/Text/PersistenceHelper.cs new file mode 100644 index 000000000..49424c856 --- /dev/null +++ b/src/Squidex.Domain.Apps.Entities/Contents/Text/PersistenceHelper.cs @@ -0,0 +1,90 @@ +// ========================================================================== +// Squidex Headless CMS +// ========================================================================== +// Copyright (c) Squidex UG (haftungsbeschraenkt) +// All rights reserved. Licensed under the MIT license. +// ========================================================================== + +using System; +using System.IO; +using System.IO.Compression; +using System.Threading.Tasks; +using Squidex.Infrastructure.Assets; + +namespace Squidex.Domain.Apps.Entities.Contents.Text +{ + public static class PersistenceHelper + { + private const string ArchiveFile = "Archive.zip"; + private const string LockFile = "write.lock"; + + public static async Task UploadDirectoryAsync(this IAssetStore assetStore, DirectoryInfo directory) + { + using (var fileStream = new FileStream( + Path.Combine(directory.FullName, ArchiveFile), + FileMode.Create, + FileAccess.ReadWrite, + FileShare.None, + 4096, + FileOptions.DeleteOnClose)) + { + using (var zipArchive = new ZipArchive(fileStream, ZipArchiveMode.Create, true)) + { + foreach (var file in directory.GetFiles()) + { + try + { + if (!file.Name.Equals(ArchiveFile, StringComparison.OrdinalIgnoreCase) && !file.Name.Equals(LockFile, StringComparison.OrdinalIgnoreCase)) + { + zipArchive.CreateEntryFromFile(file.FullName, file.Name); + } + } + catch (IOException) + { + continue; + } + } + } + + fileStream.Position = 0; + + await assetStore.UploadAsync(directory.Name, 0, string.Empty, fileStream, true); + } + } + + public static async Task DownloadAsync(this IAssetStore assetStore, DirectoryInfo directory) + { + if (directory.Exists) + { + directory.Delete(true); + } + + directory.Create(); + + using (var fileStream = new FileStream( + Path.Combine(directory.FullName, ArchiveFile), + FileMode.Create, + FileAccess.ReadWrite, + FileShare.None, + 4096, + FileOptions.DeleteOnClose)) + { + try + { + await assetStore.DownloadAsync(directory.Name, 0, string.Empty, fileStream); + + fileStream.Position = 0; + + using (var zipArchive = new ZipArchive(fileStream, ZipArchiveMode.Read, true)) + { + zipArchive.ExtractToDirectory(directory.FullName); + } + } + catch (AssetNotFoundException) + { + return; + } + } + } + } +} diff --git a/src/Squidex.Domain.Apps.Entities/Contents/Text/TextIndexer.cs b/src/Squidex.Domain.Apps.Entities/Contents/Text/TextIndexer.cs new file mode 100644 index 000000000..db309826f --- /dev/null +++ b/src/Squidex.Domain.Apps.Entities/Contents/Text/TextIndexer.cs @@ -0,0 +1,47 @@ +// ========================================================================== +// Squidex Headless CMS +// ========================================================================== +// Copyright (c) Squidex UG (haftungsbeschraenkt) +// All rights reserved. Licensed under the MIT license. +// ========================================================================== + +using System.Threading.Tasks; +using Squidex.Domain.Apps.Events.Contents; +using Squidex.Infrastructure.EventSourcing; +using Squidex.Infrastructure.Tasks; + +namespace Squidex.Domain.Apps.Entities.Contents.Text +{ + public sealed class TextIndexer : IEventConsumer + { + public string Name + { + get { return GetType().Name; } + } + + public string EventsFilter + { + get { return "^content-"; } + } + + public Task ClearAsync() + { + return TaskHelper.Done; + } + + public Task On(Envelope @event) + { + switch (@event.Payload) + { + case ContentCreated contentCreated: + break; + case ContentUpdated contentUpdated: + break; + case ContentUpdateProposed contentUpdateProposed: + break; + } + + return Task.CompletedTask; + } + } +} diff --git a/src/Squidex.Domain.Apps.Entities/Contents/Text/TextIndexerGrain.cs b/src/Squidex.Domain.Apps.Entities/Contents/Text/TextIndexerGrain.cs new file mode 100644 index 000000000..119df813f --- /dev/null +++ b/src/Squidex.Domain.Apps.Entities/Contents/Text/TextIndexerGrain.cs @@ -0,0 +1,268 @@ +// ========================================================================== +// Squidex Headless CMS +// ========================================================================== +// Copyright (c) Squidex UG (haftungsbeschraenkt) +// All rights reserved. Licensed under the MIT license. +// ========================================================================== + +using System; +using System.Collections.Generic; +using System.IO; +using System.Threading.Tasks; +using Lucene.Net.Analysis; +using Lucene.Net.Analysis.Standard; +using Lucene.Net.Documents; +using Lucene.Net.Index; +using Lucene.Net.QueryParsers.Classic; +using Lucene.Net.Search; +using Lucene.Net.Store; +using Lucene.Net.Util; +using Squidex.Domain.Apps.Core; +using Squidex.Domain.Apps.Core.Contents; +using Squidex.Domain.Apps.Core.Schemas; +using Squidex.Infrastructure; +using Squidex.Infrastructure.Assets; +using Squidex.Infrastructure.Json.Objects; +using Squidex.Infrastructure.Orleans; + +namespace Squidex.Domain.Apps.Entities.Contents.Text +{ + public sealed class TextIndexerGrain : GrainOfGuid + { + private const LuceneVersion Version = LuceneVersion.LUCENE_48; + private const int MaxResults = 2000; + private const int MaxUpdates = 100; + private static readonly HashSet IdFields = new HashSet(); + private static readonly Analyzer Analyzer = new MultiLanguageAnalyzer(Version); + private readonly IAssetStore assetStore; + private DirectoryInfo directory; + private IndexWriter indexWriter; + private IndexReader indexReader; + private QueryParser queryParser; + private int currentAppVersion; + private int currentSchemaVersion; + private int updates; + + public TextIndexerGrain(IAssetStore assetStore) + { + Guard.NotNull(assetStore, nameof(assetStore)); + + this.assetStore = assetStore; + } + + public override Task OnActivateAsync() + { + RegisterTimer(_ => FlushAsync(), null, TimeSpan.Zero, TimeSpan.FromMinutes(10)); + + return base.OnActivateAsync(); + } + + public override async Task OnDeactivateAsync() + { + await DeactivateAsync(true); + } + + protected override async Task OnActivateAsync(Guid key) + { + directory = new DirectoryInfo(Path.Combine(Path.GetTempPath(), $"Index_{key}")); + + await assetStore.DownloadAsync(directory); + + indexWriter = new IndexWriter(FSDirectory.Open(directory), new IndexWriterConfig(Version, Analyzer)); + indexReader = indexWriter.GetReader(true); + } + + public Task DeleteContentAsync(Guid id) + { + indexWriter.DeleteDocuments(new Term("id", id.ToString())); + + return TryFlushAsync(); + } + + public Task AddContentAsync(Guid id, NamedContentData data, bool isUpdate, bool isDraft) + { + var idString = id.ToString(); + + if (isUpdate) + { + indexWriter.DeleteDocuments(new Term("id", idString)); + } + + var document = new Document(); + + document.AddStringField("id", idString, Field.Store.YES); + document.AddInt32Field("draft", isDraft ? 1 : 0, Field.Store.YES); + + foreach (var field in data) + { + foreach (var fieldValue in field.Value) + { + var value = fieldValue.Value; + + if (value.Type == JsonValueType.String) + { + var fieldName = BuildFieldName(fieldValue.Key, field.Key); + + document.AddTextField(fieldName, fieldValue.Value.ToString(), Field.Store.YES); + } + else if (value.Type == JsonValueType.Object) + { + foreach (var property in (JsonObject)value) + { + if (property.Value.Type == JsonValueType.String) + { + var fieldName = BuildFieldName(fieldValue.Key, field.Key, property.Key); + + document.AddTextField(fieldName, property.Value.ToString(), Field.Store.YES); + } + } + } + } + } + + indexWriter.AddDocument(document); + + return TryFlushAsync(); + } + + public Task> SearchAsync(string term, int appVersion, int schemaVersion, J schema, List languages) + { + var query = BuildQuery(term, appVersion, schemaVersion, schema, languages); + + var result = new List(); + + if (indexReader != null) + { + var hits = new IndexSearcher(indexReader).Search(query, MaxResults).ScoreDocs; + + foreach (var hit in hits) + { + var document = indexReader.Document(hit.Doc); + + var idField = document.GetField("id")?.GetStringValue(); + + if (idField != null && Guid.TryParse(idField, out var guid)) + { + result.Add(guid); + } + } + } + + return Task.FromResult(result); + } + + private Query BuildQuery(string query, int appVersion, int schemaVersion, J schema, List language) + { + if (queryParser == null || currentAppVersion != appVersion || currentSchemaVersion != schemaVersion) + { + var fields = BuildFields(schema, language); + + queryParser = new MultiFieldQueryParser(Version, fields, Analyzer); + + currentAppVersion = appVersion; + currentSchemaVersion = schemaVersion; + } + + return queryParser.Parse(query); + } + + private string[] BuildFields(Schema schema, IEnumerable languages) + { + var fieldNames = new List(); + + var iv = InvariantPartitioning.Instance.Master.Key; + + foreach (var field in schema.Fields) + { + if (field.RawProperties is StringFieldProperties) + { + if (field.Partitioning.Equals(Partitioning.Invariant)) + { + fieldNames.Add(BuildFieldName(iv, field.Name)); + } + else + { + foreach (var language in languages) + { + fieldNames.Add(BuildFieldName(language, field.Name)); + } + } + } + else if (field is IArrayField arrayField) + { + foreach (var nested in arrayField.Fields) + { + if (nested.RawProperties is StringFieldProperties) + { + if (field.Partitioning.Equals(Partitioning.Invariant)) + { + fieldNames.Add(BuildFieldName(iv, field.Name, nested.Name)); + } + else + { + foreach (var language in languages) + { + fieldNames.Add(BuildFieldName(language, field.Name, nested.Name)); + } + } + } + } + } + } + + return fieldNames.ToArray(); + } + + private async Task TryFlushAsync() + { + updates++; + + if (updates >= MaxUpdates) + { + await FlushAsync(); + } + } + + public async Task FlushAsync() + { + if (updates > 0 && indexWriter != null) + { + indexWriter.Flush(true, true); + indexWriter.Commit(); + + indexReader?.Dispose(); + indexReader = indexWriter.GetReader(true); + + await assetStore.UploadDirectoryAsync(directory); + + updates = 0; + } + } + + public async Task DeactivateAsync(bool deleteFolder = false) + { + await TryFlushAsync(); + + indexWriter?.Dispose(); + indexWriter = null; + + indexReader?.Dispose(); + indexReader = null; + + if (deleteFolder && directory.Exists) + { + directory.Delete(true); + } + } + + private static string BuildFieldName(string language, string name) + { + return $"{language}_{name}"; + } + + private static string BuildFieldName(string language, string name, string nested) + { + return $"{language}_{name}_{nested}"; + } + } +} diff --git a/src/Squidex.Domain.Apps.Entities/Squidex.Domain.Apps.Entities.csproj b/src/Squidex.Domain.Apps.Entities/Squidex.Domain.Apps.Entities.csproj index 6adb6a190..b076b00cc 100644 --- a/src/Squidex.Domain.Apps.Entities/Squidex.Domain.Apps.Entities.csproj +++ b/src/Squidex.Domain.Apps.Entities/Squidex.Domain.Apps.Entities.csproj @@ -16,6 +16,9 @@ + + + all runtime; build; native; contentfiles; analyzers diff --git a/src/Squidex.Infrastructure.Azure/Assets/AzureBlobAssetStore.cs b/src/Squidex.Infrastructure.Azure/Assets/AzureBlobAssetStore.cs index 291fda481..e09a88879 100644 --- a/src/Squidex.Infrastructure.Azure/Assets/AzureBlobAssetStore.cs +++ b/src/Squidex.Infrastructure.Azure/Assets/AzureBlobAssetStore.cs @@ -110,14 +110,14 @@ namespace Squidex.Infrastructure.Assets } } - public Task UploadAsync(string id, long version, string suffix, Stream stream, CancellationToken ct = default) + public Task UploadAsync(string id, long version, string suffix, Stream stream, bool overwrite = false, CancellationToken ct = default) { - return UploadCoreAsync(GetObjectName(id, version, suffix), stream, ct); + return UploadCoreAsync(GetObjectName(id, version, suffix), stream, overwrite, ct); } public Task UploadAsync(string fileName, Stream stream, CancellationToken ct = default) { - return UploadCoreAsync(fileName, stream, ct); + return UploadCoreAsync(fileName, stream, false, ct); } public Task DeleteAsync(string id, long version, string suffix) @@ -137,13 +137,13 @@ namespace Squidex.Infrastructure.Assets return blob.DeleteIfExistsAsync(); } - private async Task UploadCoreAsync(string blobName, Stream stream, CancellationToken ct = default) + private async Task UploadCoreAsync(string blobName, Stream stream, bool overwrite = false, CancellationToken ct = default) { try { var tempBlob = blobContainer.GetBlockBlobReference(blobName); - await tempBlob.UploadFromStreamAsync(stream, AccessCondition.GenerateIfNotExistsCondition(), null, null, ct); + await tempBlob.UploadFromStreamAsync(stream, overwrite ? null : AccessCondition.GenerateIfNotExistsCondition(), null, null, ct); } catch (StorageException ex) when (ex.RequestInformation.HttpStatusCode == 409) { diff --git a/src/Squidex.Infrastructure.Azure/EventSourcing/CosmosDbEventStore_Writer.cs b/src/Squidex.Infrastructure.Azure/EventSourcing/CosmosDbEventStore_Writer.cs index ac0024573..45144e56e 100644 --- a/src/Squidex.Infrastructure.Azure/EventSourcing/CosmosDbEventStore_Writer.cs +++ b/src/Squidex.Infrastructure.Azure/EventSourcing/CosmosDbEventStore_Writer.cs @@ -7,7 +7,6 @@ using System; using System.Collections.Generic; -using System.Linq; using System.Net; using System.Threading.Tasks; using Microsoft.Azure.Documents; diff --git a/src/Squidex.Infrastructure.GoogleCloud/Assets/GoogleCloudAssetStore.cs b/src/Squidex.Infrastructure.GoogleCloud/Assets/GoogleCloudAssetStore.cs index e76b0d884..f76589938 100644 --- a/src/Squidex.Infrastructure.GoogleCloud/Assets/GoogleCloudAssetStore.cs +++ b/src/Squidex.Infrastructure.GoogleCloud/Assets/GoogleCloudAssetStore.cs @@ -80,14 +80,14 @@ namespace Squidex.Infrastructure.Assets } } - public Task UploadAsync(string id, long version, string suffix, Stream stream, CancellationToken ct = default) + public Task UploadAsync(string id, long version, string suffix, Stream stream, bool overwrite = false, CancellationToken ct = default) { - return UploadCoreAsync(GetObjectName(id, version, suffix), stream, ct); + return UploadCoreAsync(GetObjectName(id, version, suffix), stream, overwrite, ct); } public Task UploadAsync(string fileName, Stream stream, CancellationToken ct = default) { - return UploadCoreAsync(fileName, stream, ct); + return UploadCoreAsync(fileName, stream, false, ct); } public Task DeleteAsync(string id, long version, string suffix) @@ -100,11 +100,11 @@ namespace Squidex.Infrastructure.Assets return DeleteCoreAsync(fileName); } - private async Task UploadCoreAsync(string objectName, Stream stream, CancellationToken ct = default) + private async Task UploadCoreAsync(string objectName, Stream stream, bool overwrite = false, CancellationToken ct = default) { try { - await storageClient.UploadObjectAsync(bucketName, objectName, "application/octet-stream", stream, IfNotExists, ct); + await storageClient.UploadObjectAsync(bucketName, objectName, "application/octet-stream", stream, overwrite ? null : IfNotExists, ct); } catch (GoogleApiException ex) when (ex.HttpStatusCode == HttpStatusCode.PreconditionFailed) { diff --git a/src/Squidex.Infrastructure.MongoDb/Assets/MongoGridFsAssetStore.cs b/src/Squidex.Infrastructure.MongoDb/Assets/MongoGridFsAssetStore.cs index 6956bff4c..997fd9068 100644 --- a/src/Squidex.Infrastructure.MongoDb/Assets/MongoGridFsAssetStore.cs +++ b/src/Squidex.Infrastructure.MongoDb/Assets/MongoGridFsAssetStore.cs @@ -52,7 +52,7 @@ namespace Squidex.Infrastructure.Assets using (var readStream = await bucket.OpenDownloadStreamAsync(sourceFileName, cancellationToken: ct)) { - await UploadFileCoreAsync(target, readStream, ct); + await UploadFileCoreAsync(target, readStream, false, ct); } } catch (GridFSFileNotFoundException ex) @@ -78,14 +78,14 @@ namespace Squidex.Infrastructure.Assets } } - public Task UploadAsync(string id, long version, string suffix, Stream stream, CancellationToken ct = default) + public Task UploadAsync(string id, long version, string suffix, Stream stream, bool overwrite = false, CancellationToken ct = default) { - return UploadFileCoreAsync(GetFileName(id, version, suffix), stream, ct); + return UploadFileCoreAsync(GetFileName(id, version, suffix), stream, overwrite, ct); } public Task UploadAsync(string fileName, Stream stream, CancellationToken ct = default) { - return UploadFileCoreAsync(fileName, stream, ct); + return UploadFileCoreAsync(fileName, stream, false, ct); } public Task DeleteAsync(string id, long version, string suffix) @@ -110,10 +110,15 @@ namespace Squidex.Infrastructure.Assets } } - private async Task UploadFileCoreAsync(string id, Stream stream, CancellationToken ct = default) + private async Task UploadFileCoreAsync(string id, Stream stream, bool overwrite = false, CancellationToken ct = default) { try { + if (overwrite) + { + await bucket.DeleteAsync(id, ct); + } + await bucket.UploadFromStreamAsync(id, id, stream, cancellationToken: ct); } catch (MongoWriteException ex) when (ex.WriteError.Category == ServerErrorCategory.DuplicateKey) diff --git a/src/Squidex.Infrastructure/Assets/AssetAlreadyExistsException.cs b/src/Squidex.Infrastructure/Assets/AssetAlreadyExistsException.cs index 954f26c4c..ddf8465e0 100644 --- a/src/Squidex.Infrastructure/Assets/AssetAlreadyExistsException.cs +++ b/src/Squidex.Infrastructure/Assets/AssetAlreadyExistsException.cs @@ -32,7 +32,7 @@ namespace Squidex.Infrastructure.Assets { Guard.NotNullOrEmpty(fileName, nameof(fileName)); - return $"An asset with name '{fileName}' already not exists."; + return $"An asset with name '{fileName}' already exists."; } } } diff --git a/src/Squidex.Infrastructure/Assets/FolderAssetStore.cs b/src/Squidex.Infrastructure/Assets/FolderAssetStore.cs index 3c00b6ea8..a61e24366 100644 --- a/src/Squidex.Infrastructure/Assets/FolderAssetStore.cs +++ b/src/Squidex.Infrastructure/Assets/FolderAssetStore.cs @@ -95,14 +95,14 @@ namespace Squidex.Infrastructure.Assets } } - public Task UploadAsync(string id, long version, string suffix, Stream stream, CancellationToken ct = default) + public Task UploadAsync(string id, long version, string suffix, Stream stream, bool overwrite = false, CancellationToken ct = default) { - return UploadCoreAsync(GetFile(id, version, suffix), stream, ct); + return UploadCoreAsync(GetFile(id, version, suffix), stream, overwrite, ct); } public Task UploadAsync(string fileName, Stream stream, CancellationToken ct = default) { - return UploadCoreAsync(GetFile(fileName), stream, ct); + return UploadCoreAsync(GetFile(fileName), stream, false, ct); } public Task DeleteAsync(string id, long version, string suffix) @@ -122,11 +122,11 @@ namespace Squidex.Infrastructure.Assets return TaskHelper.Done; } - private static async Task UploadCoreAsync(FileInfo file, Stream stream, CancellationToken ct = default) + private static async Task UploadCoreAsync(FileInfo file, Stream stream, bool overwrite = false, CancellationToken ct = default) { try { - using (var fileStream = file.Open(FileMode.CreateNew, FileAccess.Write)) + using (var fileStream = file.Open(overwrite ? FileMode.Create : FileMode.CreateNew, FileAccess.Write)) { await stream.CopyToAsync(fileStream, BufferSize, ct); } diff --git a/src/Squidex.Infrastructure/Assets/IAssetStore.cs b/src/Squidex.Infrastructure/Assets/IAssetStore.cs index b4170bfff..65d3c4f84 100644 --- a/src/Squidex.Infrastructure/Assets/IAssetStore.cs +++ b/src/Squidex.Infrastructure/Assets/IAssetStore.cs @@ -21,7 +21,7 @@ namespace Squidex.Infrastructure.Assets Task UploadAsync(string fileName, Stream stream, CancellationToken ct = default); - Task UploadAsync(string id, long version, string suffix, Stream stream, CancellationToken ct = default); + Task UploadAsync(string id, long version, string suffix, Stream stream, bool overwrite = false, CancellationToken ct = default); Task DeleteAsync(string fileName); diff --git a/src/Squidex.Infrastructure/Assets/MemoryAssetStore.cs b/src/Squidex.Infrastructure/Assets/MemoryAssetStore.cs index 7c00f9a85..bc3b8804e 100644 --- a/src/Squidex.Infrastructure/Assets/MemoryAssetStore.cs +++ b/src/Squidex.Infrastructure/Assets/MemoryAssetStore.cs @@ -36,7 +36,7 @@ namespace Squidex.Infrastructure.Assets using (await readerLock.LockAsync()) { - await UploadAsync(id, version, suffix, sourceStream, ct); + await UploadAsync(id, version, suffix, sourceStream, false, ct); } } @@ -64,18 +64,23 @@ namespace Squidex.Infrastructure.Assets } } - public Task UploadAsync(string id, long version, string suffix, Stream stream, CancellationToken ct = default) + public Task UploadAsync(string id, long version, string suffix, Stream stream, bool overwrite = false, CancellationToken ct = default) { Guard.NotNullOrEmpty(id, nameof(id)); - return UploadAsync(GetFileName(id, version, suffix), stream, ct); + return UploadCoreAsync(GetFileName(id, version, suffix), stream, overwrite, ct); } - public async Task UploadAsync(string fileName, Stream stream, CancellationToken ct = default) + public Task UploadAsync(string fileName, Stream stream, CancellationToken ct = default) + { + return UploadCoreAsync(fileName, stream, false); + } + + private async Task UploadCoreAsync(string fileName, Stream stream, bool overwrite, CancellationToken ct = default) { var memoryStream = new MemoryStream(); - if (streams.TryAdd(fileName, memoryStream)) + async Task CopyAsync() { using (await writerLock.LockAsync()) { @@ -89,6 +94,17 @@ namespace Squidex.Infrastructure.Assets } } } + + if (overwrite) + { + await CopyAsync(); + + streams[fileName] = memoryStream; + } + else if (streams.TryAdd(fileName, memoryStream)) + { + await CopyAsync(); + } else { throw new AssetAlreadyExistsException(fileName); diff --git a/src/Squidex.Infrastructure/Assets/NoopAssetStore.cs b/src/Squidex.Infrastructure/Assets/NoopAssetStore.cs index 5c98cd77a..b48eb7269 100644 --- a/src/Squidex.Infrastructure/Assets/NoopAssetStore.cs +++ b/src/Squidex.Infrastructure/Assets/NoopAssetStore.cs @@ -34,7 +34,7 @@ namespace Squidex.Infrastructure.Assets throw new NotSupportedException(); } - public Task UploadAsync(string id, long version, string suffix, Stream stream, CancellationToken ct = default) + public Task UploadAsync(string id, long version, string suffix, Stream stream, bool overwrite = false, CancellationToken ct = default) { throw new NotSupportedException(); } diff --git a/src/Squidex.Infrastructure/Orleans/GrainOfGuid.cs b/src/Squidex.Infrastructure/Orleans/GrainOfGuid.cs index 0a816c381..a179c78d8 100644 --- a/src/Squidex.Infrastructure/Orleans/GrainOfGuid.cs +++ b/src/Squidex.Infrastructure/Orleans/GrainOfGuid.cs @@ -16,7 +16,7 @@ namespace Squidex.Infrastructure.Orleans { public Guid Key { get; private set; } - public sealed override Task OnActivateAsync() + public override Task OnActivateAsync() { return ActivateAsync(this.GetPrimaryKey()); } diff --git a/src/Squidex.Infrastructure/Orleans/GrainOfString.cs b/src/Squidex.Infrastructure/Orleans/GrainOfString.cs index 041f67df5..13d737ae0 100644 --- a/src/Squidex.Infrastructure/Orleans/GrainOfString.cs +++ b/src/Squidex.Infrastructure/Orleans/GrainOfString.cs @@ -15,7 +15,7 @@ namespace Squidex.Infrastructure.Orleans { public string Key { get; private set; } - public sealed override Task OnActivateAsync() + public override Task OnActivateAsync() { return ActivateAsync(this.GetPrimaryKeyString()); } diff --git a/tests/Squidex.Domain.Apps.Entities.Tests/Contents/Text/TextIndexerGrainTests.cs b/tests/Squidex.Domain.Apps.Entities.Tests/Contents/Text/TextIndexerGrainTests.cs new file mode 100644 index 000000000..ba4c93d1c --- /dev/null +++ b/tests/Squidex.Domain.Apps.Entities.Tests/Contents/Text/TextIndexerGrainTests.cs @@ -0,0 +1,165 @@ +// ========================================================================== +// Squidex Headless CMS +// ========================================================================== +// Copyright (c) Squidex UG (haftungsbeschraenkt) +// All rights reserved. Licensed under the MIT license. +// ========================================================================== + +using System; +using System.Collections.Generic; +using System.Threading.Tasks; +using Squidex.Domain.Apps.Core; +using Squidex.Domain.Apps.Core.Contents; +using Squidex.Domain.Apps.Core.Schemas; +using Squidex.Infrastructure.Assets; +using Xunit; + +namespace Squidex.Domain.Apps.Entities.Contents.Text +{ + public class TextIndexerGrainTests : IDisposable + { + private readonly Schema schema = + new Schema("test") + .AddString(1, "test", Partitioning.Invariant) + .AddString(2, "localized", Partitioning.Language); + private readonly List languages = new List { "de", "en" }; + private readonly Guid schemaId = Guid.NewGuid(); + private readonly List ids1 = new List { Guid.NewGuid() }; + private readonly List ids2 = new List { Guid.NewGuid() }; + private readonly IAssetStore assetStore = new MemoryAssetStore(); + private readonly TextIndexerGrain sut; + + public TextIndexerGrainTests() + { + sut = new TextIndexerGrain(assetStore); + sut.ActivateAsync(schemaId).Wait(); + } + + public void Dispose() + { + sut.OnDeactivateAsync().Wait(); + } + + [Fact] + public async Task Should_read_index_and_retrieve() + { + await AddInvariantContent(); + + await sut.DeactivateAsync(); + + var other = new TextIndexerGrain(assetStore); + try + { + await other.ActivateAsync(schemaId); + + var helloIds = await other.SearchAsync("Hello", 0, 0, schema, languages); + + Assert.Equal(ids1, helloIds); + + var worldIds = await other.SearchAsync("World", 0, 0, schema, languages); + + Assert.Equal(ids2, worldIds); + } + finally + { + await other.OnDeactivateAsync(); + } + } + + [Fact] + public async Task Should_index_invariant_content_and_retrieve() + { + await AddInvariantContent(); + + var helloIds = await sut.SearchAsync("Hello", 0, 0, schema, languages); + + Assert.Equal(ids1, helloIds); + + var worldIds = await sut.SearchAsync("World", 0, 0, schema, languages); + + Assert.Equal(ids2, worldIds); + } + + [Fact] + public async Task Should_delete_documents_from_index() + { + await AddInvariantContent(); + + await sut.DeleteContentAsync(ids1[0]); + await sut.FlushAsync(); + + var helloIds = await sut.SearchAsync("Hello", 0, 0, schema, languages); + + Assert.Empty(helloIds); + + var worldIds = await sut.SearchAsync("World", 0, 0, schema, languages); + + Assert.Equal(ids2, worldIds); + } + + [Fact] + public async Task Should_index_localized_content_and_retrieve() + { + await AddLocalizedContent(); + + var german1 = await sut.SearchAsync("Stadt", 0, 0, schema, languages); + var german2 = await sut.SearchAsync("and", 0, 0, schema, languages); + + var germanStopwordsIds = await sut.SearchAsync("und", 0, 0, schema, languages); + + Assert.Equal(ids1, german1); + Assert.Equal(ids1, german2); + + Assert.Equal(ids2, germanStopwordsIds); + + var english1 = await sut.SearchAsync("City", 0, 0, schema, languages); + var english2 = await sut.SearchAsync("und", 0, 0, schema, languages); + + var englishStopwordsIds = await sut.SearchAsync("and", 0, 0, schema, languages); + + Assert.Equal(ids2, english1); + Assert.Equal(ids2, english2); + + Assert.Equal(ids1, englishStopwordsIds); + } + + private async Task AddLocalizedContent() + { + var germanData = + new NamedContentData() + .AddField("localized", + new ContentFieldData() + .AddValue("de", "Stadt und Umgebung and whatever")); + + var englishData = + new NamedContentData() + .AddField("localized", + new ContentFieldData() + .AddValue("en", "City and Surroundings und sonstiges")); + + await sut.AddContentAsync(ids1[0], germanData, false, false); + await sut.AddContentAsync(ids2[0], englishData, false, false); + await sut.FlushAsync(); + } + + private async Task AddInvariantContent() + { + var data1 = + new NamedContentData() + .AddField("test", + new ContentFieldData() + .AddValue("iv", "Hello")); + + var data2 = + new NamedContentData() + .AddField("test", + new ContentFieldData() + .AddValue("iv", "World")); + + await sut.AddContentAsync(ids1[0], data1, false, false); + await sut.AddContentAsync(ids2[0], data2, false, false); + + await sut.FlushAsync(); + } + } +} diff --git a/tests/Squidex.Infrastructure.Tests/Assets/AssetStoreTests.cs b/tests/Squidex.Infrastructure.Tests/Assets/AssetStoreTests.cs index 40202902e..080a68d64 100644 --- a/tests/Squidex.Infrastructure.Tests/Assets/AssetStoreTests.cs +++ b/tests/Squidex.Infrastructure.Tests/Assets/AssetStoreTests.cs @@ -60,6 +60,19 @@ namespace Squidex.Infrastructure.Assets Assert.Equal(assetData.ToArray(), readData.ToArray()); } + [Fact] + public async Task Should_read_and_override_file() + { + await Sut.UploadAsync(assetId, 1, "suffix", new MemoryStream(new byte[] { 0x3, 0x4, 0x5, 0x6 })); + await Sut.UploadAsync(assetId, 1, "suffix", assetData, true); + + var readData = new MemoryStream(); + + await Sut.DownloadAsync(assetId, 1, "suffix", readData); + + Assert.Equal(assetData.ToArray(), readData.ToArray()); + } + [Fact] public async Task Should_throw_exception_when_file_to_write_already_exists() {