mirror of https://github.com/Squidex/squidex.git
Browse Source
* Always add scope. * Storage performance. * Storage improvements. * Fix tests. * Assets counts. * Fix for deletion. * Simplify deletion. * Fix bulk insert. * Added migration for total collection. * More flexible total calculation. * Fix reload. * Updates * Introduce persistence action. * Fix tests * Fix tests * Improve queries. * Improved mapping. * Temp * Revert count collection. * Easier count cache.pull/891/head
Sebastian Stehle
4 years ago
committed by
GitHub
GitHub
82 changed files with 1374 additions and 514 deletions
@ -0,0 +1,93 @@ |
|||||
|
// ==========================================================================
|
||||
|
// Squidex Headless CMS
|
||||
|
// ==========================================================================
|
||||
|
// Copyright (c) Squidex UG (haftungsbeschraenkt)
|
||||
|
// All rights reserved. Licensed under the MIT license.
|
||||
|
// ==========================================================================
|
||||
|
|
||||
|
using MongoDB.Driver; |
||||
|
using NodaTime; |
||||
|
using Squidex.Infrastructure; |
||||
|
using Squidex.Infrastructure.MongoDb; |
||||
|
using Squidex.Infrastructure.Tasks; |
||||
|
|
||||
|
namespace Squidex.Domain.Apps.Entities.MongoDb |
||||
|
{ |
||||
|
internal sealed class MongoCountCollection : MongoRepositoryBase<MongoCountEntity> |
||||
|
{ |
||||
|
private readonly string name; |
||||
|
|
||||
|
public MongoCountCollection(IMongoDatabase database, string name) |
||||
|
: base(database) |
||||
|
{ |
||||
|
this.name = $"{name}_Count"; |
||||
|
|
||||
|
InitializeAsync(default).Wait(); |
||||
|
} |
||||
|
|
||||
|
protected override string CollectionName() |
||||
|
{ |
||||
|
return name; |
||||
|
} |
||||
|
|
||||
|
public Task<long> GetOrAddAsync(DomainId key, Func<CancellationToken, Task<long>> provider, |
||||
|
CancellationToken ct) |
||||
|
{ |
||||
|
return GetOrAddAsync(key.ToString(), provider, ct); |
||||
|
} |
||||
|
|
||||
|
public async Task<long> GetOrAddAsync(string key, Func<CancellationToken, Task<long>> provider, |
||||
|
CancellationToken ct) |
||||
|
{ |
||||
|
var (cachedTotal, isOutdated) = await CountAsync(key, ct); |
||||
|
|
||||
|
if (cachedTotal < 5_000) |
||||
|
{ |
||||
|
return await RefreshTotalAsync(key, cachedTotal, provider, ct); |
||||
|
} |
||||
|
|
||||
|
if (isOutdated) |
||||
|
{ |
||||
|
// If we have a loot of items, the query might be slow and therefore we execute it in the background.
|
||||
|
RefreshTotalAsync(key, cachedTotal, provider, ct).Forget(); |
||||
|
} |
||||
|
|
||||
|
return cachedTotal; |
||||
|
} |
||||
|
|
||||
|
private async Task<long> RefreshTotalAsync(string key, long cachedCount, Func<CancellationToken, Task<long>> provider, |
||||
|
CancellationToken ct) |
||||
|
{ |
||||
|
var actualCount = await provider(ct); |
||||
|
|
||||
|
if (actualCount != cachedCount) |
||||
|
{ |
||||
|
var now = SystemClock.Instance.GetCurrentInstant(); |
||||
|
|
||||
|
await Collection.UpdateOneAsync(x => x.Key == key, |
||||
|
Update |
||||
|
.Set(x => x.Key, key) |
||||
|
.SetOnInsert(x => x.Count, actualCount) |
||||
|
.SetOnInsert(x => x.Created, now), |
||||
|
Upsert, ct); |
||||
|
} |
||||
|
|
||||
|
return actualCount; |
||||
|
} |
||||
|
|
||||
|
private async Task<(long, bool)> CountAsync(string key, |
||||
|
CancellationToken ct) |
||||
|
{ |
||||
|
var entity = await Collection.Find(x => x.Key == key).FirstOrDefaultAsync(ct); |
||||
|
|
||||
|
if (entity != null) |
||||
|
{ |
||||
|
var now = SystemClock.Instance.GetCurrentInstant(); |
||||
|
|
||||
|
return (entity.Count, now - entity.Created > Duration.FromSeconds(10)); |
||||
|
} |
||||
|
|
||||
|
return (0, true); |
||||
|
} |
||||
|
} |
||||
|
} |
||||
@ -0,0 +1,25 @@ |
|||||
|
// ==========================================================================
|
||||
|
// Squidex Headless CMS
|
||||
|
// ==========================================================================
|
||||
|
// Copyright (c) Squidex UG (haftungsbeschraenkt)
|
||||
|
// All rights reserved. Licensed under the MIT license.
|
||||
|
// ==========================================================================
|
||||
|
|
||||
|
using MongoDB.Bson.Serialization.Attributes; |
||||
|
using NodaTime; |
||||
|
|
||||
|
namespace Squidex.Domain.Apps.Entities.MongoDb |
||||
|
{ |
||||
|
internal sealed class MongoCountEntity |
||||
|
{ |
||||
|
[BsonId] |
||||
|
[BsonRequired] |
||||
|
public string Key { get; set; } |
||||
|
|
||||
|
[BsonElement] |
||||
|
public long Count { get; set; } |
||||
|
|
||||
|
[BsonElement] |
||||
|
public Instant Created { get; set; } |
||||
|
} |
||||
|
} |
||||
@ -0,0 +1,89 @@ |
|||||
|
// ==========================================================================
|
||||
|
// Squidex Headless CMS
|
||||
|
// ==========================================================================
|
||||
|
// Copyright (c) Squidex UG (haftungsbeschraenkt)
|
||||
|
// All rights reserved. Licensed under the MIT license.
|
||||
|
// ==========================================================================
|
||||
|
|
||||
|
using FluentAssertions; |
||||
|
using NodaTime; |
||||
|
using Squidex.Domain.Apps.Core.Assets; |
||||
|
using Squidex.Domain.Apps.Entities.Assets.DomainObject; |
||||
|
using Squidex.Domain.Apps.Entities.MongoDb.Assets; |
||||
|
using Squidex.Infrastructure; |
||||
|
using Squidex.Infrastructure.States; |
||||
|
using Xunit; |
||||
|
|
||||
|
namespace Squidex.Domain.Apps.Entities.Assets.MongoDb |
||||
|
{ |
||||
|
public class AssetMappingTests |
||||
|
{ |
||||
|
[Fact] |
||||
|
public void Should_map_asset() |
||||
|
{ |
||||
|
var user = new RefToken(RefTokenType.Subject, "1"); |
||||
|
|
||||
|
var time = SystemClock.Instance.GetCurrentInstant(); |
||||
|
|
||||
|
var source = new AssetDomainObject.State |
||||
|
{ |
||||
|
Id = DomainId.NewGuid(), |
||||
|
AppId = NamedId.Of(DomainId.NewGuid(), "my-app"), |
||||
|
Created = time, |
||||
|
CreatedBy = user, |
||||
|
FileHash = "my-hash", |
||||
|
FileName = "my-image.png", |
||||
|
FileSize = 1024, |
||||
|
FileVersion = 13, |
||||
|
IsDeleted = true, |
||||
|
IsProtected = true, |
||||
|
LastModified = time, |
||||
|
LastModifiedBy = user, |
||||
|
Metadata = new AssetMetadata().SetPixelHeight(600), |
||||
|
MimeType = "image/png", |
||||
|
ParentId = DomainId.NewGuid(), |
||||
|
Slug = "my-image", |
||||
|
Tags = new HashSet<string> { "image" }, |
||||
|
TotalSize = 1024 * 2, |
||||
|
Type = AssetType.Image, |
||||
|
Version = 42, |
||||
|
}; |
||||
|
|
||||
|
var snapshotJob = new SnapshotWriteJob<AssetDomainObject.State>(source.UniqueId, source, source.Version); |
||||
|
var snapshot = MongoAssetEntity.Create(snapshotJob); |
||||
|
|
||||
|
var mapped = snapshot.ToState(); |
||||
|
|
||||
|
mapped.Should().BeEquivalentTo(source); |
||||
|
} |
||||
|
|
||||
|
[Fact] |
||||
|
public void Should_map_asset_folder() |
||||
|
{ |
||||
|
var user = new RefToken(RefTokenType.Subject, "1"); |
||||
|
|
||||
|
var time = SystemClock.Instance.GetCurrentInstant(); |
||||
|
|
||||
|
var source = new AssetFolderDomainObject.State |
||||
|
{ |
||||
|
Id = DomainId.NewGuid(), |
||||
|
AppId = NamedId.Of(DomainId.NewGuid(), "my-app"), |
||||
|
Created = time, |
||||
|
CreatedBy = user, |
||||
|
FolderName = "my-folder", |
||||
|
IsDeleted = true, |
||||
|
LastModified = time, |
||||
|
LastModifiedBy = user, |
||||
|
ParentId = DomainId.NewGuid(), |
||||
|
Version = 42, |
||||
|
}; |
||||
|
|
||||
|
var snapshotJob = new SnapshotWriteJob<AssetFolderDomainObject.State>(source.UniqueId, source, source.Version); |
||||
|
var snapshot = MongoAssetFolderEntity.Create(snapshotJob); |
||||
|
|
||||
|
var mapped = snapshot.ToState(); |
||||
|
|
||||
|
mapped.Should().BeEquivalentTo(source); |
||||
|
} |
||||
|
} |
||||
|
} |
||||
@ -0,0 +1,159 @@ |
|||||
|
// ==========================================================================
|
||||
|
// Squidex Headless CMS
|
||||
|
// ==========================================================================
|
||||
|
// Copyright (c) Squidex UG (haftungsbeschraenkt)
|
||||
|
// All rights reserved. Licensed under the MIT license.
|
||||
|
// ==========================================================================
|
||||
|
|
||||
|
using FakeItEasy; |
||||
|
using FluentAssertions; |
||||
|
using NodaTime; |
||||
|
using Squidex.Domain.Apps.Core.Contents; |
||||
|
using Squidex.Domain.Apps.Entities.Contents.DomainObject; |
||||
|
using Squidex.Domain.Apps.Entities.MongoDb.Contents; |
||||
|
using Squidex.Infrastructure; |
||||
|
using Squidex.Infrastructure.States; |
||||
|
using Xunit; |
||||
|
|
||||
|
namespace Squidex.Domain.Apps.Entities.Contents.MongoDb |
||||
|
{ |
||||
|
public class ContentMappingTests |
||||
|
{ |
||||
|
private readonly IAppProvider appProvider = A.Fake<IAppProvider>(); |
||||
|
|
||||
|
[Fact] |
||||
|
public async Task Should_map_content_without_new_version_to_draft() |
||||
|
{ |
||||
|
var source = CreateContentWithoutNewVersion(); |
||||
|
|
||||
|
var snapshotJob = new SnapshotWriteJob<ContentDomainObject.State>(source.UniqueId, source, source.Version); |
||||
|
var snapshot = await MongoContentEntity.CreateDraftAsync(snapshotJob, appProvider); |
||||
|
|
||||
|
Assert.Equal(source.CurrentVersion.Data, snapshot.Data); |
||||
|
Assert.Null(snapshot.DraftData); |
||||
|
Assert.Null(snapshot.NewStatus); |
||||
|
Assert.NotNull(snapshot.ScheduleJob); |
||||
|
Assert.True(snapshot.IsSnapshot); |
||||
|
|
||||
|
var mapped = snapshot.ToState(); |
||||
|
|
||||
|
mapped.Should().BeEquivalentTo(source); |
||||
|
} |
||||
|
|
||||
|
[Fact] |
||||
|
public async Task Should_map_content_without_new_version_to_published() |
||||
|
{ |
||||
|
var source = CreateContentWithoutNewVersion(); |
||||
|
|
||||
|
var snapshotJob = new SnapshotWriteJob<ContentDomainObject.State>(source.UniqueId, source, source.Version); |
||||
|
var snapshot = await MongoContentEntity.CreatePublishedAsync(snapshotJob, appProvider); |
||||
|
|
||||
|
Assert.Equal(source.CurrentVersion.Data, snapshot.Data); |
||||
|
Assert.Null(snapshot.DraftData); |
||||
|
Assert.Null(snapshot.NewStatus); |
||||
|
Assert.Null(snapshot.ScheduleJob); |
||||
|
Assert.False(snapshot.IsSnapshot); |
||||
|
} |
||||
|
|
||||
|
[Fact] |
||||
|
public async Task Should_map_content_with_new_version_to_draft() |
||||
|
{ |
||||
|
var source = CreateContentWithNewVersion(); |
||||
|
|
||||
|
var snapshotJob = new SnapshotWriteJob<ContentDomainObject.State>(source.UniqueId, source, source.Version); |
||||
|
var snapshot = await MongoContentEntity.CreateDraftAsync(snapshotJob, appProvider); |
||||
|
|
||||
|
Assert.Equal(source.NewVersion?.Data, snapshot.Data); |
||||
|
Assert.Equal(source.CurrentVersion.Data, snapshot.DraftData); |
||||
|
Assert.NotNull(snapshot.NewStatus); |
||||
|
Assert.NotNull(snapshot.ScheduleJob); |
||||
|
Assert.True(snapshot.IsSnapshot); |
||||
|
|
||||
|
var mapped = snapshot.ToState(); |
||||
|
|
||||
|
mapped.Should().BeEquivalentTo(source); |
||||
|
} |
||||
|
|
||||
|
[Fact] |
||||
|
public async Task Should_map_content_with_new_version_to_published() |
||||
|
{ |
||||
|
var source = CreateContentWithNewVersion(); |
||||
|
|
||||
|
var snapshotJob = new SnapshotWriteJob<ContentDomainObject.State>(source.UniqueId, source, source.Version); |
||||
|
var snapshot = await MongoContentEntity.CreatePublishedAsync(snapshotJob, appProvider); |
||||
|
|
||||
|
Assert.Equal(source.CurrentVersion?.Data, snapshot.Data); |
||||
|
Assert.Null(snapshot.DraftData); |
||||
|
Assert.Null(snapshot.NewStatus); |
||||
|
Assert.Null(snapshot.ScheduleJob); |
||||
|
Assert.False(snapshot.IsSnapshot); |
||||
|
} |
||||
|
|
||||
|
private static ContentDomainObject.State CreateContentWithoutNewVersion() |
||||
|
{ |
||||
|
var user = new RefToken(RefTokenType.Subject, "1"); |
||||
|
|
||||
|
var data = |
||||
|
new ContentData() |
||||
|
.AddField("my-field", |
||||
|
new ContentFieldData() |
||||
|
.AddInvariant(42)); |
||||
|
|
||||
|
var time = SystemClock.Instance.GetCurrentInstant(); |
||||
|
|
||||
|
var state = new ContentDomainObject.State |
||||
|
{ |
||||
|
Id = DomainId.NewGuid(), |
||||
|
AppId = NamedId.Of(DomainId.NewGuid(), "my-app"), |
||||
|
Created = time, |
||||
|
CreatedBy = user, |
||||
|
CurrentVersion = new ContentVersion(Status.Archived, data), |
||||
|
IsDeleted = true, |
||||
|
LastModified = time, |
||||
|
LastModifiedBy = user, |
||||
|
ScheduleJob = new ScheduleJob(DomainId.NewGuid(), Status.Published, user, time), |
||||
|
SchemaId = NamedId.Of(DomainId.NewGuid(), "my-schema"), |
||||
|
Version = 42, |
||||
|
}; |
||||
|
|
||||
|
return state; |
||||
|
} |
||||
|
|
||||
|
private static ContentDomainObject.State CreateContentWithNewVersion() |
||||
|
{ |
||||
|
var user = new RefToken(RefTokenType.Subject, "1"); |
||||
|
|
||||
|
var data = |
||||
|
new ContentData() |
||||
|
.AddField("my-field", |
||||
|
new ContentFieldData() |
||||
|
.AddInvariant(42)); |
||||
|
|
||||
|
var newData = |
||||
|
new ContentData() |
||||
|
.AddField("my-field", |
||||
|
new ContentFieldData() |
||||
|
.AddInvariant(13)); |
||||
|
|
||||
|
var time = SystemClock.Instance.GetCurrentInstant(); |
||||
|
|
||||
|
var state = new ContentDomainObject.State |
||||
|
{ |
||||
|
Id = DomainId.NewGuid(), |
||||
|
AppId = NamedId.Of(DomainId.NewGuid(), "my-app"), |
||||
|
Created = time, |
||||
|
CreatedBy = user, |
||||
|
CurrentVersion = new ContentVersion(Status.Archived, data), |
||||
|
IsDeleted = true, |
||||
|
LastModified = time, |
||||
|
LastModifiedBy = user, |
||||
|
NewVersion = new ContentVersion(Status.Published, newData), |
||||
|
ScheduleJob = new ScheduleJob(DomainId.NewGuid(), Status.Published, user, time), |
||||
|
SchemaId = NamedId.Of(DomainId.NewGuid(), "my-schema"), |
||||
|
Version = 42, |
||||
|
}; |
||||
|
|
||||
|
return state; |
||||
|
} |
||||
|
} |
||||
|
} |
||||
@ -0,0 +1,50 @@ |
|||||
|
// ==========================================================================
|
||||
|
// Squidex Headless CMS
|
||||
|
// ==========================================================================
|
||||
|
// Copyright (c) Squidex UG (haftungsbeschraenkt)
|
||||
|
// All rights reserved. Licensed under the MIT license.
|
||||
|
// ==========================================================================
|
||||
|
|
||||
|
using MongoDB.Bson; |
||||
|
using MongoDB.Bson.Serialization.Attributes; |
||||
|
|
||||
|
namespace Squidex.Infrastructure.MongoDb |
||||
|
{ |
||||
|
public static class Entities |
||||
|
{ |
||||
|
public sealed class DateTimeEntity<T> |
||||
|
{ |
||||
|
[BsonRepresentation(BsonType.DateTime)] |
||||
|
public T Value { get; set; } |
||||
|
} |
||||
|
|
||||
|
public sealed class Int64Entity<T> |
||||
|
{ |
||||
|
[BsonRepresentation(BsonType.Int64)] |
||||
|
public T Value { get; set; } |
||||
|
} |
||||
|
|
||||
|
public sealed class Int32Entity<T> |
||||
|
{ |
||||
|
[BsonRepresentation(BsonType.Int32)] |
||||
|
public T Value { get; set; } |
||||
|
} |
||||
|
|
||||
|
public sealed class StringEntity<T> |
||||
|
{ |
||||
|
[BsonRepresentation(BsonType.String)] |
||||
|
public T Value { get; set; } |
||||
|
} |
||||
|
|
||||
|
public sealed class BinaryEntity<T> |
||||
|
{ |
||||
|
[BsonRepresentation(BsonType.Binary)] |
||||
|
public T Value { get; set; } |
||||
|
} |
||||
|
|
||||
|
public sealed class DefaultEntity<T> |
||||
|
{ |
||||
|
public T Value { get; set; } |
||||
|
} |
||||
|
} |
||||
|
} |
||||
@ -0,0 +1,93 @@ |
|||||
|
// ==========================================================================
|
||||
|
// Squidex Headless CMS
|
||||
|
// ==========================================================================
|
||||
|
// Copyright (c) Squidex UG (haftungsbeschraenkt)
|
||||
|
// All rights reserved. Licensed under the MIT license.
|
||||
|
// ==========================================================================
|
||||
|
|
||||
|
using MongoDB.Bson.IO; |
||||
|
using MongoDB.Bson.Serialization; |
||||
|
using NodaTime; |
||||
|
using Xunit; |
||||
|
|
||||
|
namespace Squidex.Infrastructure.MongoDb |
||||
|
{ |
||||
|
public class InstantSerializerTests |
||||
|
{ |
||||
|
public InstantSerializerTests() |
||||
|
{ |
||||
|
InstantSerializer.Register(); |
||||
|
} |
||||
|
|
||||
|
[Fact] |
||||
|
public void Should_serialize_as_default() |
||||
|
{ |
||||
|
var source = new Entities.DefaultEntity<Instant> { Value = GetTime() }; |
||||
|
|
||||
|
var result1 = SerializeAndDeserializeBson(source); |
||||
|
|
||||
|
Assert.Equal(source.Value, result1.Value); |
||||
|
} |
||||
|
|
||||
|
[Fact] |
||||
|
public void Should_serialize_as_string() |
||||
|
{ |
||||
|
var source = new Entities.StringEntity<Instant> { Value = GetTime() }; |
||||
|
|
||||
|
var result1 = SerializeAndDeserializeBson(source); |
||||
|
|
||||
|
Assert.Equal(source.Value, result1.Value); |
||||
|
} |
||||
|
|
||||
|
[Fact] |
||||
|
public void Should_serialize_as_int64() |
||||
|
{ |
||||
|
var source = new Entities.Int64Entity<Instant> { Value = GetTime() }; |
||||
|
|
||||
|
var result1 = SerializeAndDeserializeBson(source); |
||||
|
|
||||
|
Assert.Equal(source.Value, result1.Value); |
||||
|
} |
||||
|
|
||||
|
[Fact] |
||||
|
public void Should_serialize_as_datetime() |
||||
|
{ |
||||
|
var source = new Entities.DateTimeEntity<Instant> { Value = GetTime() }; |
||||
|
|
||||
|
var result1 = SerializeAndDeserializeBson(source); |
||||
|
|
||||
|
Assert.Equal(source.Value, result1.Value); |
||||
|
} |
||||
|
|
||||
|
private static Instant GetTime() |
||||
|
{ |
||||
|
return SystemClock.Instance.GetCurrentInstant().WithoutNs(); |
||||
|
} |
||||
|
|
||||
|
private static T SerializeAndDeserializeBson<T>(T source) |
||||
|
{ |
||||
|
return SerializeAndDeserializeBson<T, T>(source); |
||||
|
} |
||||
|
|
||||
|
private static TOut SerializeAndDeserializeBson<TIn, TOut>(TIn source) |
||||
|
{ |
||||
|
var stream = new MemoryStream(); |
||||
|
|
||||
|
using (var writer = new BsonBinaryWriter(stream)) |
||||
|
{ |
||||
|
BsonSerializer.Serialize(writer, source); |
||||
|
|
||||
|
writer.Flush(); |
||||
|
} |
||||
|
|
||||
|
stream.Position = 0; |
||||
|
|
||||
|
using (var reader = new BsonBinaryReader(stream)) |
||||
|
{ |
||||
|
var target = BsonSerializer.Deserialize<TOut>(reader); |
||||
|
|
||||
|
return target; |
||||
|
} |
||||
|
} |
||||
|
} |
||||
|
} |
||||
@ -0,0 +1,28 @@ |
|||||
|
<div class="clearfix" *ngIf="!autoHide || canGoPrev || canGoNext"> |
||||
|
<div class="float-right pagination" *ngIf="paging"> |
||||
|
<select class="custom-select custom-select-sm" [ngModel]="paging.pageSize" (ngModelChange)="setPageSize($event)"> |
||||
|
<option *ngFor="let pageSize of pageSizes" [ngValue]="pageSize">{{pageSize}}</option> |
||||
|
</select> |
||||
|
|
||||
|
<span class="page-info"> |
||||
|
<ng-container *ngIf="paging.count > 0 && paging.total > 0"> |
||||
|
<button class="btn deactivated"> |
||||
|
{{ 'common.pagerInfo' | sqxTranslate: translationInfo }} |
||||
|
</button> |
||||
|
</ng-container> |
||||
|
|
||||
|
<ng-container *ngIf="paging.count > 0 && paging.total <= 0"> |
||||
|
<button class="btn" title="{{ 'common.pagerReload' | sqxTranslate }}" (click)="loadTotal.emit()"> |
||||
|
{{ 'common.pagerInfoNoTotal' | sqxTranslate: translationInfo }} |
||||
|
</button> |
||||
|
</ng-container> |
||||
|
|
||||
|
<button type="button" class="btn btn-text-secondary pagination-button" [disabled]="!canGoPrev" (click)="goPrev()"> |
||||
|
<i class="icon-angle-left"></i> |
||||
|
</button> |
||||
|
<button type="button" class="btn btn-text-secondary pagination-button" [disabled]="!canGoNext" (click)="goNext()"> |
||||
|
<i class="icon-angle-right"></i> |
||||
|
</button> |
||||
|
</span> |
||||
|
</div> |
||||
|
</div> |
||||
Loading…
Reference in new issue