mirror of https://github.com/Squidex/squidex.git
Browse Source
* Improve rule performance. * Back to async enumerable. * Migration to usage tracker. * Use proper data type. * Fine tuning. * Fix tests * Fix backend tests. * More tests fixed. * Some unrelated testspull/978/head
Sebastian Stehle
3 years ago
committed by
GitHub
GitHub
159 changed files with 2698 additions and 1643 deletions
@ -1,89 +0,0 @@ |
|||
// ==========================================================================
|
|||
// Squidex Headless CMS
|
|||
// ==========================================================================
|
|||
// Copyright (c) Squidex UG (haftungsbeschraenkt)
|
|||
// All rights reserved. Licensed under the MIT license.
|
|||
// ==========================================================================
|
|||
|
|||
using MongoDB.Bson.Serialization; |
|||
using MongoDB.Driver; |
|||
using NodaTime; |
|||
using Squidex.Domain.Apps.Entities.Rules.Repositories; |
|||
using Squidex.Infrastructure; |
|||
using Squidex.Infrastructure.MongoDb; |
|||
|
|||
namespace Squidex.Domain.Apps.Entities.MongoDb.Rules; |
|||
|
|||
public sealed class MongoRuleStatisticsCollection : MongoRepositoryBase<RuleStatistics> |
|||
{ |
|||
static MongoRuleStatisticsCollection() |
|||
{ |
|||
BsonClassMap.RegisterClassMap<RuleStatistics>(cm => |
|||
{ |
|||
cm.AutoMap(); |
|||
|
|||
cm.SetIgnoreExtraElements(true); |
|||
}); |
|||
} |
|||
|
|||
public MongoRuleStatisticsCollection(IMongoDatabase database) |
|||
: base(database) |
|||
{ |
|||
} |
|||
|
|||
protected override string CollectionName() |
|||
{ |
|||
return "RuleStatistics"; |
|||
} |
|||
|
|||
protected override Task SetupCollectionAsync(IMongoCollection<RuleStatistics> collection, |
|||
CancellationToken ct) |
|||
{ |
|||
return collection.Indexes.CreateOneAsync( |
|||
new CreateIndexModel<RuleStatistics>( |
|||
Index |
|||
.Ascending(x => x.AppId) |
|||
.Ascending(x => x.RuleId)), |
|||
cancellationToken: ct); |
|||
} |
|||
|
|||
public async Task DeleteAppAsync(DomainId appId, |
|||
CancellationToken ct) |
|||
{ |
|||
await Collection.DeleteManyAsync(Filter.Eq(x => x.AppId, appId), ct); |
|||
} |
|||
|
|||
public async Task<IReadOnlyList<RuleStatistics>> QueryByAppAsync(DomainId appId, |
|||
CancellationToken ct) |
|||
{ |
|||
var statistics = await Collection.Find(x => x.AppId == appId).ToListAsync(ct); |
|||
|
|||
return statistics; |
|||
} |
|||
|
|||
public Task IncrementSuccessAsync(DomainId appId, DomainId ruleId, Instant now, |
|||
CancellationToken ct) |
|||
{ |
|||
return Collection.UpdateOneAsync( |
|||
x => x.AppId == appId && x.RuleId == ruleId, |
|||
Update |
|||
.Inc(x => x.NumSucceeded, 1) |
|||
.Set(x => x.LastExecuted, now) |
|||
.SetOnInsert(x => x.AppId, appId) |
|||
.SetOnInsert(x => x.RuleId, ruleId), |
|||
Upsert, ct); |
|||
} |
|||
|
|||
public Task IncrementFailedAsync(DomainId appId, DomainId ruleId, Instant now, |
|||
CancellationToken ct) |
|||
{ |
|||
return Collection.UpdateOneAsync( |
|||
x => x.AppId == appId && x.RuleId == ruleId, |
|||
Update |
|||
.Inc(x => x.NumFailed, 1) |
|||
.Set(x => x.LastExecuted, now) |
|||
.SetOnInsert(x => x.AppId, appId) |
|||
.SetOnInsert(x => x.RuleId, ruleId), |
|||
Upsert, ct); |
|||
} |
|||
} |
|||
@ -1,12 +0,0 @@ |
|||
// ==========================================================================
|
|||
// Squidex Headless CMS
|
|||
// ==========================================================================
|
|||
// Copyright (c) Squidex UG (haftungsbeschraenkt)
|
|||
// All rights reserved. Licensed under the MIT license.
|
|||
// ==========================================================================
|
|||
|
|||
#pragma warning disable SA1313 // Parameter names should begin with lower-case letter
|
|||
|
|||
namespace Squidex.Domain.Apps.Entities.Assets; |
|||
|
|||
public sealed record AssetStats(DateTime Date, long TotalCount, long TotalSize); |
|||
@ -0,0 +1,145 @@ |
|||
// ==========================================================================
|
|||
// Squidex Headless CMS
|
|||
// ==========================================================================
|
|||
// Copyright (c) Squidex UG (haftungsbeschraenkt)
|
|||
// All rights reserved. Licensed under the MIT license.
|
|||
// ==========================================================================
|
|||
|
|||
using Squidex.Domain.Apps.Entities.Assets; |
|||
using Squidex.Infrastructure; |
|||
using Squidex.Infrastructure.UsageTracking; |
|||
|
|||
namespace Squidex.Domain.Apps.Entities.Billing; |
|||
|
|||
public sealed partial class UsageGate : IAssetUsageTracker |
|||
{ |
|||
public static class AssetsKeys |
|||
{ |
|||
public const string TotalAssets = nameof(AssetCounters.TotalAssets); |
|||
public const string TotalSize = nameof(AssetCounters.TotalSize); |
|||
} |
|||
|
|||
Task IAssetUsageTracker.DeleteUsageAsync(DomainId appId, |
|||
CancellationToken ct) |
|||
{ |
|||
// Do not delete the team, as this is only called when an app is deleted.
|
|||
return usageTracker.DeleteAsync(AppAssetsKey(appId), ct); |
|||
} |
|||
|
|||
Task IAssetUsageTracker.DeleteUsageAsync( |
|||
CancellationToken ct) |
|||
{ |
|||
// Use a well defined prefix query for the deletion to improve performance.
|
|||
return usageTracker.DeleteByKeyPatternAsync("^([a-zA-Z0-9]+)_[A-Za-z]+Assets", ct); |
|||
} |
|||
|
|||
Task<AssetCounters> IAssetUsageTracker.GetTotalByAppAsync(DomainId appId, |
|||
CancellationToken ct) |
|||
{ |
|||
return GetTotalForAssetsAsync(AppAssetsKey(appId), ct); |
|||
} |
|||
|
|||
Task<AssetCounters> IAssetUsageTracker.GetTotalByTeamAsync(DomainId teamId, |
|||
CancellationToken ct) |
|||
{ |
|||
return GetTotalForAssetsAsync(TeamAssetsKey(teamId), ct); |
|||
} |
|||
|
|||
Task<IReadOnlyList<AssetStats>> IAssetUsageTracker.QueryByAppAsync(DomainId appId, DateOnly fromDate, DateOnly toDate, |
|||
CancellationToken ct) |
|||
{ |
|||
return QueryForAssetsAsync(AppAssetsKey(appId), fromDate, toDate, ct); |
|||
} |
|||
|
|||
Task<IReadOnlyList<AssetStats>> IAssetUsageTracker.QueryByTeamAsync(DomainId teamId, DateOnly fromDate, DateOnly toDate, |
|||
CancellationToken ct) |
|||
{ |
|||
return QueryForAssetsAsync(TeamAssetsKey(teamId), fromDate, toDate, ct); |
|||
} |
|||
|
|||
private async Task<AssetCounters> GetTotalForAssetsAsync(string key, |
|||
CancellationToken ct) |
|||
{ |
|||
var counters = await usageTracker.GetAsync(key, SummaryDate, SummaryDate, null, ct); |
|||
|
|||
return GetAssetCounters(counters); |
|||
} |
|||
|
|||
private async Task<IReadOnlyList<AssetStats>> QueryForAssetsAsync(string key, DateOnly fromDate, DateOnly toDate, |
|||
CancellationToken ct) |
|||
{ |
|||
var result = new List<AssetStats>(); |
|||
|
|||
var usages = await usageTracker.QueryAsync(key, fromDate, toDate, ct); |
|||
|
|||
for (var date = fromDate; date <= toDate; date = date.AddDays(1)) |
|||
{ |
|||
var aggregated = default(AssetCounters); |
|||
|
|||
foreach (var (_, byCategory) in usages) |
|||
{ |
|||
foreach (var (counterDate, counters) in byCategory) |
|||
{ |
|||
if (counterDate == date) |
|||
{ |
|||
var currentCounters = GetAssetCounters(counters); |
|||
|
|||
aggregated.TotalSize += currentCounters.TotalSize; |
|||
aggregated.TotalAssets += currentCounters.TotalAssets; |
|||
} |
|||
} |
|||
} |
|||
|
|||
result.Add(new AssetStats(date, aggregated)); |
|||
} |
|||
|
|||
return result; |
|||
} |
|||
|
|||
async Task IAssetUsageTracker.TrackAsync(DomainId appId, DateOnly date, long fileSize, long count, |
|||
CancellationToken ct) |
|||
{ |
|||
var counters = new Counters |
|||
{ |
|||
[AssetsKeys.TotalSize] = fileSize, |
|||
[AssetsKeys.TotalAssets] = count |
|||
}; |
|||
|
|||
var appKey = AppAssetsKey(appId); |
|||
|
|||
var tasks = new List<Task> |
|||
{ |
|||
usageTracker.TrackAsync(date, appKey, null, counters, ct), |
|||
usageTracker.TrackAsync(SummaryDate, appKey, null, counters, ct) |
|||
}; |
|||
|
|||
var (_, _, teamId) = await GetPlanForAppAsync(appId, true, ct); |
|||
|
|||
if (teamId != null) |
|||
{ |
|||
var teamKey = TeamAssetsKey(teamId.Value); |
|||
|
|||
tasks.Add(usageTracker.TrackAsync(date, teamKey, appId.ToString(), counters, ct)); |
|||
tasks.Add(usageTracker.TrackAsync(SummaryDate, teamKey, appId.ToString(), counters, ct)); |
|||
} |
|||
|
|||
await Task.WhenAll(tasks); |
|||
} |
|||
|
|||
private static AssetCounters GetAssetCounters(Counters counters) |
|||
{ |
|||
return new AssetCounters( |
|||
counters.GetInt64(AssetsKeys.TotalSize), |
|||
counters.GetInt64(AssetsKeys.TotalAssets)); |
|||
} |
|||
|
|||
private static string AppAssetsKey(DomainId appId) |
|||
{ |
|||
return $"{appId}_Assets"; |
|||
} |
|||
|
|||
private static string TeamAssetsKey(DomainId teamId) |
|||
{ |
|||
return $"{teamId}_TeamAssets"; |
|||
} |
|||
} |
|||
@ -0,0 +1,140 @@ |
|||
// ==========================================================================
|
|||
// Squidex Headless CMS
|
|||
// ==========================================================================
|
|||
// Copyright (c) Squidex UG (haftungsbeschraenkt)
|
|||
// All rights reserved. Licensed under the MIT license.
|
|||
// ==========================================================================
|
|||
|
|||
using Squidex.Domain.Apps.Entities.Rules; |
|||
using Squidex.Infrastructure; |
|||
using Squidex.Infrastructure.UsageTracking; |
|||
|
|||
namespace Squidex.Domain.Apps.Entities.Billing; |
|||
|
|||
public sealed partial class UsageGate : IRuleUsageTracker |
|||
{ |
|||
public static class RulesKeys |
|||
{ |
|||
public const string TotalCreated = nameof(RuleCounters.TotalCreated); |
|||
public const string TotalSucceeded = nameof(RuleCounters.TotalSucceeded); |
|||
public const string TotalFailed = nameof(RuleCounters.TotalFailed); |
|||
} |
|||
|
|||
Task IRuleUsageTracker.DeleteUsageAsync(DomainId appId, |
|||
CancellationToken ct) |
|||
{ |
|||
// Use a well defined prefix query for the deletion to improve performance.
|
|||
return usageTracker.DeleteAsync(AppRulesKey(appId), ct); |
|||
} |
|||
|
|||
Task<IReadOnlyList<RuleStats>> IRuleUsageTracker.QueryByAppAsync(DomainId appId, DateOnly fromDate, DateOnly toDate, |
|||
CancellationToken ct) |
|||
{ |
|||
return QueryForRulesAsync(AppRulesKey(appId), fromDate, toDate, ct); |
|||
} |
|||
|
|||
Task<IReadOnlyList<RuleStats>> IRuleUsageTracker.QueryByTeamAsync(DomainId appId, DateOnly fromDate, DateOnly toDate, |
|||
CancellationToken ct) |
|||
{ |
|||
return QueryForRulesAsync(TeamRulesKey(appId), fromDate, toDate, ct); |
|||
} |
|||
|
|||
async Task<IReadOnlyDictionary<DomainId, RuleCounters>> IRuleUsageTracker.GetTotalByAppAsync(DomainId appId, |
|||
CancellationToken ct) |
|||
{ |
|||
var result = new Dictionary<DomainId, RuleCounters>(); |
|||
|
|||
var counters = await usageTracker.QueryAsync(AppRulesKey(appId), SummaryDate, SummaryDate, ct); |
|||
|
|||
foreach (var (category, byCategory) in counters) |
|||
{ |
|||
if (byCategory.Count > 0) |
|||
{ |
|||
result[DomainId.Create(category)] = GetRuleCounters(byCategory[0].Item2); |
|||
} |
|||
} |
|||
|
|||
return result; |
|||
} |
|||
|
|||
private async Task<IReadOnlyList<RuleStats>> QueryForRulesAsync(string key, DateOnly fromDate, DateOnly toDate, |
|||
CancellationToken ct) |
|||
{ |
|||
var result = new List<RuleStats>(); |
|||
|
|||
var usages = await usageTracker.QueryAsync(key, fromDate, toDate, ct); |
|||
|
|||
for (var date = fromDate; date <= toDate; date = date.AddDays(1)) |
|||
{ |
|||
var aggregated = default(RuleCounters); |
|||
|
|||
foreach (var (_, byCategory) in usages) |
|||
{ |
|||
foreach (var (counterDate, counters) in byCategory) |
|||
{ |
|||
if (counterDate == date) |
|||
{ |
|||
var currentCounters = GetRuleCounters(counters); |
|||
|
|||
aggregated.TotalCreated += currentCounters.TotalCreated; |
|||
aggregated.TotalSucceeded += currentCounters.TotalSucceeded; |
|||
aggregated.TotalFailed += currentCounters.TotalFailed; |
|||
} |
|||
} |
|||
} |
|||
|
|||
result.Add(new RuleStats(date, aggregated)); |
|||
} |
|||
|
|||
return result; |
|||
} |
|||
|
|||
async Task IRuleUsageTracker.TrackAsync(DomainId appId, DomainId ruleId, DateOnly date, int created, int succeeded, int failed, |
|||
CancellationToken ct) |
|||
{ |
|||
var counters = new Counters |
|||
{ |
|||
[RulesKeys.TotalCreated] = created, |
|||
[RulesKeys.TotalSucceeded] = succeeded, |
|||
[RulesKeys.TotalFailed] = failed |
|||
}; |
|||
|
|||
var appKey = AppRulesKey(appId); |
|||
|
|||
var tasks = new List<Task> |
|||
{ |
|||
usageTracker.TrackAsync(date, appKey, ruleId.ToString(), counters, ct), |
|||
usageTracker.TrackAsync(SummaryDate, appKey, ruleId.ToString(), counters, ct) |
|||
}; |
|||
|
|||
var (_, _, teamId) = await GetPlanForAppAsync(appId, true, ct); |
|||
|
|||
if (teamId != null) |
|||
{ |
|||
var teamKey = TeamRulesKey(teamId.Value); |
|||
|
|||
tasks.Add(usageTracker.TrackAsync(date, teamKey, appId.ToString(), counters, ct)); |
|||
tasks.Add(usageTracker.TrackAsync(SummaryDate, teamKey, appId.ToString(), counters, ct)); |
|||
} |
|||
|
|||
await Task.WhenAll(tasks); |
|||
} |
|||
|
|||
private static RuleCounters GetRuleCounters(Counters counters) |
|||
{ |
|||
return new RuleCounters( |
|||
counters.GetInt64(RulesKeys.TotalCreated), |
|||
counters.GetInt64(RulesKeys.TotalSucceeded), |
|||
counters.GetInt64(RulesKeys.TotalFailed)); |
|||
} |
|||
|
|||
private static string AppRulesKey(DomainId appId) |
|||
{ |
|||
return $"{appId}_Rules"; |
|||
} |
|||
|
|||
private static string TeamRulesKey(DomainId teamId) |
|||
{ |
|||
return $"{teamId}_TeamRules"; |
|||
} |
|||
} |
|||
@ -0,0 +1,35 @@ |
|||
// ==========================================================================
|
|||
// Squidex Headless CMS
|
|||
// ==========================================================================
|
|||
// Copyright (c) Squidex UG (haftungsbeschraenkt)
|
|||
// All rights reserved. Licensed under the MIT license.
|
|||
// ==========================================================================
|
|||
|
|||
using Squidex.Infrastructure; |
|||
|
|||
#pragma warning disable SA1313 // Parameter names should begin with lower-case letter
|
|||
#pragma warning disable MA0048 // File name must match type name
|
|||
|
|||
namespace Squidex.Domain.Apps.Entities.Rules; |
|||
|
|||
public interface IRuleUsageTracker |
|||
{ |
|||
Task<IReadOnlyList<RuleStats>> QueryByAppAsync(DomainId appId, DateOnly fromDate, DateOnly toDate, |
|||
CancellationToken ct = default); |
|||
|
|||
Task<IReadOnlyList<RuleStats>> QueryByTeamAsync(DomainId teamId, DateOnly fromDate, DateOnly toDate, |
|||
CancellationToken ct = default); |
|||
|
|||
Task<IReadOnlyDictionary<DomainId, RuleCounters>> GetTotalByAppAsync(DomainId appId, |
|||
CancellationToken ct = default); |
|||
|
|||
Task TrackAsync(DomainId appId, DomainId ruleId, DateOnly date, int created, int succeeded, int failed, |
|||
CancellationToken ct = default); |
|||
|
|||
Task DeleteUsageAsync(DomainId appId, |
|||
CancellationToken ct = default); |
|||
} |
|||
|
|||
public record struct RuleStats(DateOnly Date, RuleCounters Counters); |
|||
|
|||
public record struct RuleCounters(long TotalCreated, long TotalSucceeded, long TotalFailed); |
|||
@ -1,24 +0,0 @@ |
|||
// ==========================================================================
|
|||
// Squidex Headless CMS
|
|||
// ==========================================================================
|
|||
// Copyright (c) Squidex UG (haftungsbeschraenkt)
|
|||
// All rights reserved. Licensed under the MIT license.
|
|||
// ==========================================================================
|
|||
|
|||
using NodaTime; |
|||
using Squidex.Infrastructure; |
|||
|
|||
namespace Squidex.Domain.Apps.Entities.Rules.Repositories; |
|||
|
|||
public sealed class RuleStatistics |
|||
{ |
|||
public DomainId AppId { get; set; } |
|||
|
|||
public DomainId RuleId { get; set; } |
|||
|
|||
public int NumSucceeded { get; set; } |
|||
|
|||
public int NumFailed { get; set; } |
|||
|
|||
public Instant? LastExecuted { get; set; } |
|||
} |
|||
@ -0,0 +1,90 @@ |
|||
// ==========================================================================
|
|||
// Squidex Headless CMS
|
|||
// ==========================================================================
|
|||
// Copyright (c) Squidex UG (haftungsbeschraenkt)
|
|||
// All rights reserved. Licensed under the MIT license.
|
|||
// ==========================================================================
|
|||
|
|||
using Microsoft.Extensions.Logging; |
|||
using Squidex.Domain.Apps.Core.HandleRules; |
|||
using Squidex.Domain.Apps.Entities.Rules.Repositories; |
|||
using Squidex.Infrastructure; |
|||
|
|||
namespace Squidex.Domain.Apps.Entities.Rules; |
|||
|
|||
internal sealed class RuleQueueWriter : IAsyncDisposable |
|||
{ |
|||
private readonly List<RuleEventWrite> writes = new List<RuleEventWrite>(); |
|||
private readonly IRuleEventRepository ruleEventRepository; |
|||
private readonly IRuleUsageTracker ruleUsageTracker; |
|||
private readonly ILogger? log; |
|||
|
|||
public RuleQueueWriter(IRuleEventRepository ruleEventRepository, IRuleUsageTracker ruleUsageTracker, ILogger? log) |
|||
{ |
|||
this.ruleEventRepository = ruleEventRepository; |
|||
this.ruleUsageTracker = ruleUsageTracker; |
|||
this.log = log; |
|||
} |
|||
|
|||
public async Task<bool> WriteAsync(JobResult result) |
|||
{ |
|||
// We do not want to handle events without a job in the normal flow.
|
|||
if (result.Job == null) |
|||
{ |
|||
return false; |
|||
} |
|||
|
|||
if (result.EnrichmentError != null || result.SkipReason is SkipReason.Failed) |
|||
{ |
|||
writes.Add(new RuleEventWrite(result.Job, Error: result.EnrichmentError)); |
|||
} |
|||
else |
|||
{ |
|||
writes.Add(new RuleEventWrite(result.Job, result.Job.Created)); |
|||
} |
|||
|
|||
log?.LogInformation("Adding rule job {jobId} for Rule(action={ruleAction}, trigger={ruleTrigger})", |
|||
result.Job.Id, |
|||
result.Rule.Action.GetType().Name, |
|||
result.Rule.Trigger.GetType().Name); |
|||
|
|||
var totalFailure = result.SkipReason == SkipReason.Failed ? 1 : 0; |
|||
var totalCreated = 1; |
|||
|
|||
// Unfortunately we cannot write in batches here, because the result could be from multiple rules.
|
|||
await ruleUsageTracker.TrackAsync(result.Job.AppId, result.RuleId, result.Job.Created.ToDateOnly(), totalCreated, 0, totalFailure); |
|||
|
|||
if (writes.Count >= 100) |
|||
{ |
|||
await FlushCoreAsync(); |
|||
return true; |
|||
} |
|||
|
|||
return false; |
|||
} |
|||
|
|||
public async Task<bool> FlushAsync() |
|||
{ |
|||
if (writes.Count > 0) |
|||
{ |
|||
await FlushCoreAsync(); |
|||
return true; |
|||
} |
|||
|
|||
return false; |
|||
} |
|||
|
|||
public async ValueTask DisposeAsync() |
|||
{ |
|||
if (writes.Count > 0) |
|||
{ |
|||
await FlushCoreAsync(); |
|||
} |
|||
} |
|||
|
|||
private async Task FlushCoreAsync() |
|||
{ |
|||
await ruleEventRepository.EnqueueAsync(writes, default); |
|||
writes.Clear(); |
|||
} |
|||
} |
|||
@ -1,37 +0,0 @@ |
|||
// ==========================================================================
|
|||
// Squidex Headless CMS
|
|||
// ==========================================================================
|
|||
// Copyright (c) Squidex UG (haftungsbeschraenkt)
|
|||
// All rights reserved. Licensed under the MIT license.
|
|||
// ==========================================================================
|
|||
|
|||
using Squidex.Domain.Apps.Core.Rules.Triggers; |
|||
using Squidex.Infrastructure; |
|||
using Squidex.Infrastructure.Reflection; |
|||
|
|||
namespace Squidex.Areas.Api.Controllers.Rules.Models.Triggers; |
|||
|
|||
public sealed class ContentChangedRuleTriggerSchemaDto |
|||
{ |
|||
/// <summary>
|
|||
/// The ID of the schema.
|
|||
/// </summary>
|
|||
public DomainId SchemaId { get; set; } |
|||
|
|||
/// <summary>
|
|||
/// Javascript condition when to trigger.
|
|||
/// </summary>
|
|||
public string? Condition { get; set; } |
|||
|
|||
public ContentChangedTriggerSchemaV2 ToTrigger() |
|||
{ |
|||
return SimpleMapper.Map(this, new ContentChangedTriggerSchemaV2()); |
|||
} |
|||
|
|||
public static ContentChangedRuleTriggerSchemaDto FromDomain(ContentChangedTriggerSchemaV2 trigger) |
|||
{ |
|||
var result = SimpleMapper.Map(trigger, new ContentChangedRuleTriggerSchemaDto()); |
|||
|
|||
return result; |
|||
} |
|||
} |
|||
Some files were not shown because too many files changed in this diff
Loading…
Reference in new issue