mirror of https://github.com/Squidex/squidex.git
Browse Source
* Started with a few repositories. * Get rid unnecessary allocations. * New repositories and cancellation token improvements. * First untested implementation. * Fix infrastructure tests. * Warnings fixed. * All tests green again. * New tests. * Better cancellation. * More tests. * Backup fix. * Markdown everything * A few more tests and fixes. * Fix app deletion flag. * Provide app to client creator. * Started with migration. * Temp. * Minor change to rename app deletion. * Toggle for deleter. * Fix build.pull/768/head
Sebastian Stehle
4 years ago
committed by
GitHub
GitHub
544 changed files with 6609 additions and 4485 deletions
@ -1,189 +0,0 @@ |
|||||
// ==========================================================================
|
|
||||
// Squidex Headless CMS
|
|
||||
// ==========================================================================
|
|
||||
// Copyright (c) Squidex UG (haftungsbeschraenkt)
|
|
||||
// All rights reserved. Licensed under the MIT license.
|
|
||||
// ==========================================================================
|
|
||||
|
|
||||
using System.Collections.Generic; |
|
||||
using System.Threading; |
|
||||
using System.Threading.Tasks; |
|
||||
using Squidex.Domain.Apps.Entities.Apps.Indexes; |
|
||||
using Squidex.Domain.Apps.Entities.Rules.Indexes; |
|
||||
using Squidex.Domain.Apps.Entities.Schemas.Indexes; |
|
||||
using Squidex.Domain.Apps.Events; |
|
||||
using Squidex.Domain.Apps.Events.Apps; |
|
||||
using Squidex.Domain.Apps.Events.Rules; |
|
||||
using Squidex.Domain.Apps.Events.Schemas; |
|
||||
using Squidex.Infrastructure; |
|
||||
using Squidex.Infrastructure.EventSourcing; |
|
||||
using Squidex.Infrastructure.Migrations; |
|
||||
|
|
||||
namespace Migrations.Migrations |
|
||||
{ |
|
||||
public class PopulateGrainIndexes : IMigration |
|
||||
{ |
|
||||
private readonly IAppsIndex indexApps; |
|
||||
private readonly IRulesIndex indexRules; |
|
||||
private readonly ISchemasIndex indexSchemas; |
|
||||
private readonly IEventDataFormatter eventDataFormatter; |
|
||||
private readonly IEventStore eventStore; |
|
||||
|
|
||||
public PopulateGrainIndexes(IAppsIndex indexApps, IRulesIndex indexRules, ISchemasIndex indexSchemas, |
|
||||
IEventDataFormatter eventDataFormatter, |
|
||||
IEventStore eventStore) |
|
||||
{ |
|
||||
this.indexApps = indexApps; |
|
||||
this.indexRules = indexRules; |
|
||||
this.indexSchemas = indexSchemas; |
|
||||
this.eventDataFormatter = eventDataFormatter; |
|
||||
this.eventStore = eventStore; |
|
||||
} |
|
||||
|
|
||||
public Task UpdateAsync(CancellationToken ct) |
|
||||
{ |
|
||||
return Task.WhenAll( |
|
||||
RebuildAppIndexes(ct), |
|
||||
RebuildRuleIndexes(ct), |
|
||||
RebuildSchemaIndexes(ct)); |
|
||||
} |
|
||||
|
|
||||
private async Task RebuildAppIndexes(CancellationToken ct) |
|
||||
{ |
|
||||
var appsByName = new Dictionary<string, DomainId>(); |
|
||||
var appsByUser = new Dictionary<string, HashSet<DomainId>>(); |
|
||||
|
|
||||
bool HasApp(NamedId<DomainId> appId, bool consistent, out DomainId id) |
|
||||
{ |
|
||||
return appsByName!.TryGetValue(appId.Name, out id) && (!consistent || id == appId.Id); |
|
||||
} |
|
||||
|
|
||||
HashSet<DomainId> Index(string contributorId) |
|
||||
{ |
|
||||
return appsByUser!.GetOrAddNew(contributorId); |
|
||||
} |
|
||||
|
|
||||
void RemoveApp(NamedId<DomainId> appId, bool consistent) |
|
||||
{ |
|
||||
if (HasApp(appId, consistent, out var id)) |
|
||||
{ |
|
||||
foreach (var apps in appsByUser!.Values) |
|
||||
{ |
|
||||
apps.Remove(id); |
|
||||
} |
|
||||
|
|
||||
appsByName!.Remove(appId.Name); |
|
||||
} |
|
||||
} |
|
||||
|
|
||||
await foreach (var storedEvent in eventStore.QueryAllAsync("^app\\-", ct: ct)) |
|
||||
{ |
|
||||
var @event = eventDataFormatter.ParseIfKnown(storedEvent); |
|
||||
|
|
||||
if (@event != null) |
|
||||
{ |
|
||||
switch (@event.Payload) |
|
||||
{ |
|
||||
case AppCreated created: |
|
||||
{ |
|
||||
RemoveApp(created.AppId, false); |
|
||||
|
|
||||
appsByName[created.Name] = created.AppId.Id; |
|
||||
break; |
|
||||
} |
|
||||
|
|
||||
case AppContributorAssigned contributorAssigned: |
|
||||
{ |
|
||||
if (HasApp(contributorAssigned.AppId, true, out _)) |
|
||||
{ |
|
||||
Index(contributorAssigned.ContributorId).Add(contributorAssigned.AppId.Id); |
|
||||
} |
|
||||
|
|
||||
break; |
|
||||
} |
|
||||
|
|
||||
case AppContributorRemoved contributorRemoved: |
|
||||
Index(contributorRemoved.ContributorId).Remove(contributorRemoved.AppId.Id); |
|
||||
break; |
|
||||
case AppArchived archived: |
|
||||
RemoveApp(archived.AppId, true); |
|
||||
break; |
|
||||
} |
|
||||
} |
|
||||
} |
|
||||
|
|
||||
await indexApps.RebuildAsync(appsByName); |
|
||||
|
|
||||
foreach (var (contributorId, apps) in appsByUser) |
|
||||
{ |
|
||||
await indexApps.RebuildByContributorsAsync(contributorId, apps); |
|
||||
} |
|
||||
} |
|
||||
|
|
||||
private async Task RebuildRuleIndexes(CancellationToken ct) |
|
||||
{ |
|
||||
var rulesByApp = new Dictionary<DomainId, HashSet<DomainId>>(); |
|
||||
|
|
||||
HashSet<DomainId> Index(RuleEvent @event) |
|
||||
{ |
|
||||
return rulesByApp!.GetOrAddNew(@event.AppId.Id); |
|
||||
} |
|
||||
|
|
||||
await foreach (var storedEvent in eventStore.QueryAllAsync("^rule\\-", ct: ct)) |
|
||||
{ |
|
||||
var @event = eventDataFormatter.ParseIfKnown(storedEvent); |
|
||||
|
|
||||
if (@event != null) |
|
||||
{ |
|
||||
switch (@event.Payload) |
|
||||
{ |
|
||||
case RuleCreated created: |
|
||||
Index(created).Add(created.RuleId); |
|
||||
break; |
|
||||
case RuleDeleted deleted: |
|
||||
Index(deleted).Remove(deleted.RuleId); |
|
||||
break; |
|
||||
} |
|
||||
} |
|
||||
} |
|
||||
|
|
||||
foreach (var (appId, rules) in rulesByApp) |
|
||||
{ |
|
||||
await indexRules.RebuildAsync(appId, rules); |
|
||||
} |
|
||||
} |
|
||||
|
|
||||
private async Task RebuildSchemaIndexes(CancellationToken ct) |
|
||||
{ |
|
||||
var schemasByApp = new Dictionary<DomainId, Dictionary<string, DomainId>>(); |
|
||||
|
|
||||
Dictionary<string, DomainId> Index(SchemaEvent @event) |
|
||||
{ |
|
||||
return schemasByApp!.GetOrAddNew(@event.AppId.Id); |
|
||||
} |
|
||||
|
|
||||
await foreach (var storedEvent in eventStore.QueryAllAsync("^schema\\-", ct: ct)) |
|
||||
{ |
|
||||
var @event = eventDataFormatter.ParseIfKnown(storedEvent); |
|
||||
|
|
||||
if (@event != null) |
|
||||
{ |
|
||||
switch (@event.Payload) |
|
||||
{ |
|
||||
case SchemaCreated created: |
|
||||
Index(created)[created.SchemaId.Name] = created.SchemaId.Id; |
|
||||
break; |
|
||||
case SchemaDeleted deleted: |
|
||||
Index(deleted).Remove(deleted.SchemaId.Name); |
|
||||
break; |
|
||||
} |
|
||||
} |
|
||||
} |
|
||||
|
|
||||
foreach (var (appId, schemas) in schemasByApp) |
|
||||
{ |
|
||||
await indexSchemas.RebuildAsync(appId, schemas); |
|
||||
} |
|
||||
} |
|
||||
} |
|
||||
} |
|
||||
@ -0,0 +1,34 @@ |
|||||
|
// ==========================================================================
|
||||
|
// Squidex Headless CMS
|
||||
|
// ==========================================================================
|
||||
|
// Copyright (c) Squidex UG (haftungsbeschraenkt)
|
||||
|
// All rights reserved. Licensed under the MIT license.
|
||||
|
// ==========================================================================
|
||||
|
|
||||
|
using System.Threading; |
||||
|
using System.Threading.Tasks; |
||||
|
using Microsoft.Extensions.Options; |
||||
|
using Squidex.Infrastructure.Commands; |
||||
|
using Squidex.Infrastructure.Migrations; |
||||
|
|
||||
|
namespace Migrations.Migrations |
||||
|
{ |
||||
|
public sealed class RebuildRules : IMigration |
||||
|
{ |
||||
|
private readonly Rebuilder rebuilder; |
||||
|
private readonly RebuildOptions rebuildOptions; |
||||
|
|
||||
|
public RebuildRules(Rebuilder rebuilder, |
||||
|
IOptions<RebuildOptions> rebuildOptions) |
||||
|
{ |
||||
|
this.rebuilder = rebuilder; |
||||
|
this.rebuildOptions = rebuildOptions.Value; |
||||
|
} |
||||
|
|
||||
|
public Task UpdateAsync( |
||||
|
CancellationToken ct) |
||||
|
{ |
||||
|
return rebuilder.RebuildRulesAsync(rebuildOptions.BatchSize, ct); |
||||
|
} |
||||
|
} |
||||
|
} |
||||
@ -0,0 +1,34 @@ |
|||||
|
// ==========================================================================
|
||||
|
// Squidex Headless CMS
|
||||
|
// ==========================================================================
|
||||
|
// Copyright (c) Squidex UG (haftungsbeschraenkt)
|
||||
|
// All rights reserved. Licensed under the MIT license.
|
||||
|
// ==========================================================================
|
||||
|
|
||||
|
using System.Threading; |
||||
|
using System.Threading.Tasks; |
||||
|
using Microsoft.Extensions.Options; |
||||
|
using Squidex.Infrastructure.Commands; |
||||
|
using Squidex.Infrastructure.Migrations; |
||||
|
|
||||
|
namespace Migrations.Migrations |
||||
|
{ |
||||
|
public sealed class RebuildSchemas : IMigration |
||||
|
{ |
||||
|
private readonly Rebuilder rebuilder; |
||||
|
private readonly RebuildOptions rebuildOptions; |
||||
|
|
||||
|
public RebuildSchemas(Rebuilder rebuilder, |
||||
|
IOptions<RebuildOptions> rebuildOptions) |
||||
|
{ |
||||
|
this.rebuilder = rebuilder; |
||||
|
this.rebuildOptions = rebuildOptions.Value; |
||||
|
} |
||||
|
|
||||
|
public Task UpdateAsync( |
||||
|
CancellationToken ct) |
||||
|
{ |
||||
|
return rebuilder.RebuildSchemasAsync(rebuildOptions.BatchSize, ct); |
||||
|
} |
||||
|
} |
||||
|
} |
||||
@ -0,0 +1,24 @@ |
|||||
|
// ==========================================================================
|
||||
|
// Squidex Headless CMS
|
||||
|
// ==========================================================================
|
||||
|
// Copyright (c) Squidex UG (haftungsbeschraenkt)
|
||||
|
// All rights reserved. Licensed under the MIT license.
|
||||
|
// ==========================================================================
|
||||
|
|
||||
|
using Squidex.Domain.Apps.Events; |
||||
|
using Squidex.Domain.Apps.Events.Apps; |
||||
|
using Squidex.Infrastructure.EventSourcing; |
||||
|
using Squidex.Infrastructure.Migrations; |
||||
|
using Squidex.Infrastructure.Reflection; |
||||
|
|
||||
|
namespace Migrations.OldEvents |
||||
|
{ |
||||
|
[EventType(nameof(AppArchived))] |
||||
|
public sealed class AppArchived : AppEvent, IMigrated<IEvent> |
||||
|
{ |
||||
|
public IEvent Migrate() |
||||
|
{ |
||||
|
return SimpleMapper.Map(this, new AppDeleted()); |
||||
|
} |
||||
|
} |
||||
|
} |
||||
@ -0,0 +1,46 @@ |
|||||
|
// ==========================================================================
|
||||
|
// Squidex Headless CMS
|
||||
|
// ==========================================================================
|
||||
|
// Copyright (c) Squidex UG (haftungsbeschraenkt)
|
||||
|
// All rights reserved. Licensed under the MIT license.
|
||||
|
// ==========================================================================
|
||||
|
|
||||
|
using System.Collections.Generic; |
||||
|
using System.Linq; |
||||
|
using MongoDB.Bson.Serialization.Attributes; |
||||
|
using Squidex.Domain.Apps.Entities.Apps.DomainObject; |
||||
|
using Squidex.Infrastructure; |
||||
|
using Squidex.Infrastructure.States; |
||||
|
|
||||
|
namespace Squidex.Domain.Apps.Entities.MongoDb.Apps |
||||
|
{ |
||||
|
public sealed class MongoAppEntity : MongoState<AppDomainObject.State> |
||||
|
{ |
||||
|
[BsonRequired] |
||||
|
[BsonElement("_an")] |
||||
|
public string IndexedName { get; set; } |
||||
|
|
||||
|
[BsonRequired] |
||||
|
[BsonElement("_ui")] |
||||
|
public string[] IndexedUserIds { get; set; } |
||||
|
|
||||
|
[BsonRequired] |
||||
|
[BsonElement("_dl")] |
||||
|
public bool IndexedDeleted { get; set; } |
||||
|
|
||||
|
public override void Prepare() |
||||
|
{ |
||||
|
var users = new HashSet<string> |
||||
|
{ |
||||
|
Document.CreatedBy.Identifier |
||||
|
}; |
||||
|
|
||||
|
users.AddRange(Document.Contributors.Keys); |
||||
|
users.AddRange(Document.Clients.Keys); |
||||
|
|
||||
|
IndexedUserIds = users.ToArray(); |
||||
|
IndexedDeleted = Document.IsDeleted; |
||||
|
IndexedName = Document.Name; |
||||
|
} |
||||
|
} |
||||
|
} |
||||
@ -0,0 +1,86 @@ |
|||||
|
// ==========================================================================
|
||||
|
// Squidex Headless CMS
|
||||
|
// ==========================================================================
|
||||
|
// Copyright (c) Squidex UG (haftungsbeschraenkt)
|
||||
|
// All rights reserved. Licensed under the MIT license.
|
||||
|
// ==========================================================================
|
||||
|
|
||||
|
using System.Collections.Generic; |
||||
|
using System.Linq; |
||||
|
using System.Threading; |
||||
|
using System.Threading.Tasks; |
||||
|
using MongoDB.Driver; |
||||
|
using Newtonsoft.Json; |
||||
|
using Squidex.Domain.Apps.Entities.Apps; |
||||
|
using Squidex.Domain.Apps.Entities.Apps.DomainObject; |
||||
|
using Squidex.Domain.Apps.Entities.Apps.Repositories; |
||||
|
using Squidex.Infrastructure; |
||||
|
using Squidex.Infrastructure.MongoDb; |
||||
|
using Squidex.Infrastructure.States; |
||||
|
|
||||
|
namespace Squidex.Domain.Apps.Entities.MongoDb.Apps |
||||
|
{ |
||||
|
public sealed class MongoAppRepository : MongoSnapshotStoreBase<AppDomainObject.State, MongoAppEntity>, IAppRepository, IDeleter |
||||
|
{ |
||||
|
public MongoAppRepository(IMongoDatabase database, JsonSerializer jsonSerializer) |
||||
|
: base(database, jsonSerializer) |
||||
|
{ |
||||
|
} |
||||
|
|
||||
|
protected override Task SetupCollectionAsync(IMongoCollection<MongoAppEntity> collection, |
||||
|
CancellationToken ct) |
||||
|
{ |
||||
|
return collection.Indexes.CreateManyAsync(new[] |
||||
|
{ |
||||
|
new CreateIndexModel<MongoAppEntity>( |
||||
|
Index |
||||
|
.Ascending(x => x.IndexedName)), |
||||
|
new CreateIndexModel<MongoAppEntity>( |
||||
|
Index |
||||
|
.Ascending(x => x.IndexedUserIds)) |
||||
|
}, ct); |
||||
|
} |
||||
|
|
||||
|
Task IDeleter.DeleteAppAsync(IAppEntity app, |
||||
|
CancellationToken ct) |
||||
|
{ |
||||
|
return Collection.DeleteManyAsync(Filter.Eq(x => x.DocumentId, app.Id), ct); |
||||
|
} |
||||
|
|
||||
|
public async Task<Dictionary<string, DomainId>> QueryIdsAsync(string contributorId, |
||||
|
CancellationToken ct = default) |
||||
|
{ |
||||
|
using (Telemetry.Activities.StartActivity("MongoAppRepository/QueryIdsAsync")) |
||||
|
{ |
||||
|
var find = Collection.Find(x => x.IndexedUserIds.Contains(contributorId) && !x.IndexedDeleted); |
||||
|
|
||||
|
return await QueryAsync(find, ct); |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
public async Task<Dictionary<string, DomainId>> QueryIdsAsync(IEnumerable<string> names, |
||||
|
CancellationToken ct = default) |
||||
|
{ |
||||
|
using (Telemetry.Activities.StartActivity("MongoAppRepository/QueryAsync")) |
||||
|
{ |
||||
|
var find = Collection.Find(x => names.Contains(x.IndexedName) && !x.IndexedDeleted); |
||||
|
|
||||
|
return await QueryAsync(find, ct); |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
private static async Task<Dictionary<string, DomainId>> QueryAsync(IFindFluent<MongoAppEntity, MongoAppEntity> find, |
||||
|
CancellationToken ct) |
||||
|
{ |
||||
|
var entities = await find.Only(x => x.DocumentId, x => x.IndexedName).ToListAsync(ct); |
||||
|
|
||||
|
return entities.Select(x => |
||||
|
{ |
||||
|
var indexedId = DomainId.Create(x["_id"].AsString); |
||||
|
var indexedName = x["_an"].AsString; |
||||
|
|
||||
|
return new { indexedName, indexedId }; |
||||
|
}).ToDictionary(x => x.indexedName, x => x.indexedId); |
||||
|
} |
||||
|
} |
||||
|
} |
||||
Some files were not shown because too many files changed in this diff
Loading…
Reference in new issue