mirror of https://github.com/Squidex/squidex.git
Browse Source
* Started with a few repositories. * Get rid unnecessary allocations. * New repositories and cancellation token improvements. * First untested implementation. * Fix infrastructure tests. * Warnings fixed. * All tests green again. * New tests. * Better cancellation. * More tests. * Backup fix. * Markdown everything * A few more tests and fixes. * Fix app deletion flag. * Provide app to client creator. * Started with migration. * Temp. * Minor change to rename app deletion. * Toggle for deleter. * Fix build.pull/768/head
Sebastian Stehle
4 years ago
committed by
GitHub
GitHub
544 changed files with 6609 additions and 4485 deletions
@ -1,189 +0,0 @@ |
|||
// ==========================================================================
|
|||
// Squidex Headless CMS
|
|||
// ==========================================================================
|
|||
// Copyright (c) Squidex UG (haftungsbeschraenkt)
|
|||
// All rights reserved. Licensed under the MIT license.
|
|||
// ==========================================================================
|
|||
|
|||
using System.Collections.Generic; |
|||
using System.Threading; |
|||
using System.Threading.Tasks; |
|||
using Squidex.Domain.Apps.Entities.Apps.Indexes; |
|||
using Squidex.Domain.Apps.Entities.Rules.Indexes; |
|||
using Squidex.Domain.Apps.Entities.Schemas.Indexes; |
|||
using Squidex.Domain.Apps.Events; |
|||
using Squidex.Domain.Apps.Events.Apps; |
|||
using Squidex.Domain.Apps.Events.Rules; |
|||
using Squidex.Domain.Apps.Events.Schemas; |
|||
using Squidex.Infrastructure; |
|||
using Squidex.Infrastructure.EventSourcing; |
|||
using Squidex.Infrastructure.Migrations; |
|||
|
|||
namespace Migrations.Migrations |
|||
{ |
|||
public class PopulateGrainIndexes : IMigration |
|||
{ |
|||
private readonly IAppsIndex indexApps; |
|||
private readonly IRulesIndex indexRules; |
|||
private readonly ISchemasIndex indexSchemas; |
|||
private readonly IEventDataFormatter eventDataFormatter; |
|||
private readonly IEventStore eventStore; |
|||
|
|||
public PopulateGrainIndexes(IAppsIndex indexApps, IRulesIndex indexRules, ISchemasIndex indexSchemas, |
|||
IEventDataFormatter eventDataFormatter, |
|||
IEventStore eventStore) |
|||
{ |
|||
this.indexApps = indexApps; |
|||
this.indexRules = indexRules; |
|||
this.indexSchemas = indexSchemas; |
|||
this.eventDataFormatter = eventDataFormatter; |
|||
this.eventStore = eventStore; |
|||
} |
|||
|
|||
public Task UpdateAsync(CancellationToken ct) |
|||
{ |
|||
return Task.WhenAll( |
|||
RebuildAppIndexes(ct), |
|||
RebuildRuleIndexes(ct), |
|||
RebuildSchemaIndexes(ct)); |
|||
} |
|||
|
|||
private async Task RebuildAppIndexes(CancellationToken ct) |
|||
{ |
|||
var appsByName = new Dictionary<string, DomainId>(); |
|||
var appsByUser = new Dictionary<string, HashSet<DomainId>>(); |
|||
|
|||
bool HasApp(NamedId<DomainId> appId, bool consistent, out DomainId id) |
|||
{ |
|||
return appsByName!.TryGetValue(appId.Name, out id) && (!consistent || id == appId.Id); |
|||
} |
|||
|
|||
HashSet<DomainId> Index(string contributorId) |
|||
{ |
|||
return appsByUser!.GetOrAddNew(contributorId); |
|||
} |
|||
|
|||
void RemoveApp(NamedId<DomainId> appId, bool consistent) |
|||
{ |
|||
if (HasApp(appId, consistent, out var id)) |
|||
{ |
|||
foreach (var apps in appsByUser!.Values) |
|||
{ |
|||
apps.Remove(id); |
|||
} |
|||
|
|||
appsByName!.Remove(appId.Name); |
|||
} |
|||
} |
|||
|
|||
await foreach (var storedEvent in eventStore.QueryAllAsync("^app\\-", ct: ct)) |
|||
{ |
|||
var @event = eventDataFormatter.ParseIfKnown(storedEvent); |
|||
|
|||
if (@event != null) |
|||
{ |
|||
switch (@event.Payload) |
|||
{ |
|||
case AppCreated created: |
|||
{ |
|||
RemoveApp(created.AppId, false); |
|||
|
|||
appsByName[created.Name] = created.AppId.Id; |
|||
break; |
|||
} |
|||
|
|||
case AppContributorAssigned contributorAssigned: |
|||
{ |
|||
if (HasApp(contributorAssigned.AppId, true, out _)) |
|||
{ |
|||
Index(contributorAssigned.ContributorId).Add(contributorAssigned.AppId.Id); |
|||
} |
|||
|
|||
break; |
|||
} |
|||
|
|||
case AppContributorRemoved contributorRemoved: |
|||
Index(contributorRemoved.ContributorId).Remove(contributorRemoved.AppId.Id); |
|||
break; |
|||
case AppArchived archived: |
|||
RemoveApp(archived.AppId, true); |
|||
break; |
|||
} |
|||
} |
|||
} |
|||
|
|||
await indexApps.RebuildAsync(appsByName); |
|||
|
|||
foreach (var (contributorId, apps) in appsByUser) |
|||
{ |
|||
await indexApps.RebuildByContributorsAsync(contributorId, apps); |
|||
} |
|||
} |
|||
|
|||
private async Task RebuildRuleIndexes(CancellationToken ct) |
|||
{ |
|||
var rulesByApp = new Dictionary<DomainId, HashSet<DomainId>>(); |
|||
|
|||
HashSet<DomainId> Index(RuleEvent @event) |
|||
{ |
|||
return rulesByApp!.GetOrAddNew(@event.AppId.Id); |
|||
} |
|||
|
|||
await foreach (var storedEvent in eventStore.QueryAllAsync("^rule\\-", ct: ct)) |
|||
{ |
|||
var @event = eventDataFormatter.ParseIfKnown(storedEvent); |
|||
|
|||
if (@event != null) |
|||
{ |
|||
switch (@event.Payload) |
|||
{ |
|||
case RuleCreated created: |
|||
Index(created).Add(created.RuleId); |
|||
break; |
|||
case RuleDeleted deleted: |
|||
Index(deleted).Remove(deleted.RuleId); |
|||
break; |
|||
} |
|||
} |
|||
} |
|||
|
|||
foreach (var (appId, rules) in rulesByApp) |
|||
{ |
|||
await indexRules.RebuildAsync(appId, rules); |
|||
} |
|||
} |
|||
|
|||
private async Task RebuildSchemaIndexes(CancellationToken ct) |
|||
{ |
|||
var schemasByApp = new Dictionary<DomainId, Dictionary<string, DomainId>>(); |
|||
|
|||
Dictionary<string, DomainId> Index(SchemaEvent @event) |
|||
{ |
|||
return schemasByApp!.GetOrAddNew(@event.AppId.Id); |
|||
} |
|||
|
|||
await foreach (var storedEvent in eventStore.QueryAllAsync("^schema\\-", ct: ct)) |
|||
{ |
|||
var @event = eventDataFormatter.ParseIfKnown(storedEvent); |
|||
|
|||
if (@event != null) |
|||
{ |
|||
switch (@event.Payload) |
|||
{ |
|||
case SchemaCreated created: |
|||
Index(created)[created.SchemaId.Name] = created.SchemaId.Id; |
|||
break; |
|||
case SchemaDeleted deleted: |
|||
Index(deleted).Remove(deleted.SchemaId.Name); |
|||
break; |
|||
} |
|||
} |
|||
} |
|||
|
|||
foreach (var (appId, schemas) in schemasByApp) |
|||
{ |
|||
await indexSchemas.RebuildAsync(appId, schemas); |
|||
} |
|||
} |
|||
} |
|||
} |
|||
@ -0,0 +1,34 @@ |
|||
// ==========================================================================
|
|||
// Squidex Headless CMS
|
|||
// ==========================================================================
|
|||
// Copyright (c) Squidex UG (haftungsbeschraenkt)
|
|||
// All rights reserved. Licensed under the MIT license.
|
|||
// ==========================================================================
|
|||
|
|||
using System.Threading; |
|||
using System.Threading.Tasks; |
|||
using Microsoft.Extensions.Options; |
|||
using Squidex.Infrastructure.Commands; |
|||
using Squidex.Infrastructure.Migrations; |
|||
|
|||
namespace Migrations.Migrations |
|||
{ |
|||
public sealed class RebuildRules : IMigration |
|||
{ |
|||
private readonly Rebuilder rebuilder; |
|||
private readonly RebuildOptions rebuildOptions; |
|||
|
|||
public RebuildRules(Rebuilder rebuilder, |
|||
IOptions<RebuildOptions> rebuildOptions) |
|||
{ |
|||
this.rebuilder = rebuilder; |
|||
this.rebuildOptions = rebuildOptions.Value; |
|||
} |
|||
|
|||
public Task UpdateAsync( |
|||
CancellationToken ct) |
|||
{ |
|||
return rebuilder.RebuildRulesAsync(rebuildOptions.BatchSize, ct); |
|||
} |
|||
} |
|||
} |
|||
@ -0,0 +1,34 @@ |
|||
// ==========================================================================
|
|||
// Squidex Headless CMS
|
|||
// ==========================================================================
|
|||
// Copyright (c) Squidex UG (haftungsbeschraenkt)
|
|||
// All rights reserved. Licensed under the MIT license.
|
|||
// ==========================================================================
|
|||
|
|||
using System.Threading; |
|||
using System.Threading.Tasks; |
|||
using Microsoft.Extensions.Options; |
|||
using Squidex.Infrastructure.Commands; |
|||
using Squidex.Infrastructure.Migrations; |
|||
|
|||
namespace Migrations.Migrations |
|||
{ |
|||
public sealed class RebuildSchemas : IMigration |
|||
{ |
|||
private readonly Rebuilder rebuilder; |
|||
private readonly RebuildOptions rebuildOptions; |
|||
|
|||
public RebuildSchemas(Rebuilder rebuilder, |
|||
IOptions<RebuildOptions> rebuildOptions) |
|||
{ |
|||
this.rebuilder = rebuilder; |
|||
this.rebuildOptions = rebuildOptions.Value; |
|||
} |
|||
|
|||
public Task UpdateAsync( |
|||
CancellationToken ct) |
|||
{ |
|||
return rebuilder.RebuildSchemasAsync(rebuildOptions.BatchSize, ct); |
|||
} |
|||
} |
|||
} |
|||
@ -0,0 +1,24 @@ |
|||
// ==========================================================================
|
|||
// Squidex Headless CMS
|
|||
// ==========================================================================
|
|||
// Copyright (c) Squidex UG (haftungsbeschraenkt)
|
|||
// All rights reserved. Licensed under the MIT license.
|
|||
// ==========================================================================
|
|||
|
|||
using Squidex.Domain.Apps.Events; |
|||
using Squidex.Domain.Apps.Events.Apps; |
|||
using Squidex.Infrastructure.EventSourcing; |
|||
using Squidex.Infrastructure.Migrations; |
|||
using Squidex.Infrastructure.Reflection; |
|||
|
|||
namespace Migrations.OldEvents |
|||
{ |
|||
[EventType(nameof(AppArchived))] |
|||
public sealed class AppArchived : AppEvent, IMigrated<IEvent> |
|||
{ |
|||
public IEvent Migrate() |
|||
{ |
|||
return SimpleMapper.Map(this, new AppDeleted()); |
|||
} |
|||
} |
|||
} |
|||
@ -0,0 +1,46 @@ |
|||
// ==========================================================================
|
|||
// Squidex Headless CMS
|
|||
// ==========================================================================
|
|||
// Copyright (c) Squidex UG (haftungsbeschraenkt)
|
|||
// All rights reserved. Licensed under the MIT license.
|
|||
// ==========================================================================
|
|||
|
|||
using System.Collections.Generic; |
|||
using System.Linq; |
|||
using MongoDB.Bson.Serialization.Attributes; |
|||
using Squidex.Domain.Apps.Entities.Apps.DomainObject; |
|||
using Squidex.Infrastructure; |
|||
using Squidex.Infrastructure.States; |
|||
|
|||
namespace Squidex.Domain.Apps.Entities.MongoDb.Apps |
|||
{ |
|||
public sealed class MongoAppEntity : MongoState<AppDomainObject.State> |
|||
{ |
|||
[BsonRequired] |
|||
[BsonElement("_an")] |
|||
public string IndexedName { get; set; } |
|||
|
|||
[BsonRequired] |
|||
[BsonElement("_ui")] |
|||
public string[] IndexedUserIds { get; set; } |
|||
|
|||
[BsonRequired] |
|||
[BsonElement("_dl")] |
|||
public bool IndexedDeleted { get; set; } |
|||
|
|||
public override void Prepare() |
|||
{ |
|||
var users = new HashSet<string> |
|||
{ |
|||
Document.CreatedBy.Identifier |
|||
}; |
|||
|
|||
users.AddRange(Document.Contributors.Keys); |
|||
users.AddRange(Document.Clients.Keys); |
|||
|
|||
IndexedUserIds = users.ToArray(); |
|||
IndexedDeleted = Document.IsDeleted; |
|||
IndexedName = Document.Name; |
|||
} |
|||
} |
|||
} |
|||
@ -0,0 +1,86 @@ |
|||
// ==========================================================================
|
|||
// Squidex Headless CMS
|
|||
// ==========================================================================
|
|||
// Copyright (c) Squidex UG (haftungsbeschraenkt)
|
|||
// All rights reserved. Licensed under the MIT license.
|
|||
// ==========================================================================
|
|||
|
|||
using System.Collections.Generic; |
|||
using System.Linq; |
|||
using System.Threading; |
|||
using System.Threading.Tasks; |
|||
using MongoDB.Driver; |
|||
using Newtonsoft.Json; |
|||
using Squidex.Domain.Apps.Entities.Apps; |
|||
using Squidex.Domain.Apps.Entities.Apps.DomainObject; |
|||
using Squidex.Domain.Apps.Entities.Apps.Repositories; |
|||
using Squidex.Infrastructure; |
|||
using Squidex.Infrastructure.MongoDb; |
|||
using Squidex.Infrastructure.States; |
|||
|
|||
namespace Squidex.Domain.Apps.Entities.MongoDb.Apps |
|||
{ |
|||
public sealed class MongoAppRepository : MongoSnapshotStoreBase<AppDomainObject.State, MongoAppEntity>, IAppRepository, IDeleter |
|||
{ |
|||
public MongoAppRepository(IMongoDatabase database, JsonSerializer jsonSerializer) |
|||
: base(database, jsonSerializer) |
|||
{ |
|||
} |
|||
|
|||
protected override Task SetupCollectionAsync(IMongoCollection<MongoAppEntity> collection, |
|||
CancellationToken ct) |
|||
{ |
|||
return collection.Indexes.CreateManyAsync(new[] |
|||
{ |
|||
new CreateIndexModel<MongoAppEntity>( |
|||
Index |
|||
.Ascending(x => x.IndexedName)), |
|||
new CreateIndexModel<MongoAppEntity>( |
|||
Index |
|||
.Ascending(x => x.IndexedUserIds)) |
|||
}, ct); |
|||
} |
|||
|
|||
Task IDeleter.DeleteAppAsync(IAppEntity app, |
|||
CancellationToken ct) |
|||
{ |
|||
return Collection.DeleteManyAsync(Filter.Eq(x => x.DocumentId, app.Id), ct); |
|||
} |
|||
|
|||
public async Task<Dictionary<string, DomainId>> QueryIdsAsync(string contributorId, |
|||
CancellationToken ct = default) |
|||
{ |
|||
using (Telemetry.Activities.StartActivity("MongoAppRepository/QueryIdsAsync")) |
|||
{ |
|||
var find = Collection.Find(x => x.IndexedUserIds.Contains(contributorId) && !x.IndexedDeleted); |
|||
|
|||
return await QueryAsync(find, ct); |
|||
} |
|||
} |
|||
|
|||
public async Task<Dictionary<string, DomainId>> QueryIdsAsync(IEnumerable<string> names, |
|||
CancellationToken ct = default) |
|||
{ |
|||
using (Telemetry.Activities.StartActivity("MongoAppRepository/QueryAsync")) |
|||
{ |
|||
var find = Collection.Find(x => names.Contains(x.IndexedName) && !x.IndexedDeleted); |
|||
|
|||
return await QueryAsync(find, ct); |
|||
} |
|||
} |
|||
|
|||
private static async Task<Dictionary<string, DomainId>> QueryAsync(IFindFluent<MongoAppEntity, MongoAppEntity> find, |
|||
CancellationToken ct) |
|||
{ |
|||
var entities = await find.Only(x => x.DocumentId, x => x.IndexedName).ToListAsync(ct); |
|||
|
|||
return entities.Select(x => |
|||
{ |
|||
var indexedId = DomainId.Create(x["_id"].AsString); |
|||
var indexedName = x["_an"].AsString; |
|||
|
|||
return new { indexedName, indexedId }; |
|||
}).ToDictionary(x => x.indexedName, x => x.indexedId); |
|||
} |
|||
} |
|||
} |
|||
Some files were not shown because too many files changed in this diff
Loading…
Reference in new issue