mirror of https://github.com/Squidex/squidex.git
Sebastian
3 years ago
19 changed files with 467 additions and 275 deletions
@ -1,154 +0,0 @@ |
|||||
// ==========================================================================
|
|
||||
// Squidex Headless CMS
|
|
||||
// ==========================================================================
|
|
||||
// Copyright (c) Squidex UG (haftungsbeschraenkt)
|
|
||||
// All rights reserved. Licensed under the MIT license.
|
|
||||
// ==========================================================================
|
|
||||
|
|
||||
using System.Threading.Channels; |
|
||||
using Squidex.Infrastructure.Tasks; |
|
||||
|
|
||||
#pragma warning disable RECS0082 // Parameter has the same name as a member and hides it
|
|
||||
#pragma warning disable SA1313 // Parameter names should begin with lower-case letter
|
|
||||
|
|
||||
namespace Squidex.Infrastructure.EventSourcing.Consume |
|
||||
{ |
|
||||
internal sealed class BatchSubscriber : IEventSubscriber, IEventSubscription |
|
||||
{ |
|
||||
private readonly IEventSubscription eventSubscription; |
|
||||
private readonly Channel<object> taskQueue; |
|
||||
private readonly Channel<EventSource> parseQueue; |
|
||||
private readonly Task handleTask; |
|
||||
private readonly CancellationTokenSource completed = new CancellationTokenSource(); |
|
||||
|
|
||||
private sealed record EventSource(StoredEvent StoredEvent); |
|
||||
private sealed record BatchItem(Envelope<IEvent>? Event, string Position); |
|
||||
private sealed record BatchJob(BatchItem[] Items); |
|
||||
private sealed record ErrorJob(Exception Exception); |
|
||||
|
|
||||
public BatchSubscriber( |
|
||||
EventConsumerProcessor processor, |
|
||||
IEventFormatter eventFormatter, |
|
||||
IEventConsumer eventConsumer, |
|
||||
Func<IEventSubscriber, IEventSubscription> factory) |
|
||||
{ |
|
||||
eventSubscription = factory(this); |
|
||||
|
|
||||
var batchSize = Math.Max(1, eventConsumer.BatchSize); |
|
||||
var batchDelay = Math.Max(100, eventConsumer.BatchDelay); |
|
||||
|
|
||||
parseQueue = Channel.CreateBounded<EventSource>(new BoundedChannelOptions(batchSize) |
|
||||
{ |
|
||||
AllowSynchronousContinuations = true, |
|
||||
SingleReader = true, |
|
||||
SingleWriter = true |
|
||||
}); |
|
||||
|
|
||||
taskQueue = Channel.CreateBounded<object>(new BoundedChannelOptions(2) |
|
||||
{ |
|
||||
SingleReader = true, |
|
||||
SingleWriter = true |
|
||||
}); |
|
||||
|
|
||||
var batchQueue = Channel.CreateBounded<object>(new BoundedChannelOptions(batchSize) |
|
||||
{ |
|
||||
AllowSynchronousContinuations = true, |
|
||||
SingleReader = true, |
|
||||
SingleWriter = true |
|
||||
}); |
|
||||
|
|
||||
#pragma warning disable MA0040 // Flow the cancellation token
|
|
||||
batchQueue.Batch<BatchItem, object>(taskQueue, x => new BatchJob(x.ToArray()), batchSize, batchDelay); |
|
||||
|
|
||||
Task.Run(async () => |
|
||||
{ |
|
||||
await foreach (var source in parseQueue.Reader.ReadAllAsync(completed.Token)) |
|
||||
{ |
|
||||
var storedEvent = source.StoredEvent; |
|
||||
try |
|
||||
{ |
|
||||
Envelope<IEvent>? @event = null; |
|
||||
|
|
||||
if (eventConsumer.Handles(storedEvent)) |
|
||||
{ |
|
||||
@event = eventFormatter.ParseIfKnown(storedEvent); |
|
||||
} |
|
||||
|
|
||||
await batchQueue.Writer.WriteAsync(new BatchItem(@event, storedEvent.EventPosition), completed.Token); |
|
||||
} |
|
||||
catch (Exception ex) |
|
||||
{ |
|
||||
await taskQueue.Writer.WriteAsync(new ErrorJob(ex), completed.Token); |
|
||||
} |
|
||||
} |
|
||||
}).ContinueWith(x => batchQueue.Writer.TryComplete(x.Exception)); |
|
||||
#pragma warning restore MA0040 // Flow the cancellation token
|
|
||||
|
|
||||
handleTask = Run(processor); |
|
||||
} |
|
||||
|
|
||||
private async Task Run(EventConsumerProcessor processor) |
|
||||
{ |
|
||||
try |
|
||||
{ |
|
||||
await foreach (var task in taskQueue.Reader.ReadAllAsync(completed.Token)) |
|
||||
{ |
|
||||
switch (task) |
|
||||
{ |
|
||||
case ErrorJob { Exception: not OperationCanceledException } error: |
|
||||
{ |
|
||||
await processor.OnErrorAsync(this, error.Exception); |
|
||||
break; |
|
||||
} |
|
||||
|
|
||||
case BatchJob batch: |
|
||||
{ |
|
||||
var eventsPosition = batch.Items[^1].Position; |
|
||||
var eventsCollection = batch.Items.Select(x => x.Event).NotNull().ToList(); |
|
||||
|
|
||||
await processor.OnEventsAsync(this, eventsCollection, eventsPosition); |
|
||||
break; |
|
||||
} |
|
||||
} |
|
||||
} |
|
||||
} |
|
||||
catch (OperationCanceledException) |
|
||||
{ |
|
||||
return; |
|
||||
} |
|
||||
catch (Exception ex) |
|
||||
{ |
|
||||
await processor.OnErrorAsync(this, ex); |
|
||||
} |
|
||||
} |
|
||||
|
|
||||
public Task CompleteAsync() |
|
||||
{ |
|
||||
parseQueue.Writer.TryComplete(); |
|
||||
|
|
||||
return handleTask; |
|
||||
} |
|
||||
|
|
||||
public void Dispose() |
|
||||
{ |
|
||||
completed.Cancel(); |
|
||||
|
|
||||
eventSubscription.Dispose(); |
|
||||
} |
|
||||
|
|
||||
public void WakeUp() |
|
||||
{ |
|
||||
eventSubscription.WakeUp(); |
|
||||
} |
|
||||
|
|
||||
ValueTask IEventSubscriber.OnEventAsync(IEventSubscription subscription, StoredEvent storedEvent) |
|
||||
{ |
|
||||
return parseQueue.Writer.WriteAsync(new EventSource(storedEvent), completed.Token); |
|
||||
} |
|
||||
|
|
||||
ValueTask IEventSubscriber.OnErrorAsync(IEventSubscription subscription, Exception exception) |
|
||||
{ |
|
||||
return taskQueue.Writer.WriteAsync(new ErrorJob(exception), completed.Token); |
|
||||
} |
|
||||
} |
|
||||
} |
|
||||
@ -0,0 +1,160 @@ |
|||||
|
// ==========================================================================
|
||||
|
// Squidex Headless CMS
|
||||
|
// ==========================================================================
|
||||
|
// Copyright (c) Squidex UG (haftungsbeschraenkt)
|
||||
|
// All rights reserved. Licensed under the MIT license.
|
||||
|
// ==========================================================================
|
||||
|
|
||||
|
using System.Threading.Channels; |
||||
|
using Squidex.Infrastructure.Tasks; |
||||
|
|
||||
|
namespace Squidex.Infrastructure.EventSourcing.Consume |
||||
|
{ |
||||
|
internal sealed class BatchSubscription : IEventSubscriber<ParsedEvent>, IEventSubscription |
||||
|
{ |
||||
|
private readonly IEventSubscription eventSubscription; |
||||
|
private readonly Channel<object> taskQueue; |
||||
|
private readonly Channel<object> batchQueue; |
||||
|
private readonly Task handleTask; |
||||
|
private readonly CancellationTokenSource completed = new CancellationTokenSource(); |
||||
|
|
||||
|
public BatchSubscription( |
||||
|
IEventConsumer eventConsumer, |
||||
|
IEventSubscriber<ParsedEvents> eventSubscriber, |
||||
|
EventSubscriptionSource<ParsedEvent> eventSource) |
||||
|
{ |
||||
|
eventSubscription = eventSource(this); |
||||
|
|
||||
|
var batchSize = Math.Max(1, eventConsumer.BatchSize); |
||||
|
var batchDelay = Math.Max(100, eventConsumer.BatchDelay); |
||||
|
|
||||
|
taskQueue = Channel.CreateBounded<object>(new BoundedChannelOptions(2) |
||||
|
{ |
||||
|
SingleReader = true, |
||||
|
SingleWriter = true |
||||
|
}); |
||||
|
|
||||
|
batchQueue = Channel.CreateBounded<object>(new BoundedChannelOptions(batchSize) |
||||
|
{ |
||||
|
AllowSynchronousContinuations = true, |
||||
|
SingleReader = true, |
||||
|
SingleWriter = true |
||||
|
}); |
||||
|
|
||||
|
batchQueue.Batch<ParsedEvent>(taskQueue, batchSize, batchDelay, completed.Token); |
||||
|
|
||||
|
handleTask = Run(eventSubscriber); |
||||
|
} |
||||
|
|
||||
|
private async Task Run(IEventSubscriber<ParsedEvents> eventSink) |
||||
|
{ |
||||
|
try |
||||
|
{ |
||||
|
var isStopped = false; |
||||
|
|
||||
|
await foreach (var task in taskQueue.Reader.ReadAllAsync(completed.Token)) |
||||
|
{ |
||||
|
switch (task) |
||||
|
{ |
||||
|
case Exception exception when exception is not OperationCanceledException: |
||||
|
{ |
||||
|
if (!completed.IsCancellationRequested) |
||||
|
{ |
||||
|
await eventSink.OnErrorAsync(this, exception); |
||||
|
} |
||||
|
|
||||
|
isStopped = true; |
||||
|
break; |
||||
|
} |
||||
|
|
||||
|
case List<ParsedEvent> batch: |
||||
|
{ |
||||
|
if (!completed.IsCancellationRequested) |
||||
|
{ |
||||
|
// Events can be null if the event consumer is not interested in the stored event.
|
||||
|
var eventList = batch.Select(x => x.Event).NotNull().ToList(); |
||||
|
var eventPosition = batch[^1].Position; |
||||
|
|
||||
|
// Use a struct here to save a few allocations.
|
||||
|
await eventSink.OnNextAsync(this, new ParsedEvents(eventList, eventPosition)); |
||||
|
} |
||||
|
|
||||
|
break; |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
if (isStopped) |
||||
|
{ |
||||
|
break; |
||||
|
} |
||||
|
} |
||||
|
} |
||||
|
catch (OperationCanceledException) |
||||
|
{ |
||||
|
return; |
||||
|
} |
||||
|
catch (Exception ex) |
||||
|
{ |
||||
|
if (!completed.IsCancellationRequested) |
||||
|
{ |
||||
|
await eventSink.OnErrorAsync(this, ex); |
||||
|
} |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
public void Dispose() |
||||
|
{ |
||||
|
if (completed.IsCancellationRequested) |
||||
|
{ |
||||
|
return; |
||||
|
} |
||||
|
|
||||
|
// It is not necessary to dispose the cancellation token source.
|
||||
|
completed.Cancel(); |
||||
|
|
||||
|
// We do not lock here, it is the responsibility of the source subscription to be thread safe.
|
||||
|
eventSubscription.Dispose(); |
||||
|
} |
||||
|
|
||||
|
public async ValueTask CompleteAsync() |
||||
|
{ |
||||
|
await eventSubscription.CompleteAsync(); |
||||
|
|
||||
|
batchQueue.Writer.TryComplete(); |
||||
|
|
||||
|
await handleTask; |
||||
|
} |
||||
|
|
||||
|
public void WakeUp() |
||||
|
{ |
||||
|
eventSubscription.WakeUp(); |
||||
|
} |
||||
|
|
||||
|
async ValueTask IEventSubscriber<ParsedEvent>.OnErrorAsync(IEventSubscription subscription, Exception exception) |
||||
|
{ |
||||
|
try |
||||
|
{ |
||||
|
// Forward the exception from one task only, but bypass the batch.
|
||||
|
await taskQueue.Writer.WriteAsync(exception, completed.Token); |
||||
|
} |
||||
|
catch (ChannelClosedException) |
||||
|
{ |
||||
|
// This exception is acceptable and happens when an exception has been thrown before.
|
||||
|
return; |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
async ValueTask IEventSubscriber<ParsedEvent>.OnNextAsync(IEventSubscription subscription, ParsedEvent @event) |
||||
|
{ |
||||
|
try |
||||
|
{ |
||||
|
await batchQueue.Writer.WriteAsync(@event, completed.Token); |
||||
|
} |
||||
|
catch (ChannelClosedException) |
||||
|
{ |
||||
|
// This exception is acceptable and happens when an exception has been thrown before.
|
||||
|
return; |
||||
|
} |
||||
|
} |
||||
|
} |
||||
|
} |
||||
@ -0,0 +1,149 @@ |
|||||
|
// ==========================================================================
|
||||
|
// Squidex Headless CMS
|
||||
|
// ==========================================================================
|
||||
|
// Copyright (c) Squidex UG (haftungsbeschraenkt)
|
||||
|
// All rights reserved. Licensed under the MIT license.
|
||||
|
// ==========================================================================
|
||||
|
|
||||
|
using System.Threading.Channels; |
||||
|
|
||||
|
namespace Squidex.Infrastructure.EventSourcing.Consume |
||||
|
{ |
||||
|
internal sealed class ParseSubscription : IEventSubscriber<StoredEvent>, IEventSubscription |
||||
|
{ |
||||
|
private readonly Channel<object> deserializeQueue; |
||||
|
private readonly CancellationTokenSource completed = new CancellationTokenSource(); |
||||
|
private readonly Task deserializeTask; |
||||
|
private readonly IEventSubscription eventSubscription; |
||||
|
|
||||
|
public ParseSubscription( |
||||
|
IEventConsumer eventConsumer, |
||||
|
IEventFormatter eventFormatter, |
||||
|
IEventSubscriber<ParsedEvent> eventSubscriber, |
||||
|
EventSubscriptionSource<StoredEvent> eventSource) |
||||
|
{ |
||||
|
eventSubscription = eventSource(this); |
||||
|
|
||||
|
deserializeQueue = Channel.CreateBounded<object>(new BoundedChannelOptions(2) |
||||
|
{ |
||||
|
AllowSynchronousContinuations = true, |
||||
|
SingleReader = true, |
||||
|
SingleWriter = true |
||||
|
}); |
||||
|
|
||||
|
#pragma warning disable MA0040 // Flow the cancellation token
|
||||
|
deserializeTask = Task.Run(async () => |
||||
|
{ |
||||
|
try |
||||
|
{ |
||||
|
var isFailed = false; |
||||
|
|
||||
|
await foreach (var input in deserializeQueue.Reader.ReadAllAsync(completed.Token)) |
||||
|
{ |
||||
|
switch (input) |
||||
|
{ |
||||
|
case Exception exception: |
||||
|
{ |
||||
|
// Not very likely that the task is cancelled.
|
||||
|
await eventSubscriber.OnErrorAsync(this, exception); |
||||
|
|
||||
|
isFailed = true; |
||||
|
break; |
||||
|
} |
||||
|
|
||||
|
case StoredEvent storedEvent: |
||||
|
{ |
||||
|
Envelope<IEvent>? @event = null; |
||||
|
|
||||
|
if (eventConsumer.Handles(storedEvent)) |
||||
|
{ |
||||
|
@event = eventFormatter.ParseIfKnown(storedEvent); |
||||
|
} |
||||
|
|
||||
|
// Parsing takes a little bit of time, so the task might have been cancelled.
|
||||
|
if (!completed.IsCancellationRequested) |
||||
|
{ |
||||
|
// Also invoke the subscriber if the event is null to update the position.
|
||||
|
await eventSubscriber.OnNextAsync(this, new ParsedEvent(@event, storedEvent.EventPosition)); |
||||
|
} |
||||
|
|
||||
|
break; |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
if (isFailed) |
||||
|
{ |
||||
|
break; |
||||
|
} |
||||
|
} |
||||
|
} |
||||
|
catch (OperationCanceledException) |
||||
|
{ |
||||
|
return; |
||||
|
} |
||||
|
catch (Exception ex) |
||||
|
{ |
||||
|
if (!completed.IsCancellationRequested) |
||||
|
{ |
||||
|
await eventSubscriber.OnErrorAsync(this, ex); |
||||
|
} |
||||
|
} |
||||
|
}).ContinueWith(x => deserializeQueue.Writer.TryComplete(x.Exception)); |
||||
|
} |
||||
|
|
||||
|
public void Dispose() |
||||
|
{ |
||||
|
if (completed.IsCancellationRequested) |
||||
|
{ |
||||
|
return; |
||||
|
} |
||||
|
|
||||
|
// It is not necessary to dispose the cancellation token source.
|
||||
|
completed.Cancel(); |
||||
|
|
||||
|
// We do not lock here, it is the responsibility of the source subscription to be thread safe.
|
||||
|
eventSubscription.Dispose(); |
||||
|
} |
||||
|
|
||||
|
public async ValueTask CompleteAsync() |
||||
|
{ |
||||
|
await eventSubscription.CompleteAsync(); |
||||
|
|
||||
|
deserializeQueue.Writer.TryComplete(); |
||||
|
|
||||
|
await deserializeTask; |
||||
|
} |
||||
|
|
||||
|
public void WakeUp() |
||||
|
{ |
||||
|
eventSubscription.WakeUp(); |
||||
|
} |
||||
|
|
||||
|
async ValueTask IEventSubscriber<StoredEvent>.OnErrorAsync(IEventSubscription subscription, Exception exception) |
||||
|
{ |
||||
|
try |
||||
|
{ |
||||
|
// Forward the exception from one task only.
|
||||
|
await deserializeQueue.Writer.WriteAsync(exception, completed.Token); |
||||
|
} |
||||
|
catch (ChannelClosedException) |
||||
|
{ |
||||
|
// This exception is acceptable and happens when an exception has been thrown before.
|
||||
|
return; |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
async ValueTask IEventSubscriber<StoredEvent>.OnNextAsync(IEventSubscription subscription, StoredEvent @event) |
||||
|
{ |
||||
|
try |
||||
|
{ |
||||
|
await deserializeQueue.Writer.WriteAsync(@event, completed.Token); |
||||
|
} |
||||
|
catch (ChannelClosedException) |
||||
|
{ |
||||
|
// This exception is acceptable and happens when an exception has been thrown before.
|
||||
|
return; |
||||
|
} |
||||
|
} |
||||
|
} |
||||
|
} |
||||
@ -0,0 +1,16 @@ |
|||||
|
// ==========================================================================
|
||||
|
// Squidex Headless CMS
|
||||
|
// ==========================================================================
|
||||
|
// Copyright (c) Squidex UG (haftungsbeschraenkt)
|
||||
|
// All rights reserved. Licensed under the MIT license.
|
||||
|
// ==========================================================================
|
||||
|
|
||||
|
#pragma warning disable SA1313 // Parameter names should begin with lower-case letter
|
||||
|
#pragma warning disable MA0048 // File name must match type name
|
||||
|
|
||||
|
namespace Squidex.Infrastructure.EventSourcing.Consume |
||||
|
{ |
||||
|
public record struct ParsedEvent(Envelope<IEvent>? Event, string Position); |
||||
|
|
||||
|
public record struct ParsedEvents(List<Envelope<IEvent>> Events, string Position); |
||||
|
} |
||||
Loading…
Reference in new issue