mirror of https://github.com/Squidex/squidex.git
Sebastian
3 years ago
19 changed files with 467 additions and 275 deletions
@ -1,154 +0,0 @@ |
|||
// ==========================================================================
|
|||
// Squidex Headless CMS
|
|||
// ==========================================================================
|
|||
// Copyright (c) Squidex UG (haftungsbeschraenkt)
|
|||
// All rights reserved. Licensed under the MIT license.
|
|||
// ==========================================================================
|
|||
|
|||
using System.Threading.Channels; |
|||
using Squidex.Infrastructure.Tasks; |
|||
|
|||
#pragma warning disable RECS0082 // Parameter has the same name as a member and hides it
|
|||
#pragma warning disable SA1313 // Parameter names should begin with lower-case letter
|
|||
|
|||
namespace Squidex.Infrastructure.EventSourcing.Consume |
|||
{ |
|||
internal sealed class BatchSubscriber : IEventSubscriber, IEventSubscription |
|||
{ |
|||
private readonly IEventSubscription eventSubscription; |
|||
private readonly Channel<object> taskQueue; |
|||
private readonly Channel<EventSource> parseQueue; |
|||
private readonly Task handleTask; |
|||
private readonly CancellationTokenSource completed = new CancellationTokenSource(); |
|||
|
|||
private sealed record EventSource(StoredEvent StoredEvent); |
|||
private sealed record BatchItem(Envelope<IEvent>? Event, string Position); |
|||
private sealed record BatchJob(BatchItem[] Items); |
|||
private sealed record ErrorJob(Exception Exception); |
|||
|
|||
public BatchSubscriber( |
|||
EventConsumerProcessor processor, |
|||
IEventFormatter eventFormatter, |
|||
IEventConsumer eventConsumer, |
|||
Func<IEventSubscriber, IEventSubscription> factory) |
|||
{ |
|||
eventSubscription = factory(this); |
|||
|
|||
var batchSize = Math.Max(1, eventConsumer.BatchSize); |
|||
var batchDelay = Math.Max(100, eventConsumer.BatchDelay); |
|||
|
|||
parseQueue = Channel.CreateBounded<EventSource>(new BoundedChannelOptions(batchSize) |
|||
{ |
|||
AllowSynchronousContinuations = true, |
|||
SingleReader = true, |
|||
SingleWriter = true |
|||
}); |
|||
|
|||
taskQueue = Channel.CreateBounded<object>(new BoundedChannelOptions(2) |
|||
{ |
|||
SingleReader = true, |
|||
SingleWriter = true |
|||
}); |
|||
|
|||
var batchQueue = Channel.CreateBounded<object>(new BoundedChannelOptions(batchSize) |
|||
{ |
|||
AllowSynchronousContinuations = true, |
|||
SingleReader = true, |
|||
SingleWriter = true |
|||
}); |
|||
|
|||
#pragma warning disable MA0040 // Flow the cancellation token
|
|||
batchQueue.Batch<BatchItem, object>(taskQueue, x => new BatchJob(x.ToArray()), batchSize, batchDelay); |
|||
|
|||
Task.Run(async () => |
|||
{ |
|||
await foreach (var source in parseQueue.Reader.ReadAllAsync(completed.Token)) |
|||
{ |
|||
var storedEvent = source.StoredEvent; |
|||
try |
|||
{ |
|||
Envelope<IEvent>? @event = null; |
|||
|
|||
if (eventConsumer.Handles(storedEvent)) |
|||
{ |
|||
@event = eventFormatter.ParseIfKnown(storedEvent); |
|||
} |
|||
|
|||
await batchQueue.Writer.WriteAsync(new BatchItem(@event, storedEvent.EventPosition), completed.Token); |
|||
} |
|||
catch (Exception ex) |
|||
{ |
|||
await taskQueue.Writer.WriteAsync(new ErrorJob(ex), completed.Token); |
|||
} |
|||
} |
|||
}).ContinueWith(x => batchQueue.Writer.TryComplete(x.Exception)); |
|||
#pragma warning restore MA0040 // Flow the cancellation token
|
|||
|
|||
handleTask = Run(processor); |
|||
} |
|||
|
|||
private async Task Run(EventConsumerProcessor processor) |
|||
{ |
|||
try |
|||
{ |
|||
await foreach (var task in taskQueue.Reader.ReadAllAsync(completed.Token)) |
|||
{ |
|||
switch (task) |
|||
{ |
|||
case ErrorJob { Exception: not OperationCanceledException } error: |
|||
{ |
|||
await processor.OnErrorAsync(this, error.Exception); |
|||
break; |
|||
} |
|||
|
|||
case BatchJob batch: |
|||
{ |
|||
var eventsPosition = batch.Items[^1].Position; |
|||
var eventsCollection = batch.Items.Select(x => x.Event).NotNull().ToList(); |
|||
|
|||
await processor.OnEventsAsync(this, eventsCollection, eventsPosition); |
|||
break; |
|||
} |
|||
} |
|||
} |
|||
} |
|||
catch (OperationCanceledException) |
|||
{ |
|||
return; |
|||
} |
|||
catch (Exception ex) |
|||
{ |
|||
await processor.OnErrorAsync(this, ex); |
|||
} |
|||
} |
|||
|
|||
public Task CompleteAsync() |
|||
{ |
|||
parseQueue.Writer.TryComplete(); |
|||
|
|||
return handleTask; |
|||
} |
|||
|
|||
public void Dispose() |
|||
{ |
|||
completed.Cancel(); |
|||
|
|||
eventSubscription.Dispose(); |
|||
} |
|||
|
|||
public void WakeUp() |
|||
{ |
|||
eventSubscription.WakeUp(); |
|||
} |
|||
|
|||
ValueTask IEventSubscriber.OnEventAsync(IEventSubscription subscription, StoredEvent storedEvent) |
|||
{ |
|||
return parseQueue.Writer.WriteAsync(new EventSource(storedEvent), completed.Token); |
|||
} |
|||
|
|||
ValueTask IEventSubscriber.OnErrorAsync(IEventSubscription subscription, Exception exception) |
|||
{ |
|||
return taskQueue.Writer.WriteAsync(new ErrorJob(exception), completed.Token); |
|||
} |
|||
} |
|||
} |
|||
@ -0,0 +1,160 @@ |
|||
// ==========================================================================
|
|||
// Squidex Headless CMS
|
|||
// ==========================================================================
|
|||
// Copyright (c) Squidex UG (haftungsbeschraenkt)
|
|||
// All rights reserved. Licensed under the MIT license.
|
|||
// ==========================================================================
|
|||
|
|||
using System.Threading.Channels; |
|||
using Squidex.Infrastructure.Tasks; |
|||
|
|||
namespace Squidex.Infrastructure.EventSourcing.Consume |
|||
{ |
|||
internal sealed class BatchSubscription : IEventSubscriber<ParsedEvent>, IEventSubscription |
|||
{ |
|||
private readonly IEventSubscription eventSubscription; |
|||
private readonly Channel<object> taskQueue; |
|||
private readonly Channel<object> batchQueue; |
|||
private readonly Task handleTask; |
|||
private readonly CancellationTokenSource completed = new CancellationTokenSource(); |
|||
|
|||
public BatchSubscription( |
|||
IEventConsumer eventConsumer, |
|||
IEventSubscriber<ParsedEvents> eventSubscriber, |
|||
EventSubscriptionSource<ParsedEvent> eventSource) |
|||
{ |
|||
eventSubscription = eventSource(this); |
|||
|
|||
var batchSize = Math.Max(1, eventConsumer.BatchSize); |
|||
var batchDelay = Math.Max(100, eventConsumer.BatchDelay); |
|||
|
|||
taskQueue = Channel.CreateBounded<object>(new BoundedChannelOptions(2) |
|||
{ |
|||
SingleReader = true, |
|||
SingleWriter = true |
|||
}); |
|||
|
|||
batchQueue = Channel.CreateBounded<object>(new BoundedChannelOptions(batchSize) |
|||
{ |
|||
AllowSynchronousContinuations = true, |
|||
SingleReader = true, |
|||
SingleWriter = true |
|||
}); |
|||
|
|||
batchQueue.Batch<ParsedEvent>(taskQueue, batchSize, batchDelay, completed.Token); |
|||
|
|||
handleTask = Run(eventSubscriber); |
|||
} |
|||
|
|||
private async Task Run(IEventSubscriber<ParsedEvents> eventSink) |
|||
{ |
|||
try |
|||
{ |
|||
var isStopped = false; |
|||
|
|||
await foreach (var task in taskQueue.Reader.ReadAllAsync(completed.Token)) |
|||
{ |
|||
switch (task) |
|||
{ |
|||
case Exception exception when exception is not OperationCanceledException: |
|||
{ |
|||
if (!completed.IsCancellationRequested) |
|||
{ |
|||
await eventSink.OnErrorAsync(this, exception); |
|||
} |
|||
|
|||
isStopped = true; |
|||
break; |
|||
} |
|||
|
|||
case List<ParsedEvent> batch: |
|||
{ |
|||
if (!completed.IsCancellationRequested) |
|||
{ |
|||
// Events can be null if the event consumer is not interested in the stored event.
|
|||
var eventList = batch.Select(x => x.Event).NotNull().ToList(); |
|||
var eventPosition = batch[^1].Position; |
|||
|
|||
// Use a struct here to save a few allocations.
|
|||
await eventSink.OnNextAsync(this, new ParsedEvents(eventList, eventPosition)); |
|||
} |
|||
|
|||
break; |
|||
} |
|||
} |
|||
|
|||
if (isStopped) |
|||
{ |
|||
break; |
|||
} |
|||
} |
|||
} |
|||
catch (OperationCanceledException) |
|||
{ |
|||
return; |
|||
} |
|||
catch (Exception ex) |
|||
{ |
|||
if (!completed.IsCancellationRequested) |
|||
{ |
|||
await eventSink.OnErrorAsync(this, ex); |
|||
} |
|||
} |
|||
} |
|||
|
|||
public void Dispose() |
|||
{ |
|||
if (completed.IsCancellationRequested) |
|||
{ |
|||
return; |
|||
} |
|||
|
|||
// It is not necessary to dispose the cancellation token source.
|
|||
completed.Cancel(); |
|||
|
|||
// We do not lock here, it is the responsibility of the source subscription to be thread safe.
|
|||
eventSubscription.Dispose(); |
|||
} |
|||
|
|||
public async ValueTask CompleteAsync() |
|||
{ |
|||
await eventSubscription.CompleteAsync(); |
|||
|
|||
batchQueue.Writer.TryComplete(); |
|||
|
|||
await handleTask; |
|||
} |
|||
|
|||
public void WakeUp() |
|||
{ |
|||
eventSubscription.WakeUp(); |
|||
} |
|||
|
|||
async ValueTask IEventSubscriber<ParsedEvent>.OnErrorAsync(IEventSubscription subscription, Exception exception) |
|||
{ |
|||
try |
|||
{ |
|||
// Forward the exception from one task only, but bypass the batch.
|
|||
await taskQueue.Writer.WriteAsync(exception, completed.Token); |
|||
} |
|||
catch (ChannelClosedException) |
|||
{ |
|||
// This exception is acceptable and happens when an exception has been thrown before.
|
|||
return; |
|||
} |
|||
} |
|||
|
|||
async ValueTask IEventSubscriber<ParsedEvent>.OnNextAsync(IEventSubscription subscription, ParsedEvent @event) |
|||
{ |
|||
try |
|||
{ |
|||
await batchQueue.Writer.WriteAsync(@event, completed.Token); |
|||
} |
|||
catch (ChannelClosedException) |
|||
{ |
|||
// This exception is acceptable and happens when an exception has been thrown before.
|
|||
return; |
|||
} |
|||
} |
|||
} |
|||
} |
|||
@ -0,0 +1,149 @@ |
|||
// ==========================================================================
|
|||
// Squidex Headless CMS
|
|||
// ==========================================================================
|
|||
// Copyright (c) Squidex UG (haftungsbeschraenkt)
|
|||
// All rights reserved. Licensed under the MIT license.
|
|||
// ==========================================================================
|
|||
|
|||
using System.Threading.Channels; |
|||
|
|||
namespace Squidex.Infrastructure.EventSourcing.Consume |
|||
{ |
|||
internal sealed class ParseSubscription : IEventSubscriber<StoredEvent>, IEventSubscription |
|||
{ |
|||
private readonly Channel<object> deserializeQueue; |
|||
private readonly CancellationTokenSource completed = new CancellationTokenSource(); |
|||
private readonly Task deserializeTask; |
|||
private readonly IEventSubscription eventSubscription; |
|||
|
|||
public ParseSubscription( |
|||
IEventConsumer eventConsumer, |
|||
IEventFormatter eventFormatter, |
|||
IEventSubscriber<ParsedEvent> eventSubscriber, |
|||
EventSubscriptionSource<StoredEvent> eventSource) |
|||
{ |
|||
eventSubscription = eventSource(this); |
|||
|
|||
deserializeQueue = Channel.CreateBounded<object>(new BoundedChannelOptions(2) |
|||
{ |
|||
AllowSynchronousContinuations = true, |
|||
SingleReader = true, |
|||
SingleWriter = true |
|||
}); |
|||
|
|||
#pragma warning disable MA0040 // Flow the cancellation token
|
|||
deserializeTask = Task.Run(async () => |
|||
{ |
|||
try |
|||
{ |
|||
var isFailed = false; |
|||
|
|||
await foreach (var input in deserializeQueue.Reader.ReadAllAsync(completed.Token)) |
|||
{ |
|||
switch (input) |
|||
{ |
|||
case Exception exception: |
|||
{ |
|||
// Not very likely that the task is cancelled.
|
|||
await eventSubscriber.OnErrorAsync(this, exception); |
|||
|
|||
isFailed = true; |
|||
break; |
|||
} |
|||
|
|||
case StoredEvent storedEvent: |
|||
{ |
|||
Envelope<IEvent>? @event = null; |
|||
|
|||
if (eventConsumer.Handles(storedEvent)) |
|||
{ |
|||
@event = eventFormatter.ParseIfKnown(storedEvent); |
|||
} |
|||
|
|||
// Parsing takes a little bit of time, so the task might have been cancelled.
|
|||
if (!completed.IsCancellationRequested) |
|||
{ |
|||
// Also invoke the subscriber if the event is null to update the position.
|
|||
await eventSubscriber.OnNextAsync(this, new ParsedEvent(@event, storedEvent.EventPosition)); |
|||
} |
|||
|
|||
break; |
|||
} |
|||
} |
|||
|
|||
if (isFailed) |
|||
{ |
|||
break; |
|||
} |
|||
} |
|||
} |
|||
catch (OperationCanceledException) |
|||
{ |
|||
return; |
|||
} |
|||
catch (Exception ex) |
|||
{ |
|||
if (!completed.IsCancellationRequested) |
|||
{ |
|||
await eventSubscriber.OnErrorAsync(this, ex); |
|||
} |
|||
} |
|||
}).ContinueWith(x => deserializeQueue.Writer.TryComplete(x.Exception)); |
|||
} |
|||
|
|||
public void Dispose() |
|||
{ |
|||
if (completed.IsCancellationRequested) |
|||
{ |
|||
return; |
|||
} |
|||
|
|||
// It is not necessary to dispose the cancellation token source.
|
|||
completed.Cancel(); |
|||
|
|||
// We do not lock here, it is the responsibility of the source subscription to be thread safe.
|
|||
eventSubscription.Dispose(); |
|||
} |
|||
|
|||
public async ValueTask CompleteAsync() |
|||
{ |
|||
await eventSubscription.CompleteAsync(); |
|||
|
|||
deserializeQueue.Writer.TryComplete(); |
|||
|
|||
await deserializeTask; |
|||
} |
|||
|
|||
public void WakeUp() |
|||
{ |
|||
eventSubscription.WakeUp(); |
|||
} |
|||
|
|||
async ValueTask IEventSubscriber<StoredEvent>.OnErrorAsync(IEventSubscription subscription, Exception exception) |
|||
{ |
|||
try |
|||
{ |
|||
// Forward the exception from one task only.
|
|||
await deserializeQueue.Writer.WriteAsync(exception, completed.Token); |
|||
} |
|||
catch (ChannelClosedException) |
|||
{ |
|||
// This exception is acceptable and happens when an exception has been thrown before.
|
|||
return; |
|||
} |
|||
} |
|||
|
|||
async ValueTask IEventSubscriber<StoredEvent>.OnNextAsync(IEventSubscription subscription, StoredEvent @event) |
|||
{ |
|||
try |
|||
{ |
|||
await deserializeQueue.Writer.WriteAsync(@event, completed.Token); |
|||
} |
|||
catch (ChannelClosedException) |
|||
{ |
|||
// This exception is acceptable and happens when an exception has been thrown before.
|
|||
return; |
|||
} |
|||
} |
|||
} |
|||
} |
|||
@ -0,0 +1,16 @@ |
|||
// ==========================================================================
|
|||
// Squidex Headless CMS
|
|||
// ==========================================================================
|
|||
// Copyright (c) Squidex UG (haftungsbeschraenkt)
|
|||
// All rights reserved. Licensed under the MIT license.
|
|||
// ==========================================================================
|
|||
|
|||
#pragma warning disable SA1313 // Parameter names should begin with lower-case letter
|
|||
#pragma warning disable MA0048 // File name must match type name
|
|||
|
|||
namespace Squidex.Infrastructure.EventSourcing.Consume |
|||
{ |
|||
public record struct ParsedEvent(Envelope<IEvent>? Event, string Position); |
|||
|
|||
public record struct ParsedEvents(List<Envelope<IEvent>> Events, string Position); |
|||
} |
|||
Loading…
Reference in new issue