Catch operation cancelled exception in TPL dataflow.

backend/src/Migrations/Migrations/MongoDb/AddAppIdToEventStream.cs

@@ -45,6 +45,8 @@ namespace Migrations.Migrations.MongoDb
             };
 
             var actionBlock = new ActionBlock<BsonDocument[]>(async batch =>
+            {
+                try
                 {
                     var writes = new List<WriteModel<BsonDocument>>();
 
@@ -102,6 +104,12 @@ namespace Migrations.Migrations.MongoDb
                     {
                         await collectionNew.BulkWriteAsync(writes, writeOptions);
                     }
+                }
+                catch (OperationCanceledException ex)
+                {
+                    // Dataflow swallows operation cancelled exception.
+                    throw new AggregateException(ex);
+                }
             }, new ExecutionDataflowBlockOptions
             {
                 MaxDegreeOfParallelism = Environment.ProcessorCount * 2,

backend/src/Migrations/Migrations/MongoDb/ConvertDocumentIds.cs

@@ -101,6 +101,8 @@ namespace Migrations.Migrations.MongoDb
             };
 
             var actionBlock = new ActionBlock<BsonDocument[]>(async batch =>
+            {
+                try
                 {
                     var writes = new List<WriteModel<BsonDocument>>();
 
@@ -136,6 +138,12 @@ namespace Migrations.Migrations.MongoDb
                     {
                         await collectionNew.BulkWriteAsync(writes, writeOptions);
                     }
+                }
+                catch (OperationCanceledException ex)
+                {
+                    // Dataflow swallows operation cancelled exception.
+                    throw new AggregateException(ex);
+                }
             }, new ExecutionDataflowBlockOptions
             {
                 MaxDegreeOfParallelism = Environment.ProcessorCount * 2,

backend/src/Squidex.Domain.Apps.Entities/Assets/DomainObject/AssetsBulkUpdateCommandMiddleware.cs

@@ -58,16 +58,32 @@ namespace Squidex.Domain.Apps.Entities.Assets.DomainObject
                     };
 
                     var createCommandsBlock = new TransformBlock<BulkTask, BulkTaskCommand?>(task =>
+                    {
+                        try
                         {
                             return CreateCommand(task);
+                        }
+                        catch (OperationCanceledException ex)
+                        {
+                            // Dataflow swallows operation cancelled exception.
+                            throw new AggregateException(ex);
+                        }
                     }, executionOptions);
 
                     var executeCommandBlock = new ActionBlock<BulkTaskCommand?>(async command =>
+                    {
+                        try
                         {
                             if (command != null)
                             {
                                 await ExecuteCommandAsync(command);
                             }
+                        }
+                        catch (OperationCanceledException ex)
+                        {
+                            // Dataflow swallows operation cancelled exception.
+                            throw new AggregateException(ex);
+                        }
                     }, executionOptions);
 
                     createCommandsBlock.LinkTo(executeCommandBlock, new DataflowLinkOptions
@@ -115,23 +131,16 @@ namespace Squidex.Domain.Apps.Entities.Assets.DomainObject
         private static async Task ExecuteCommandAsync(BulkTaskCommand bulkCommand)
         {
             var (task, id, command) = bulkCommand;
-
-            Exception? exception = null;
             try
             {
                 await task.Bus.PublishAsync(command);
+
+                task.Results.Add(new BulkUpdateResultItem(id, task.JobIndex));
             }
             catch (Exception ex)
             {
-                exception = ex;
+                task.Results.Add(new BulkUpdateResultItem(id, task.JobIndex, ex));
             }
-
-            task.Results.Add(new BulkUpdateResultItem
-            {
-                Id = id,
-                JobIndex = task.JobIndex,
-                Exception = exception
-            });
         }
 
         private BulkTaskCommand? CreateCommand(BulkTask task)
@@ -148,13 +157,7 @@ namespace Squidex.Domain.Apps.Entities.Assets.DomainObject
             }
             catch (Exception ex)
             {
-                task.Results.Add(new BulkUpdateResultItem
-                {
-                    Id = id,
-                    JobIndex = task.JobIndex,
-                    Exception = ex
-                });
-
+                task.Results.Add(new BulkUpdateResultItem(id, task.JobIndex, ex));
                 return null;
             }
         }

backend/src/Squidex.Domain.Apps.Entities/Backup/RestoreGrain.cs

@@ -324,6 +324,8 @@ namespace Squidex.Domain.Apps.Entities.Backup
             var handled = 0;
 
             var writeBlock = new ActionBlock<(string, Envelope<IEvent>)[]>(async batch =>
+            {
+                try
                 {
                     var commits = new List<EventCommit>(batch.Length);
 
@@ -339,6 +341,12 @@ namespace Squidex.Domain.Apps.Entities.Backup
                     handled += commits.Count;
 
                     Log($"Reading {reader.ReadEvents}/{handled} events and {reader.ReadAttachments} attachments completed.", true);
+                }
+                catch (OperationCanceledException ex)
+                {
+                    // Dataflow swallows operation cancelled exception.
+                    throw new AggregateException(ex);
+                }
             }, new ExecutionDataflowBlockOptions
             {
                 MaxDegreeOfParallelism = 1,

backend/src/Squidex.Domain.Apps.Entities/BulkUpdateResultItem.cs

@@ -8,14 +8,9 @@
 using System;
 using Squidex.Infrastructure;
 
+#pragma warning disable SA1313 // Parameter names should begin with lower-case letter
+
 namespace Squidex.Domain.Apps.Entities
 {
-    public sealed class BulkUpdateResultItem
-    {
-        public DomainId? Id { get; set; }
-
-        public int JobIndex { get; set; }
-
-        public Exception? Exception { get; set; }
-    }
+    public sealed record BulkUpdateResultItem(DomainId? Id, int JobIndex, Exception? Exception = null);
 }

backend/src/Squidex.Domain.Apps.Entities/Contents/DomainObject/ContentsBulkUpdateCommandMiddleware.cs

@@ -66,13 +66,29 @@ namespace Squidex.Domain.Apps.Entities.Contents.DomainObject
                     };
 
                     var createCommandsBlock = new TransformManyBlock<BulkTask, BulkTaskCommand>(async task =>
+                    {
+                        try
                         {
                             return await CreateCommandsAsync(task);
+                        }
+                        catch (OperationCanceledException ex)
+                        {
+                            // Dataflow swallows operation cancelled exception.
+                            throw new AggregateException(ex);
+                        }
                 }, executionOptions);
 
                     var executeCommandBlock = new ActionBlock<BulkTaskCommand>(async command =>
+                    {
+                        try
                         {
                             await ExecuteCommandAsync(command);
+                        }
+                        catch (OperationCanceledException ex)
+                        {
+                            // Dataflow swallows operation cancelled exception.
+                            throw new AggregateException(ex);
+                        }
                     }, executionOptions);
 
                     createCommandsBlock.LinkTo(executeCommandBlock, new DataflowLinkOptions
@@ -124,22 +140,16 @@ namespace Squidex.Domain.Apps.Entities.Contents.DomainObject
         {
             var (task, id, command) = bulkCommand;
 
-            Exception? exception = null;
             try
             {
                 await task.Bus.PublishAsync(command);
+
+                task.Results.Add(new BulkUpdateResultItem(id, task.JobIndex));
             }
             catch (Exception ex)
             {
-                exception = ex;
+                task.Results.Add(new BulkUpdateResultItem(id, task.JobIndex, ex));
             }
-
-            task.Results.Add(new BulkUpdateResultItem
-            {
-                Id = id,
-                JobIndex = task.JobIndex,
-                Exception = exception
-            });
         }
 
         private async Task<IEnumerable<BulkTaskCommand>> CreateCommandsAsync(BulkTask task)
@@ -167,22 +177,13 @@ namespace Squidex.Domain.Apps.Entities.Contents.DomainObject
                     }
                     catch (Exception ex)
                     {
-                        task.Results.Add(new BulkUpdateResultItem
-                        {
-                            Id = id,
-                            JobIndex = task.JobIndex,
-                            Exception = ex
-                        });
+                        task.Results.Add(new BulkUpdateResultItem(id, task.JobIndex, ex));
                     }
                 }
             }
             catch (Exception ex)
             {
-                task.Results.Add(new BulkUpdateResultItem
-                {
-                    JobIndex = task.JobIndex,
-                    Exception = ex
-                });
+                task.Results.Add(new BulkUpdateResultItem(null, task.JobIndex, ex));
             }
 
             return commands;

backend/src/Squidex.Infrastructure.MongoDb/MongoDb/MongoExtensions.cs

@@ -189,11 +189,19 @@ namespace Squidex.Infrastructure.MongoDb
                 {
                     var actionBlock =
                         new ActionBlock<T>(async x =>
+                            {
+                                try
                                 {
                                     if (!combined.IsCancellationRequested)
                                     {
                                         await processor(x);
                                     }
+                                }
+                                catch (OperationCanceledException ex)
+                                {
+                                    // Dataflow swallows operation cancelled exception.
+                                    throw new AggregateException(ex);
+                                }
                             },
                             new ExecutionDataflowBlockOptions
                             {

backend/src/Squidex.Infrastructure/Commands/Rebuilder.cs

@@ -90,6 +90,8 @@ namespace Squidex.Infrastructure.Commands
             const int BatchSize = 100;
 
             var workerBlock = new ActionBlock<DomainId[]>(async ids =>
+            {
+                try
                 {
                     await using (var context = store.WithBatchContext(typeof(T)))
                     {
@@ -111,6 +113,12 @@ namespace Squidex.Infrastructure.Commands
                             }
                         }
                     }
+                }
+                catch (OperationCanceledException ex)
+                {
+                    // Dataflow swallows operation cancelled exception.
+                    throw new AggregateException(ex);
+                }
             },
             new ExecutionDataflowBlockOptions
             {

backend/src/Squidex.Infrastructure/EventSourcing/Grains/BatchSubscriber.cs

@@ -49,6 +49,8 @@ namespace Squidex.Infrastructure.EventSourcing.Grains
             var batchDelay = Math.Max(100, eventConsumer.BatchDelay);
 
             var parse = new TransformBlock<Job, Job>(job =>
+            {
+                try
                 {
                     if (job.StoredEvent != null)
                     {
@@ -68,6 +70,12 @@ namespace Squidex.Infrastructure.EventSourcing.Grains
                     }
 
                     return job;
+                }
+                catch (OperationCanceledException ex)
+                {
+                    // Dataflow swallows operation cancelled exception.
+                    throw new AggregateException(ex);
+                }
             }, new ExecutionDataflowBlockOptions
             {
                 MaxDegreeOfParallelism = 1,
@@ -81,6 +89,8 @@ namespace Squidex.Infrastructure.EventSourcing.Grains
             });
 
             var handle = new ActionBlock<IList<Job>>(async jobs =>
+            {
+                try
                 {
                     var sender = eventSubscription?.Sender;
 
@@ -100,6 +110,12 @@ namespace Squidex.Infrastructure.EventSourcing.Grains
                             }
                         }
                     }
+                }
+                catch (OperationCanceledException ex)
+                {
+                    // Dataflow swallows operation cancelled exception.
+                    throw new AggregateException(ex);
+                }
             },
             new ExecutionDataflowBlockOptions
             {

backend/src/Squidex.Infrastructure/Tasks/PartitionedActionBlock.cs

@@ -44,7 +44,18 @@ namespace Squidex.Infrastructure.Tasks
                 workerOption.MaxDegreeOfParallelism = 1;
                 workerOption.MaxMessagesPerTask = 1;
 
-                workers[i] = new ActionBlock<TInput>(action, workerOption);
+                workers[i] = new ActionBlock<TInput>(async input =>
+                {
+                    try
+                    {
+                        await action(input);
+                    }
+                    catch (OperationCanceledException ex)
+                    {
+                        // Dataflow swallows operation cancelled exception.
+                        throw new AggregateException(ex);
+                    }
+                }, workerOption);
             }
 
             var distributorOption = new ExecutionDataflowBlockOptions
@@ -54,11 +65,19 @@ namespace Squidex.Infrastructure.Tasks
                 BoundedCapacity = 1
             };
 
-            distributor = new ActionBlock<TInput>(x =>
+            distributor = new ActionBlock<TInput>(async input =>
             {
-                var partition = Math.Abs(partitioner(x)) % workers.Length;
+                try
+                {
+                    var partition = Math.Abs(partitioner(input)) % workers.Length;
 
-                return workers[partition].SendAsync(x);
+                    await workers[partition].SendAsync(input);
+                }
+                catch (OperationCanceledException ex)
+                {
+                    // Dataflow swallows operation cancelled exception.
+                    throw new AggregateException(ex);
+                }
         }, distributorOption);
 
             distributor.Completion.ContinueWith(x =>