From 2f98f7fea08f04856a3e15fd0e267e596c805597 Mon Sep 17 00:00:00 2001 From: maliming Date: Thu, 4 Sep 2025 10:07:29 +0800 Subject: [PATCH] Add configurable retry backoff for inbox processor Introduces the InboxProcessorRetryBackoffFactor option to allow configuration of the initial retry delay for the inbox processor when using the RetryLater failure policy. Updates documentation and logic to use this factor for exponential backoff, providing more flexibility in handling event retries. --- .../infrastructure/event-bus/distributed/index.md | 4 ++-- .../Abp/EventBus/Distributed/AbpEventBusBoxesOptions.cs | 7 +++++++ .../Volo/Abp/EventBus/Distributed/InboxProcessor.cs | 8 ++++---- .../EventBus/Distributed/InboxProcessorFailurePolicy.cs | 7 ++++--- 4 files changed, 17 insertions(+), 9 deletions(-) diff --git a/docs/en/framework/infrastructure/event-bus/distributed/index.md b/docs/en/framework/infrastructure/event-bus/distributed/index.md index bef921c851..f379a44f40 100644 --- a/docs/en/framework/infrastructure/event-bus/distributed/index.md +++ b/docs/en/framework/infrastructure/event-bus/distributed/index.md @@ -645,9 +645,9 @@ Configure(options => * `DistributedLockWaitDuration`: ABP uses [distributed locking](../../distributed-locking.md) to prevent concurrent access to the inbox and outbox messages in the database, when running multiple instance of the same application. If an instance of the application can not obtain the lock, it tries after a duration. This is the configuration of that duration. Default value is 15 seconds (`TimeSpan.FromSeconds(15)`). * `InboxProcessorFailurePolicy`: The policy to handle the failure of the inbox processor. Default value is `Retry`. Possible values are: * `Retry`: The current exception and subsequent events will continue to be processed in order in the next cycle. - * `RetryLater`: Skip the event that caused the exception and continue with the following events. The failed event will be retried after a delay that doubles each time (1, 2, 4, 8, 16 seconds). The default maximum retry count is 10 (configurable). + * `RetryLater`: Skip the event that caused the exception and continue with the following events. The failed event will be retried after a delay that doubles with each retry, starting from the configured `InboxProcessorRetryBackoffFactor` (e.g., 10, 20, 40, 80 seconds). The default maximum retry count is 10 (configurable). Discard the event if it still fails after reaching the maximum retry count. * `Discard`: The event that caused the exception will be discarded and will not be retried. -* `InboxProcessorMaxRetryCount`: The maximum number of retries for the inbox processor. Default value is 10. Only used when `InboxProcessorFailurePolicy` is `RetryLater`. +* `InboxProcessorRetryBackoffFactor`: The initial retry delay factor (double) used when `InboxProcessorFailurePolicy` is `RetryLater`. The retry delay is calculated as: `delay = InboxProcessorRetryBackoffFactor × 2^retryCount`. Default value is `10`. ### Skipping Outbox diff --git a/framework/src/Volo.Abp.EventBus/Volo/Abp/EventBus/Distributed/AbpEventBusBoxesOptions.cs b/framework/src/Volo.Abp.EventBus/Volo/Abp/EventBus/Distributed/AbpEventBusBoxesOptions.cs index ec7b603176..6febf2f065 100644 --- a/framework/src/Volo.Abp.EventBus/Volo/Abp/EventBus/Distributed/AbpEventBusBoxesOptions.cs +++ b/framework/src/Volo.Abp.EventBus/Volo/Abp/EventBus/Distributed/AbpEventBusBoxesOptions.cs @@ -46,6 +46,13 @@ public class AbpEventBusBoxesOptions /// public int InboxProcessorMaxRetryCount { get; set; } = 10; + /// + /// Default value is 10 + /// The initial retry delay factor (double) when `InboxProcessorFailurePolicy` is `RetryLater`. + /// The delay is calculated as: `delay = InboxProcessorRetryBackoffFactor × 2^retryCount` + /// + public double InboxProcessorRetryBackoffFactor { get; set; } = 10; + /// /// Default: 15 seconds /// diff --git a/framework/src/Volo.Abp.EventBus/Volo/Abp/EventBus/Distributed/InboxProcessor.cs b/framework/src/Volo.Abp.EventBus/Volo/Abp/EventBus/Distributed/InboxProcessor.cs index 10251ffedb..b00ba81242 100644 --- a/framework/src/Volo.Abp.EventBus/Volo/Abp/EventBus/Distributed/InboxProcessor.cs +++ b/framework/src/Volo.Abp.EventBus/Volo/Abp/EventBus/Distributed/InboxProcessor.cs @@ -148,14 +148,14 @@ public class InboxProcessor : IInboxProcessor, ITransientDependency continue; } - waitingEvent.NextRetryTime = GetNextRetryTime(waitingEvent.RetryCount); + waitingEvent.NextRetryTime = GetNextRetryTime(waitingEvent.RetryCount, EventBusBoxesOptions.InboxProcessorRetryBackoffFactor); Logger.LogInformation($"Event with id = {waitingEvent.Id:N} will retry later. " + $"Current retry count: {waitingEvent.RetryCount}, " + $"Next retry time: {waitingEvent.NextRetryTime}, " + $"Max retry count: {EventBusBoxesOptions.InboxProcessorMaxRetryCount}."); - await Inbox.RetryLaterAsync(waitingEvent.Id, waitingEvent.RetryCount, GetNextRetryTime(waitingEvent.RetryCount)); + await Inbox.RetryLaterAsync(waitingEvent.Id, waitingEvent.RetryCount, GetNextRetryTime(waitingEvent.RetryCount, EventBusBoxesOptions.InboxProcessorRetryBackoffFactor)); await uow.CompleteAsync(StoppingToken); } continue; @@ -188,9 +188,9 @@ public class InboxProcessor : IInboxProcessor, ITransientDependency } } - protected virtual DateTime? GetNextRetryTime(int retryCount) + protected virtual DateTime? GetNextRetryTime(int retryCount, double factor) { - var delaySeconds = (int)Math.Pow(2, retryCount); + var delaySeconds = factor * Math.Pow(2, retryCount); return DateTime.Now.AddSeconds(delaySeconds); } diff --git a/framework/src/Volo.Abp.EventBus/Volo/Abp/EventBus/Distributed/InboxProcessorFailurePolicy.cs b/framework/src/Volo.Abp.EventBus/Volo/Abp/EventBus/Distributed/InboxProcessorFailurePolicy.cs index 371fb00548..31095b62a3 100644 --- a/framework/src/Volo.Abp.EventBus/Volo/Abp/EventBus/Distributed/InboxProcessorFailurePolicy.cs +++ b/framework/src/Volo.Abp.EventBus/Volo/Abp/EventBus/Distributed/InboxProcessorFailurePolicy.cs @@ -8,9 +8,10 @@ public enum InboxProcessorFailurePolicy Retry, /// - /// Skip and retry the event in next period time, but with a delay. - /// The delay increases in every fail, and it is discarded after a specified amount of time - /// (e.g. 1 second, 2 seconds, 4 seconds, 8 seconds, etc.), + /// Skip the failed event and retry it after a delay. + /// The delay doubles with each retry, starting from the configured InboxProcessorRetryBackoffFactor + /// (e.g., 10, 20, 40, 80 seconds, etc.). + /// The event is discarded if it still fails after reaching the maximum retry count. /// RetryLater,