Background job quartz integration retry

framework/src/Volo.Abp.BackgroundJobs.Quartz/Volo/Abp/BackgroundJobs/Quartz/AbpBackgroundJobQuartzOptions.cs

@@ -0,0 +1,48 @@
+using System;
+using System.Threading.Tasks;
+using JetBrains.Annotations;
+using Quartz;
+
+namespace Volo.Abp.BackgroundJobs.Quartz
+{
+    public class AbpBackgroundJobQuartzOptions
+    {
+        public int RetryCount { get; set; }
+
+        public int RetryIntervalMillisecond { get; set; }
+
+        
+        public const string RetryIndex = "RetryIndex";
+        
+        [NotNull]
+        public Func<int, IJobExecutionContext, JobExecutionException,Task> RetryStrategy
+        {
+            get => _retryStrategy;
+            set => _retryStrategy = Check.NotNull(value, nameof(value));
+        }
+        private Func<int, IJobExecutionContext, JobExecutionException,Task> _retryStrategy;
+        
+        public AbpBackgroundJobQuartzOptions()
+        {
+            RetryCount = 3;
+            RetryIntervalMillisecond = 3000;
+            _retryStrategy = DefaultRetryStrategy;
+        }
+        
+        private async Task DefaultRetryStrategy(int retryIndex, IJobExecutionContext executionContext, JobExecutionException exception)
+        {
+            exception.RefireImmediately = true;
+            
+            var retryCount = executionContext.JobDetail.JobDataMap.GetIntValue(nameof(RetryCount));
+            if (retryIndex > retryCount)
+            {
+                exception.RefireImmediately = false;
+                exception.UnscheduleAllTriggers = true;
+                return;
+            }
+            
+            var retryInterval = executionContext.JobDetail.JobDataMap.GetIntValue(nameof(RetryIntervalMillisecond));
+            await Task.Delay(retryInterval);
+        }
+    }
+}

framework/src/Volo.Abp.BackgroundJobs.Quartz/Volo/Abp/BackgroundJobs/Quartz/QuartzBackgroundJobManageExtensions.cs

@@ -0,0 +1,22 @@
+using System;
+using System.Threading.Tasks;
+using Quartz;
+
+namespace Volo.Abp.BackgroundJobs.Quartz
+{
+    public static class QuartzBackgroundJobManageExtensions
+    {
+        public static async Task<string> EnqueueAsync<TArgs>(this IBackgroundJobManager backgroundJobManager,
+            TArgs args, int retryCount, int retryIntervalMillisecond,
+            BackgroundJobPriority priority = BackgroundJobPriority.Normal, TimeSpan? delay = null)
+        {
+            if (backgroundJobManager is QuartzBackgroundJobManager quartzBackgroundJobManager)
+            {
+                return await quartzBackgroundJobManager.ReEnqueueAsync(args, retryCount, retryIntervalMillisecond,
+                    priority, delay);
+            }
+
+            return null;
+        }
+    }
+}

framework/src/Volo.Abp.BackgroundJobs.Quartz/Volo/Abp/BackgroundJobs/Quartz/QuartzBackgroundJobManager.cs

@@ -1,5 +1,6 @@
 using System;
 using System.Threading.Tasks;
+using Microsoft.Extensions.Options;
 using Quartz;
 using Volo.Abp.DependencyInjection;
 
@@ -7,20 +8,38 @@ namespace Volo.Abp.BackgroundJobs.Quartz
 {
     [Dependency(ReplaceServices = true)]
     public class QuartzBackgroundJobManager : IBackgroundJobManager, ITransientDependency
-    {
-        private readonly IScheduler _scheduler;
+    { 
+        protected IScheduler Scheduler { get; }
+        
+        protected AbpBackgroundJobQuartzOptions Options { get; }
 
-        public QuartzBackgroundJobManager(IScheduler scheduler)
+        
+        public QuartzBackgroundJobManager(IScheduler scheduler,IOptions<AbpBackgroundJobQuartzOptions> options)
         {
-            _scheduler = scheduler;
+            Scheduler = scheduler;
+            Options = options.Value;
         }
 
         public virtual async Task<string> EnqueueAsync<TArgs>(TArgs args, BackgroundJobPriority priority = BackgroundJobPriority.Normal,
             TimeSpan? delay = null)
         {
-            var jobDetail = JobBuilder.Create<QuartzJobExecutionAdapter<TArgs>>().SetJobData(new JobDataMap { { nameof(TArgs), args } }).Build();
+            return await ReEnqueueAsync(args, Options.RetryCount, Options.RetryIntervalMillisecond, priority, delay);
+        }
+
+        public virtual async Task<string> ReEnqueueAsync<TArgs>(TArgs args, int retryCount, int retryIntervalMillisecond,
+            BackgroundJobPriority priority = BackgroundJobPriority.Normal, TimeSpan? delay = null)
+        {
+            var jobDataMap = new JobDataMap
+            {
+                {nameof(TArgs), args},
+                {nameof(Options.RetryCount), retryCount},
+                {nameof(Options.RetryIntervalMillisecond), retryIntervalMillisecond},
+                {nameof(AbpBackgroundJobQuartzOptions.RetryIndex), 0}
+            };
+            
+            var jobDetail = JobBuilder.Create<QuartzJobExecutionAdapter<TArgs>>().RequestRecovery().SetJobData(jobDataMap).Build();
             var trigger = !delay.HasValue ? TriggerBuilder.Create().StartNow().Build() : TriggerBuilder.Create().StartAt(new DateTimeOffset(DateTime.Now.Add(delay.Value))).Build();
-            await _scheduler.ScheduleJob(jobDetail, trigger);
+            await Scheduler.ScheduleJob(jobDetail, trigger);
             return jobDetail.Key.ToString();
         }
     }

framework/src/Volo.Abp.BackgroundJobs.Quartz/Volo/Abp/BackgroundJobs/Quartz/QuartzJobExecutionAdapter.cs

@@ -1,4 +1,5 @@
-using System.Threading.Tasks;
+using System;
+using System.Threading.Tasks;
 using Microsoft.Extensions.DependencyInjection;
 using Microsoft.Extensions.Logging;
 using Microsoft.Extensions.Logging.Abstractions;
@@ -10,18 +11,22 @@ namespace Volo.Abp.BackgroundJobs.Quartz
     public class QuartzJobExecutionAdapter<TArgs> : IJob
     {
         public ILogger<QuartzJobExecutionAdapter<TArgs>> Logger { get; set; }
-        
+
         protected AbpBackgroundJobOptions Options { get; }
+        protected AbpBackgroundJobQuartzOptions BackgroundJobQuartzOptions { get; }
         protected IServiceScopeFactory ServiceScopeFactory { get; }
         protected IBackgroundJobExecuter JobExecuter { get; }
+
         public QuartzJobExecutionAdapter(
             IOptions<AbpBackgroundJobOptions> options,
+            IOptions<AbpBackgroundJobQuartzOptions> backgroundJobQuartzOptions,
             IBackgroundJobExecuter jobExecuter,
             IServiceScopeFactory serviceScopeFactory)
         {
             JobExecuter = jobExecuter;
             ServiceScopeFactory = serviceScopeFactory;
             Options = options.Value;
+            BackgroundJobQuartzOptions = backgroundJobQuartzOptions.Value;
             Logger = NullLogger<QuartzJobExecutionAdapter<TArgs>>.Instance;
         }
 
@@ -32,14 +37,29 @@ namespace Volo.Abp.BackgroundJobs.Quartz
                 Logger.LogWarning("Background jobs system is disabled");
                 return;
             }
-            
+
             using (var scope = ServiceScopeFactory.CreateScope())
             {
-                var args = (TArgs)context.JobDetail.JobDataMap.Get(nameof(TArgs));
+                var args = (TArgs) context.JobDetail.JobDataMap.Get(nameof(TArgs));
                 var jobType = Options.GetJob(typeof(TArgs)).JobType;
                 var jobContext = new JobExecutionContext(scope.ServiceProvider, jobType, args);
-                await JobExecuter.ExecuteAsync(jobContext);
+                try
+                {
+                    await JobExecuter.ExecuteAsync(jobContext);
+                }
+                catch (Exception exception)
+                {
+                    var jobExecutionException = new JobExecutionException(exception);
+                    
+                    var retryIndex = context.JobDetail.JobDataMap.GetIntValue(nameof(AbpBackgroundJobQuartzOptions.RetryIndex));
+                    retryIndex++;
+                    context.JobDetail.JobDataMap.Put(AbpBackgroundJobQuartzOptions.RetryIndex, retryIndex);
+                    
+                    await BackgroundJobQuartzOptions.RetryStrategy.Invoke(retryIndex, context, jobExecutionException);
+                    
+                    throw jobExecutionException;
+                }
             }
         }
     }
-}
+}