fix(tasks): fix repair exception retry

apps/vue/src/api/task-management/model/backgroundJobInfoModel.ts

@@ -4,6 +4,7 @@ export enum JobStatus {
   None = -1,
   Completed = 0,
   Running = 10,
+  FailedRetry = 15,
   Paused = 20,
   Stopped = 30,
 }

apps/vue/src/views/task-management/background-jobs/components/BackgroundJobInfoModal.vue

@@ -4,6 +4,7 @@
     :width="800"
     :height="400"
     :title="modalTitle"
+    :help-message="modalTips"
     :mask-closable="false"
     @ok="handleSubmit"
   >
@@ -27,7 +28,7 @@
             <Input :disabled="isEditModal" v-model:value="modelRef.name" autocomplete="off" />
           </FormItem>
           <FormItem name="type" required :label="L('DisplayName:Type')" :extra="L('Description:Type')">
-            <Textarea :readonly="isEditModal" v-model:value="modelRef.type" :auto-size="{ minRows: 3, maxRows: 6 }" />
+            <Textarea :disabled="isEditModal" v-model:value="modelRef.type" :auto-size="{ minRows: 3, maxRows: 6 }" />
           </FormItem>
           <FormItem name="beginTime" :label="L('DisplayName:BeginTime')">
             <DatePicker style="width: 100%;" v-model:value="modelRef.beginTime" />
@@ -204,7 +205,19 @@
     return false;
   });
   const modalTitle = computed(() => {
-    return isEditModal.value ? L('BackgroundJobs:Edit') : L('BackgroundJobs:AddNew');
+    if (!isEditModal.value) {
+      return L('BackgroundJobs:AddNew');
+    }
+    if (modelRef.value.isAbandoned) {
+      return `${L('BackgroundJobs:Edit')} - ${L('DisplayName:IsAbandoned')}`;
+    }
+    return L('BackgroundJobs:Edit');
+  });
+  const modalTips = computed(() => {
+    if (modelRef.value.isAbandoned) {
+      return L('Description:IsAbandoned');
+    }
+    return '';
   });
   const modelRules = reactive({
     group: ruleCreator.fieldRequired({

apps/vue/src/views/task-management/background-jobs/components/BackgroundJobInfoTable.vue

@@ -13,7 +13,10 @@
         <Switch :checked="record.isEnabled" disabled />
       </template>
       <template #status="{ record }">
-        <Tag :color="JobStatusColor[record.status]">{{ JobStatusMap[record.status] }}</Tag>
+        <Tooltip v-if="record.isAbandoned" color="orange" :title="L('Description:IsAbandoned')">
+          <Tag :color="JobStatusColor[record.status]">{{ JobStatusMap[record.status] }}</Tag>
+        </Tooltip>
+        <Tag v-else :color="JobStatusColor[record.status]">{{ JobStatusMap[record.status] }}</Tag>
       </template>
       <template #type="{ record }">
         <Tag color="blue">{{ JobTypeMap[record.jobType] }}</Tag>
@@ -43,7 +46,7 @@
             {
               auth: 'TaskManagement.BackgroundJobs.Pause',
               label: L('BackgroundJobs:Pause'),
-              ifShow: record.status === JobStatus.Running,
+              ifShow: [JobStatus.Running, JobStatus.FailedRetry].includes(record.status),
               onClick: handlePause.bind(null, record),
             },
             {
@@ -55,13 +58,13 @@
             {
               auth: 'TaskManagement.BackgroundJobs.Trigger',
               label: L('BackgroundJobs:Trigger'),
-              ifShow: [JobStatus.Running, JobStatus.Completed].includes(record.status),
+              ifShow: [JobStatus.Running, JobStatus.Completed, JobStatus.FailedRetry].includes(record.status),
               onClick: handleTrigger.bind(null, record),
             },
             {
               auth: 'TaskManagement.BackgroundJobs.Stop',
               label: L('BackgroundJobs:Stop'),
-              ifShow: record.status === JobStatus.Running,
+              ifShow: [JobStatus.Running, JobStatus.FailedRetry].includes(record.status),
               onClick: handleStop.bind(null, record),
             },
           ]"
@@ -73,7 +76,7 @@
 </template>
 
 <script lang="ts" setup>
-  import { Switch, Modal, Tag, message } from 'ant-design-vue';
+  import { Switch, Modal, Tag, Tooltip, message } from 'ant-design-vue';
   import { useLocalization } from '/@/hooks/abp/useLocalization';
   import { usePermission } from '/@/hooks/web/usePermission';
   import { useModal } from '/@/components/Modal';

apps/vue/src/views/task-management/background-jobs/datas/typing.ts

@@ -7,6 +7,7 @@ export const JobStatusMap = {
   [JobStatus.None]: L('DisplayName:None'),
   [JobStatus.Completed]: L('DisplayName:Completed'),
   [JobStatus.Running]: L('DisplayName:Running'),
+  [JobStatus.FailedRetry]: L('DisplayName:FailedRetry'),
   [JobStatus.Paused]: L('DisplayName:Paused'),
   [JobStatus.Stopped]: L('DisplayName:Stopped'),
 }
@@ -14,8 +15,9 @@ export const JobStatusColor = {
   [JobStatus.None]: '',
   [JobStatus.Completed]: '#339933',
   [JobStatus.Running]: '#3399CC',
+  [JobStatus.FailedRetry]: '#FF6600',
   [JobStatus.Paused]: '#CC6633',
-  [JobStatus.Stopped]: '#FF6600',
+  [JobStatus.Stopped]: '#F00000',
 }
 
 export const JobTypeMap = {

aspnet-core/modules/task-management/LINGYUN.Abp.BackgroundTasks.Abstractions/LINGYUN/Abp/BackgroundTasks/JobStatus.cs

@@ -15,6 +15,10 @@ public enum JobStatus
     /// </summary>
     Running = 10,
     /// <summary>
+    /// 失败重试
+    /// </summary>
+    FailedRetry = 15,
+    /// <summary>
     /// 已暂停
     /// </summary>
     Paused = 20,

aspnet-core/modules/task-management/LINGYUN.Abp.BackgroundTasks.Quartz/LINGYUN/Abp/BackgroundTasks/Quartz/QuartzJobExecutorProvider.cs

@@ -90,6 +90,10 @@ public class QuartzJobExecutorProvider : IQuartzJobExecutorProvider, ISingletonD
                 {
                     maxCount = 0;
                 }
+                if (job.Status == JobStatus.FailedRetry && job.TryCount < job.MaxTryCount)
+                {
+                    maxCount = job.MaxTryCount <= 0 ? -1 : job.MaxTryCount - 1;
+                }
                 triggerBuilder
                     .WithIdentity(job.Name, job.Group)
                     .WithDescription(job.Description)

aspnet-core/modules/task-management/LINGYUN.Abp.BackgroundTasks/LINGYUN/Abp/BackgroundTasks/Internal/BackgroundPollingJob.cs

@@ -15,7 +15,7 @@ internal class BackgroundPollingJob : IJobRunnable
         var store = context.ServiceProvider.GetRequiredService<IJobStore>();
 
         // TODO: 如果积压有大量持续性任务, 可能后面的队列无法被检索到
-        // 通过 NextRunTime 属性过滤，已入队任务重启应用后将无法再次被检索到
+        // 尽量让任务重复次数在可控范围内
         // 需要借助队列提供者来持久化已入队任务
         var waitingJobs = await store.GetWaitingListAsync(options.MaxJobFetchCount);
 

aspnet-core/modules/task-management/LINGYUN.Abp.BackgroundTasks/LINGYUN/Abp/BackgroundTasks/Internal/JobExecutedEvent.cs

@@ -14,8 +14,8 @@ public class JobExecutedEvent : JobEventBase<JobExecutedEvent>, ITransientDepend
         if (job != null)
         {
             job.TriggerCount += 1;
-            job.NextRunTime = context.EventData.NextRunTime;
             job.LastRunTime = context.EventData.RunTime;
+            job.NextRunTime = context.EventData.NextRunTime;
             job.Result = context.EventData.Result ?? "OK";
             job.Status = JobStatus.Running;
 
@@ -23,15 +23,15 @@ public class JobExecutedEvent : JobEventBase<JobExecutedEvent>, ITransientDepend
             if (job.JobType == JobType.Once)
             {
                 job.Status = JobStatus.Completed;
-                job.NextRunTime = null;
             }
 
             // 任务异常后可重试
             if (context.EventData.Exception != null)
             {
                 job.TryCount += 1;
+                job.IsAbandoned = false;
                 // 将任务标记为运行中, 会被轮询重新进入队列
-                job.Status = JobStatus.Running;
+                job.Status = JobStatus.FailedRetry;
                 job.Result = context.EventData.Exception.Message;
 
                 // 多次异常后需要重新计算优先级
@@ -54,14 +54,16 @@ public class JobExecutedEvent : JobEventBase<JobExecutedEvent>, ITransientDepend
                     await RemoveJobAsync(context, job);
                 }
             }
-
-            // 所有任务达到上限则标记已完成
-            if (job.MaxCount > 0 && job.TriggerCount >= job.MaxCount)
+            else
             {
-                job.Status = JobStatus.Completed;
-                job.NextRunTime = null;
+                // 所有任务达到上限则标记已完成
+                if (job.MaxCount > 0 && job.TriggerCount >= job.MaxCount)
+                {
+                    job.Status = JobStatus.Completed;
+                    job.NextRunTime = null;
 
-                await RemoveJobAsync(context, job);
+                    await RemoveJobAsync(context, job);
+                }
             }
 
             await store.StoreAsync(job);

aspnet-core/modules/task-management/LINGYUN.Abp.TaskManagement.Application/LINGYUN/Abp/TaskManagement/BackgroundJobInfoAppService.cs

@@ -157,10 +157,7 @@ public class BackgroundJobInfoAppService : TaskManagementApplicationService, IBa
         backgroundJobInfo.MaxTryCount = input.MaxTryCount;
 
         backgroundJobInfo.Args.RemoveAll(x => !input.Args.ContainsKey(x.Key));
-        foreach (var arg in input.Args)
-        {
-            backgroundJobInfo.SetProperty(arg.Key, arg.Value);
-        }
+        backgroundJobInfo.Args.AddIfNotContains(input.Args);
 
         backgroundJobInfo.SetPriority(input.Priority);
         switch (input.JobType)

aspnet-core/modules/task-management/LINGYUN.Abp.TaskManagement.Domain.Shared/LINGYUN/Abp/TaskManagement/Localization/Resources/en.json

@@ -35,7 +35,8 @@
     "Description:LockTimeOut": "If the job is not allowed to be fired at the same time, it will work, allowing other jobs to proceed after a certain number of seconds",
     "DisplayName:BeginCreationTime": "From CreationTime",
     "DisplayName:EndCreationTime": "To CreationTime",
-    "DisplayName:IsAbandoned": "Is Abandoned",
+    "DisplayName:IsAbandoned": "Abandoned Job",
+    "Description:IsAbandoned": "If a job fails for several times and reaches the maximum number of retries, the job is marked as no longer being executed. You can manually add the job to the queue by changing the maximum number of retries.",
     "DisplayName:CreationTime": "CreationTime",
     "DisplayName:Result": "Last Execution Result",
     "DisplayName:LastRunTime": "LastRunTime",
@@ -54,6 +55,7 @@
     "DisplayName:Completed": "Completed",
     "DisplayName:Running": "Running",
     "DisplayName:Paused": "Paused",
+    "DisplayName:FailedRetry": "Failed Retry",
     "DisplayName:Stopped": "Stopped",
     "DisplayName:Once": "Once",
     "DisplayName:Period": "Period",

aspnet-core/modules/task-management/LINGYUN.Abp.TaskManagement.Domain.Shared/LINGYUN/Abp/TaskManagement/Localization/Resources/zh-Hans.json

@@ -35,7 +35,8 @@
     "Description:LockTimeOut": "如果不允许作业被同时触发,它将会起作用,在一定数值秒之后允许其他作业进行",
     "DisplayName:BeginCreationTime": "起始创建时间",
     "DisplayName:EndCreationTime": "截止创建时间",
-    "DisplayName:IsAbandoned": "是否放弃",
+    "DisplayName:IsAbandoned": "已放弃的作业",
+    "Description:IsAbandoned": "当作业执行中多次失败且达到最大重试次数时,将被标记为不再执行,可以通过修改最大重试次数手动触发再次加入到队列.",
     "DisplayName:CreationTime": "创建时间",
     "DisplayName:Result": "上次执行结果",
     "DisplayName:LastRunTime": "上次执行时间",
@@ -54,6 +55,7 @@
     "DisplayName:Completed": "已完成",
     "DisplayName:Running": "运行中",
     "DisplayName:Paused": "已暂停",
+    "DisplayName:FailedRetry": "失败重试",
     "DisplayName:Stopped": "已停止",
     "DisplayName:Once": "一次性",
     "DisplayName:Period": "周期性",

aspnet-core/modules/task-management/LINGYUN.Abp.TaskManagement.Domain/LINGYUN/Abp/TaskManagement/BackgroundJobStore.cs

@@ -77,17 +77,17 @@ public class BackgroundJobStore : IJobStore, ITransientDependency
                 jobInfo.MaxCount,
                 jobInfo.MaxTryCount)
             {
-                IsEnabled = true
+                IsEnabled = true,
+                TriggerCount = jobInfo.TriggerCount,
+                IsAbandoned = jobInfo.IsAbandoned,
+                TryCount = jobInfo.TryCount,
+                LockTimeOut = jobInfo.LockTimeOut,
+                Description = jobInfo.Description
             };
             backgroundJobInfo.SetNextRunTime(jobInfo.NextRunTime);
             backgroundJobInfo.SetLastRunTime(jobInfo.LastRunTime);
             backgroundJobInfo.SetStatus(jobInfo.Status);
             backgroundJobInfo.SetResult(jobInfo.Result);
-            backgroundJobInfo.TriggerCount = jobInfo.TriggerCount;
-            backgroundJobInfo.IsAbandoned = jobInfo.IsAbandoned;
-            backgroundJobInfo.TryCount = jobInfo.TryCount;
-            backgroundJobInfo.LockTimeOut = jobInfo.LockTimeOut;
-            backgroundJobInfo.Description = jobInfo.Description;
             switch (jobInfo.JobType)
             {
                 case JobType.Once:

aspnet-core/modules/task-management/LINGYUN.Abp.TaskManagement.EntityFrameworkCore/LINGYUN/Abp/TaskManagement/EntityFrameworkCore/EfCoreBackgroundJobInfoRepository.cs

@@ -53,11 +53,11 @@ public class EfCoreBackgroundJobInfoRepository :
 
     public virtual async Task<List<BackgroundJobInfo>> GetAllPeriodTasksAsync(CancellationToken cancellationToken = default)
     {
+        var status = new JobStatus[] { JobStatus.Running, JobStatus.FailedRetry };
+
         return await (await GetDbSetAsync())
             .Where(x => x.IsEnabled && !x.IsAbandoned)
-            .Where(x => x.JobType == JobType.Period && x.Status == JobStatus.Running && !x.NextRunTime.HasValue)
-            .Where(x => x.MaxCount == 0 || x.TriggerCount < x.MaxCount)
-            .Where(x => x.MaxTryCount == 0 || x.TryCount < x.MaxTryCount)
+            .Where(x => x.JobType == JobType.Period && status.Contains(x.Status))
             .OrderByDescending(x => x.Priority)
             .ToListAsync(GetCancellationToken(cancellationToken));
     }
@@ -101,7 +101,7 @@ public class EfCoreBackgroundJobInfoRepository :
             .WhereIf(filter.EndTime.HasValue, x => filter.EndTime.Value.CompareTo(x.EndTime) >= 0)
             .WhereIf(filter.BeginCreationTime.HasValue, x => x.CreationTime.CompareTo(filter.BeginCreationTime.Value) >= 0)
             .WhereIf(filter.EndCreationTime.HasValue, x => x.CreationTime.CompareTo(filter.EndCreationTime.Value) <= 0)
-            .OrderBy(sorting ?? nameof(BackgroundJobInfo.Name))
+            .OrderBy(sorting ?? nameof(BackgroundJobInfo.CreationTime))
             .PageBy(skipCount, maxResultCount)
             .ToListAsync(GetCancellationToken(cancellationToken));
     }
@@ -109,12 +109,11 @@ public class EfCoreBackgroundJobInfoRepository :
     public virtual async Task<List<BackgroundJobInfo>> GetWaitingListAsync(int maxResultCount, CancellationToken cancellationToken = default)
     {
         var now = Clock.Now;
+        var status = new JobStatus[] { JobStatus.Running, JobStatus.FailedRetry };
 
         return await (await GetDbSetAsync())
             .Where(x => x.IsEnabled && !x.IsAbandoned)
-            .Where(x => x.JobType != JobType.Period && x.Status == JobStatus.Running && !x.NextRunTime.HasValue)
-            .Where(x => x.MaxCount == 0 || x.TriggerCount < x.MaxCount)
-            .Where(x => x.MaxTryCount == 0 || x.TryCount < x.MaxTryCount)
+            .Where(x => x.JobType != JobType.Period && status.Contains(x.Status))
             .OrderByDescending(x => x.Priority)
             .ThenBy(x => x.TryCount)
             .ThenBy(x => x.NextRunTime)