Merge pull request #804 from colinin/checking-job

Checking job

apps/vue/src/api/task-management/model/backgroundJobInfoModel.ts

@@ -1,6 +1,7 @@
 export enum JobStatus {
   None = -1,
   Completed = 0,
+  Queuing = 5,
   Running = 10,
   FailedRetry = 15,
   Paused = 20,

apps/vue/src/views/task-management/background-jobs/components/JobModal.vue

@@ -54,14 +54,14 @@
             <DatePicker
               style="width: 100%"
               :value="getDate('beginTime')"
-              @change="(val) => dateChange('beginTime', val, 'YYYY-MM-DD 00:00:00')"
+              @change="(val) => dateChange('beginTime', val, 'YYYY-MM-DDT00:00:00')"
             />
           </FormItem>
           <FormItem name="endTime" :label="L('DisplayName:EndTime')">
             <DatePicker
               style="width: 100%"
               :value="getDate('endTime')"
-              @change="(val) => dateChange('endTime', val, 'YYYY-MM-DD 23:59:59')"
+              @change="(val) => dateChange('endTime', val, 'YYYY-MM-DDT23:59:59')"
             />
           </FormItem>
           <FormItem

apps/vue/src/views/task-management/background-jobs/datas/ModalData.ts

@@ -41,8 +41,10 @@ export function getSearchFormSchemas(): Partial<FormProps> {
         componentProps: {
           options: [
             { label: JobStatusMap[JobStatus.None], value: JobStatus.None },
+            { label: JobStatusMap[JobStatus.Queuing], value: JobStatus.Queuing },
             { label: JobStatusMap[JobStatus.Running], value: JobStatus.Running },
             { label: JobStatusMap[JobStatus.Completed], value: JobStatus.Completed },
+            { label: JobStatusMap[JobStatus.FailedRetry], value: JobStatus.FailedRetry },
             { label: JobStatusMap[JobStatus.Paused], value: JobStatus.Paused },
             { label: JobStatusMap[JobStatus.Stopped], value: JobStatus.Stopped },
           ],

apps/vue/src/views/task-management/background-jobs/datas/typing.ts

@@ -6,6 +6,7 @@ const { L } = useLocalization('TaskManagement');
 export const JobStatusMap = {
   [JobStatus.None]: L('DisplayName:None'),
   [JobStatus.Completed]: L('DisplayName:Completed'),
+  [JobStatus.Queuing]: L('DisplayName:Queuing'),
   [JobStatus.Running]: L('DisplayName:Running'),
   [JobStatus.FailedRetry]: L('DisplayName:FailedRetry'),
   [JobStatus.Paused]: L('DisplayName:Paused'),
@@ -14,6 +15,7 @@ export const JobStatusMap = {
 export const JobStatusColor = {
   [JobStatus.None]: '',
   [JobStatus.Completed]: '#339933',
+  [JobStatus.Queuing]: '#008B8B',
   [JobStatus.Running]: '#3399CC',
   [JobStatus.FailedRetry]: '#FF6600',
   [JobStatus.Paused]: '#CC6633',

aspnet-core/modules/task-management/LINGYUN.Abp.BackgroundTasks.Abstractions/LINGYUN/Abp/BackgroundTasks/AbpBackgroundTasksOptions.cs

@@ -85,6 +85,32 @@ public class AbpBackgroundTasksOptions
     /// 轮询任务也属于一个后台任务, 需要对每一次轮询加锁，防止重复任务入库
     /// </remarks>
     public int JobFetchLockTimeOut { get; set; }
+
+    /// <summary>
+    /// 启用检查任务
+    /// </summary>
+    /// <remarks>
+    /// 主节点启用
+    /// </remarks>
+    public bool JobCheckEnabled { get; set; }
+    /// <summary>
+    /// 每次检查任务批次大小
+    /// 默认: 100
+    /// </summary>
+    public int MaxJobCheckCount { get; set; }
+    /// <summary>
+    /// 检查任务批次Cron表达式
+    /// 默认: 每2小时执行一次（0 0 0/2 * * ? ）
+    /// </summary>
+    /// <remarks>
+    /// Cron表达式
+    /// </remarks>
+    public string JobCheckCronExpression { get; set; }
+    /// <summary>
+    /// 检查任务批次时锁定任务超时时长（秒）
+    /// 默认：120
+    /// </summary>
+    public int JobCheckLockTimeOut { get; set; }
     /// <summary>
     /// 指定运行节点
     /// </summary>
@@ -96,6 +122,11 @@ public class AbpBackgroundTasksOptions
         JobFetchLockTimeOut = 120;
         JobFetchCronExpression = "0/30 * * * * ? ";
 
+        JobCheckEnabled = false;
+        MaxJobCheckCount = 100;
+        JobCheckLockTimeOut = 120;
+        JobCheckCronExpression = "0 0 0/2 * * ? ";
+
         JobCleanEnabled = false;
         MaxJobCleanCount = 1000;
         JobExpiratime = TimeSpan.FromDays(15d);

aspnet-core/modules/task-management/LINGYUN.Abp.BackgroundTasks.Abstractions/LINGYUN/Abp/BackgroundTasks/DisableJobActionAttribute.cs

@@ -1,7 +1,9 @@
 using System;
 
 namespace LINGYUN.Abp.BackgroundTasks;
-
+/// <summary>
+/// 禁用作业调度行为
+/// </summary>
 [AttributeUsage(AttributeTargets.Class)]
 public class DisableJobActionAttribute : Attribute
 {

aspnet-core/modules/task-management/LINGYUN.Abp.BackgroundTasks.Abstractions/LINGYUN/Abp/BackgroundTasks/DisableJobStatusAttribute.cs

@@ -0,0 +1,10 @@
+using System;
+
+namespace LINGYUN.Abp.BackgroundTasks;
+/// <summary>
+/// 禁用作业调度状态
+/// </summary>
+[AttributeUsage(AttributeTargets.Class)]
+public class DisableJobStatusAttribute : Attribute
+{
+}

aspnet-core/modules/task-management/LINGYUN.Abp.BackgroundTasks.Abstractions/LINGYUN/Abp/BackgroundTasks/JobStatus.cs

@@ -11,6 +11,10 @@ public enum JobStatus
     /// </summary>
     Completed = 0,
     /// <summary>
+    /// 队列中
+    /// </summary>
+    Queuing = 5,
+    /// <summary>
     /// 运行中
     /// </summary>
     Running = 10,

aspnet-core/modules/task-management/LINGYUN.Abp.BackgroundTasks.Quartz/LINGYUN/Abp/BackgroundTasks/Quartz/QuartzJobListener.cs

@@ -40,7 +40,11 @@ public class QuartzJobListener : JobListenerSupport, ISingletonDependency
             jobName = !jobType.IsGenericType ? jobType.Name : jobType.GetGenericArguments()[0].Name;
         }
         
-        Logger.LogWarning($"The task {jobName} could not be performed...");
+        // 作业被锁定才记录warn事件
+        if (context.TryGetCache("JobLocked", out var time) && time != null && int.TryParse(time.ToString(), out var lockTime))
+        {
+            Logger.LogWarning($"The task {jobName} could not be performed, Because it is being scheduled by another job scheduler");
+        }
 
         return Task.FromResult(-1);
     }

aspnet-core/modules/task-management/LINGYUN.Abp.BackgroundTasks.Quartz/LINGYUN/Abp/BackgroundTasks/Quartz/QuartzJobSearchJobAdapter.cs

@@ -30,7 +30,7 @@ public class QuartzJobSearchJobAdapter : IJob
             jobDefinition.JobType,
             scope.ServiceProvider,
             context.MergedJobDataMap.ToImmutableDictionary(),
-            getCache: (key) => context.Get(key),
+            getCache: context.Get,
             setCache: context.Put,
             cancellationToken: context.CancellationToken);
 

aspnet-core/modules/task-management/LINGYUN.Abp.BackgroundTasks.Quartz/LINGYUN/Abp/BackgroundTasks/Quartz/QuartzTriggerListener.cs

@@ -49,6 +49,7 @@ public class QuartzTriggerListener : TriggerListenerSupport, ISingletonDependenc
             // 传递令牌将清除本次锁, 那并无意义
             if (!await JobLockProvider.TryLockAsync(NormalizeKey(context, jobId), time))
             {
+                context.Put("JobLocked", time);
                 Logger.LogDebug("The exclusive job is already in use by another scheduler. Ignore this schedule.");
                 return true;
             }

aspnet-core/modules/task-management/LINGYUN.Abp.BackgroundTasks.Quartz/Quartz/IJobExecutionContextExtensions.cs

@@ -42,4 +42,11 @@ public static class IJobExecutionContextExtensions
         }
         return false;
     }
+
+    public static bool TryGetCache(this IJobExecutionContext context, string key, out object value)
+    {
+        value = context.Get(key);
+
+        return value != null;
+    }
 }

aspnet-core/modules/task-management/LINGYUN.Abp.BackgroundTasks/LINGYUN/Abp/BackgroundTasks/BackgroundWorkerAdapter.cs

@@ -75,7 +75,6 @@ public class BackgroundWorkerAdapter<TWorker> : BackgroundWorkerBase, IBackgroun
             JobType = JobType.Persistent,
             Interval = period.Value / 1000,
             MaxCount = 0,
-            // TODO: 可配置
             MaxTryCount = 10,
             // 确保不会被轮询入队
             Status = JobStatus.None,

aspnet-core/modules/task-management/LINGYUN.Abp.BackgroundTasks/LINGYUN/Abp/BackgroundTasks/IJobStore.cs

@@ -7,6 +7,10 @@ namespace LINGYUN.Abp.BackgroundTasks;
 
 public interface IJobStore
 {
+    Task<List<JobInfo>> GetRuningListAsync(
+        int maxResultCount,
+        CancellationToken cancellationToken = default);
+
     Task<List<JobInfo>> GetWaitingListAsync(
         int maxResultCount,
         CancellationToken cancellationToken = default);

aspnet-core/modules/task-management/LINGYUN.Abp.BackgroundTasks/LINGYUN/Abp/BackgroundTasks/Internal/BackgroundCheckingJob.cs

@@ -0,0 +1,57 @@
+using Microsoft.Extensions.DependencyInjection;
+using Microsoft.Extensions.Logging;
+using Microsoft.Extensions.Options;
+using System;
+using System.Linq;
+using System.Threading.Tasks;
+using Volo.Abp.Auditing;
+using Volo.Abp.MultiTenancy;
+
+namespace LINGYUN.Abp.BackgroundTasks.Internal;
+
+[DisableAuditing]
+[DisableJobAction]
+public class BackgroundCheckingJob : IJobRunnable
+{
+    public async virtual Task ExecuteAsync(JobRunnableContext context)
+    {
+        try
+        {
+            var options = context.ServiceProvider.GetRequiredService<IOptions<AbpBackgroundTasksOptions>>().Value;
+            var store = context.ServiceProvider.GetRequiredService<IJobStore>();
+            var currentTenant = context.ServiceProvider.GetRequiredService<ICurrentTenant>();
+
+            context.TryGetMultiTenantId(out var tenantId);
+
+            using (currentTenant.Change(tenantId))
+            {
+                var runingTasks = await store.GetRuningListAsync(
+                    options.MaxJobCheckCount, context.CancellationToken);
+
+                if (!runingTasks.Any())
+                {
+                    return;
+                }
+
+                var jobScheduler = context.ServiceProvider.GetRequiredService<IJobScheduler>();
+
+                foreach (var job in runingTasks)
+                {
+                    // 当标记为运行中的作业不在调度器中时，改变为已停止作业
+                    if (!await jobScheduler.ExistsAsync(job, context.CancellationToken))
+                    {
+                        job.Status = JobStatus.Stopped;
+
+                        await store.StoreAsync(job);
+                    }
+                }
+            }
+        }
+        catch(Exception ex)
+        {
+            context.ServiceProvider
+                .GetService<ILogger<BackgroundCheckingJob>>()
+                ?.LogError(ex, "check job status error.");
+        }
+    }
+}

aspnet-core/modules/task-management/LINGYUN.Abp.BackgroundTasks/LINGYUN/Abp/BackgroundTasks/Internal/DefaultBackgroundWorker.cs

@@ -27,6 +27,7 @@ internal class DefaultBackgroundWorker : BackgroundService
     {
         await QueuePollingJob();
         await QueueCleaningJob();
+        await QueueCheckingJob();
     }
 
     private async Task QueuePollingJob()
@@ -47,6 +48,17 @@ internal class DefaultBackgroundWorker : BackgroundService
         }
     }
 
+    private async Task QueueCheckingJob()
+    {
+        if (_options.JobCheckEnabled)
+        {
+            var checkingJob = BuildCheckingJobInfo();
+            await _jobPublisher.PublishAsync(checkingJob);
+        }
+    }
+
+
+
     private JobInfo BuildPollingJobInfo()
     {
         return new JobInfo
@@ -89,4 +101,26 @@ internal class DefaultBackgroundWorker : BackgroundService
             Type = typeof(BackgroundCleaningJob).AssemblyQualifiedName,
         };
     }
+
+    private JobInfo BuildCheckingJobInfo()
+    {
+        return new JobInfo
+        {
+            Id = "Checking",
+            Name = nameof(BackgroundCheckingJob),
+            Group = "Checking",
+            Description = "Checking tasks to be executed",
+            Args = new Dictionary<string, object>(),
+            Status = JobStatus.Running,
+            BeginTime = DateTime.Now,
+            CreationTime = DateTime.Now,
+            Cron = _options.JobCheckCronExpression,
+            LockTimeOut = _options.JobCheckLockTimeOut,
+            JobType = JobType.Period,
+            Priority = JobPriority.High,
+            Source = JobSource.System,
+            NodeName = _options.NodeName,
+            Type = typeof(BackgroundCheckingJob).AssemblyQualifiedName,
+        };
+    }
 }

aspnet-core/modules/task-management/LINGYUN.Abp.BackgroundTasks/LINGYUN/Abp/BackgroundTasks/Internal/InMemoryJobStore.cs

@@ -11,17 +11,18 @@ namespace LINGYUN.Abp.BackgroundTasks.Internal;
 internal class InMemoryJobStore : IJobStore, ISingletonDependency
 {
     private readonly List<JobInfo> _memoryJobStore;
+    private readonly static object _jobSync = new();
 
     public InMemoryJobStore()
     {
         _memoryJobStore = new List<JobInfo>();
     }
 
-    public Task<List<JobInfo>> GetAllPeriodTasksAsync(CancellationToken cancellationToken = default)
+    public virtual Task<List<JobInfo>> GetAllPeriodTasksAsync(CancellationToken cancellationToken = default)
     {
         cancellationToken.ThrowIfCancellationRequested();
 
-        var status = new JobStatus[] { JobStatus.Running, JobStatus.FailedRetry };
+        var status = new JobStatus[] { JobStatus.Queuing, JobStatus.FailedRetry };
 
         var jobs = _memoryJobStore
             .Where(x => !x.IsAbandoned)
@@ -33,12 +34,26 @@ internal class InMemoryJobStore : IJobStore, ISingletonDependency
         return Task.FromResult(jobs);
     }
 
-    public Task<List<JobInfo>> GetWaitingListAsync(int maxResultCount, CancellationToken cancellationToken = default)
+    public virtual Task<List<JobInfo>> GetRuningListAsync(int maxResultCount, CancellationToken cancellationToken = default)
+    {
+        cancellationToken.ThrowIfCancellationRequested();
+
+        var status = new JobStatus[] { JobStatus.Running };
+
+        var jobs = _memoryJobStore
+            .Where(x => status.Contains(x.Status))
+            .Take(maxResultCount)
+            .ToList();
+
+        return Task.FromResult(jobs);
+    }
+
+    public virtual Task<List<JobInfo>> GetWaitingListAsync(int maxResultCount, CancellationToken cancellationToken = default)
     {
         cancellationToken.ThrowIfCancellationRequested();
 
         var now = DateTime.Now;
-        var status = new JobStatus[] { JobStatus.Running, JobStatus.FailedRetry };
+        var status = new JobStatus[] { JobStatus.Queuing, JobStatus.FailedRetry };
 
         var jobs = _memoryJobStore
             .Where(x => !x.IsAbandoned)
@@ -63,62 +78,76 @@ internal class InMemoryJobStore : IJobStore, ISingletonDependency
         return Task.FromResult(job);
     }
 
-    public Task StoreAsync(
+    public virtual Task StoreAsync(
         JobInfo jobInfo,
         CancellationToken cancellationToken = default)
     {
         cancellationToken.ThrowIfCancellationRequested();
 
-        var job = _memoryJobStore.FirstOrDefault(x => x.Id.Equals(jobInfo.Id));
-        if (job != null)
+        lock(_jobSync)
         {
-            job.NextRunTime = jobInfo.NextRunTime;
-            job.LastRunTime = jobInfo.LastRunTime;
-            job.Status = jobInfo.Status;
-            job.TriggerCount = jobInfo.TriggerCount;
-            job.TryCount = jobInfo.TryCount;
-            job.IsAbandoned = jobInfo.IsAbandoned;
-        }
-        else
-        {
-            _memoryJobStore.Add(jobInfo);
+            var job = _memoryJobStore.FirstOrDefault(x => x.Id.Equals(jobInfo.Id));
+            if (job != null)
+            {
+                job.NextRunTime = jobInfo.NextRunTime;
+                job.LastRunTime = jobInfo.LastRunTime;
+                job.Status = jobInfo.Status;
+                job.TriggerCount = jobInfo.TriggerCount;
+                job.TryCount = jobInfo.TryCount;
+                job.IsAbandoned = jobInfo.IsAbandoned;
+            }
+            else
+            {
+                _memoryJobStore.Add(jobInfo);
+            }
         }
+
         return Task.CompletedTask;
     }
 
-    public async Task RemoveAsync(
+    public virtual Task RemoveAsync(
         string jobId,
         CancellationToken cancellationToken = default)
     {
         cancellationToken.ThrowIfCancellationRequested();
 
-        var job = await FindAsync(jobId, cancellationToken);
-        if (job != null)
+        lock (_jobSync)
         {
-            _memoryJobStore.Remove(job);
+            var job = _memoryJobStore.FirstOrDefault(x => x.Id.Equals(jobId));
+            if (job != null)
+            {
+                _memoryJobStore.Remove(job);
+            }
         }
+
+        return Task.CompletedTask;
     }
 
-    public Task StoreLogAsync(JobEventData eventData)
+    public virtual Task StoreLogAsync(JobEventData eventData)
     {
         eventData.CancellationToken.ThrowIfCancellationRequested();
 
         return Task.CompletedTask;
     }
 
-    public Task<List<JobInfo>> CleanupAsync(int maxResultCount, TimeSpan jobExpiratime, CancellationToken cancellationToken = default)
+    public virtual Task<List<JobInfo>> CleanupAsync(int maxResultCount, TimeSpan jobExpiratime, CancellationToken cancellationToken = default)
     {
         cancellationToken.ThrowIfCancellationRequested();
 
-        var expiratime = DateTime.Now - jobExpiratime;
+        var expriaJobs = new List<JobInfo>();
 
-        var expriaJobs = _memoryJobStore
-            .Where(x => x.Status == JobStatus.Completed &&
-                expiratime.CompareTo(x.LastRunTime ?? x.EndTime ?? x.CreationTime) <= 0)
-            .Take(maxResultCount)
-            .ToList();
+        lock (_jobSync)
+        {
+            var expiratime = DateTime.Now - jobExpiratime;
 
-        _memoryJobStore.RemoveAll(expriaJobs);
+            expriaJobs = _memoryJobStore
+                    .Where(x => x.Status == JobStatus.Completed &&
+                        expiratime.CompareTo(x.LastRunTime ?? x.EndTime ?? x.CreationTime) <= 0)
+                    .Take(maxResultCount)
+                    .ToList();
+
+            _memoryJobStore.RemoveAll(expriaJobs);
+        }
 
         return Task.FromResult(expriaJobs);
     }

aspnet-core/modules/task-management/LINGYUN.Abp.BackgroundTasks/LINGYUN/Abp/BackgroundTasks/Internal/JobExecutedEvent.cs

@@ -10,6 +10,10 @@ public class JobExecutedEvent : JobEventBase<JobExecutedEvent>, ITransientDepend
 {
     protected override async Task OnJobAfterExecutedAsync(JobEventContext context)
     {
+        if (context.EventData.Type.IsDefined(typeof(DisableJobStatusAttribute), true))
+        {
+            return;
+        }
         var store = context.ServiceProvider.GetRequiredService<IJobStore>();
         var job = await store.FindAsync(context.EventData.Key, context.EventData.CancellationToken);
         if (job != null)

aspnet-core/modules/task-management/LINGYUN.Abp.TaskManagement.Domain.Shared/LINGYUN/Abp/TaskManagement/Localization/Resources/en.json

@@ -57,6 +57,7 @@
     "DisplayName:None": "None",
     "DisplayName:Completed": "Completed",
     "DisplayName:Running": "Running",
+    "DisplayName:Queuing": "Queuing",
     "DisplayName:Paused": "Paused",
     "DisplayName:FailedRetry": "Failed Retry",
     "DisplayName:Stopped": "Stopped",

aspnet-core/modules/task-management/LINGYUN.Abp.TaskManagement.Domain.Shared/LINGYUN/Abp/TaskManagement/Localization/Resources/zh-Hans.json

@@ -57,6 +57,7 @@
     "DisplayName:None": "未定义",
     "DisplayName:Completed": "已完成",
     "DisplayName:Running": "运行中",
+    "DisplayName:Queuing": "队列中",
     "DisplayName:Paused": "已暂停",
     "DisplayName:FailedRetry": "失败重试",
     "DisplayName:Stopped": "已停止",

aspnet-core/modules/task-management/LINGYUN.Abp.TaskManagement.Domain/LINGYUN/Abp/TaskManagement/BackgroundJobInfo.cs

@@ -141,7 +141,7 @@ public class BackgroundJobInfo : AuditedAggregateRoot<string>, IMultiTenant
         NodeName = Check.Length(nodeName, nameof(nodeName), BackgroundJobInfoConsts.MaxNodeNameLength);
         TenantId = tenantId;
 
-        Status = JobStatus.Running;
+        Status = JobStatus.Queuing;
 
         // TODO: 是否需要将参数挪到另一个实体? 
         // 任务参数的建议是尽量最小化, 仅存储关键信息

aspnet-core/modules/task-management/LINGYUN.Abp.TaskManagement.Domain/LINGYUN/Abp/TaskManagement/BackgroundJobInfoWaitingPeriodSpecification.cs

@@ -8,7 +8,7 @@ public class BackgroundJobInfoWaitingPeriodSpecification : BackgroundJobInfoWait
 {
     public override Expression<Func<BackgroundJobInfo, bool>> ToExpression()
     {
-        var status = new JobStatus[] { JobStatus.Running, JobStatus.FailedRetry };
+        var status = new JobStatus[] { JobStatus.Queuing, JobStatus.FailedRetry };
 
         Expression<Func<BackgroundJobInfo, bool>> expression = _ => true;
         return expression

aspnet-core/modules/task-management/LINGYUN.Abp.TaskManagement.Domain/LINGYUN/Abp/TaskManagement/BackgroundJobInfoWaitingSpecification.cs

@@ -9,7 +9,7 @@ public class BackgroundJobInfoWaitingSpecification : Volo.Abp.Specifications.Spe
 {
     public override Expression<Func<BackgroundJobInfo, bool>> ToExpression()
     {
-        var status = new JobStatus[] { JobStatus.Running, JobStatus.FailedRetry };
+        var status = new JobStatus[] { JobStatus.Queuing, JobStatus.FailedRetry };
 
         Expression<Func<BackgroundJobInfo, bool>> expression = _ => true;
 

aspnet-core/modules/task-management/LINGYUN.Abp.TaskManagement.Domain/LINGYUN/Abp/TaskManagement/BackgroundJobManager.cs

@@ -170,7 +170,7 @@ public class BackgroundJobManager : DomainService
 
     public async virtual Task ResumeAsync(BackgroundJobInfo jobInfo)
     {
-        jobInfo.SetStatus(JobStatus.Running);
+        jobInfo.SetStatus(JobStatus.Queuing);
         jobInfo.IsAbandoned = false;
         jobInfo.IsEnabled = true;
 
@@ -192,7 +192,7 @@ public class BackgroundJobManager : DomainService
     {
         foreach (var jobInfo in jobInfos)
         {
-            jobInfo.SetStatus(JobStatus.Running);
+            jobInfo.SetStatus(JobStatus.Queuing);
             jobInfo.IsAbandoned = false;
             jobInfo.IsEnabled = true;
         }

aspnet-core/modules/task-management/LINGYUN.Abp.TaskManagement.Domain/LINGYUN/Abp/TaskManagement/BackgroundJobStore.cs

@@ -40,6 +40,20 @@ public class BackgroundJobStore : IJobStore, ITransientDependency
         return ObjectMapper.Map<List<BackgroundJobInfo>, List<JobInfo>>(jobInfos);
     }
 
+    public async virtual Task<List<JobInfo>> GetRuningListAsync(int maxResultCount, CancellationToken cancellationToken = default)
+    {
+        var filter = new BackgroundJobInfoFilter
+        {
+            Status = JobStatus.Running
+        };
+        var specification = new BackgroundJobInfoSpecification(filter);
+
+        var jobInfos = await JobInfoRepository.GetListAsync(
+            specification, maxResultCount: maxResultCount, cancellationToken: cancellationToken);
+
+        return ObjectMapper.Map<List<BackgroundJobInfo>, List<JobInfo>>(jobInfos);
+    }
+
     public async virtual Task<List<JobInfo>> GetWaitingListAsync(
         int maxResultCount, 
         CancellationToken cancellationToken = default)

aspnet-core/services/LY.MicroService.TaskManagement.HttpApi.Host/EventBus/Handlers/TenantSynchronizer.cs

@@ -60,6 +60,10 @@ namespace LY.MicroService.TaskManagement.EventBus.Handlers
             var cleaningJob = BuildCleaningJobInfo(eventData.Entity.Id, eventData.Entity.Name);
             await JobScheduler.RemoveAsync(cleaningJob);
             await JobStore.RemoveAsync(cleaningJob.Id);
+
+            var checkingJob = BuildCheckingJobInfo(eventData.Entity.Id, eventData.Entity.Name);
+            await JobScheduler.RemoveAsync(checkingJob);
+            await JobStore.RemoveAsync(checkingJob.Id);
         }
 
         public async Task HandleEventAsync(CreateEventData eventData)
@@ -79,6 +83,10 @@ namespace LY.MicroService.TaskManagement.EventBus.Handlers
             var cleaningJob = BuildCleaningJobInfo(eventData.Id, eventData.Name);
             await JobStore.StoreAsync(cleaningJob);
             await JobScheduler.QueueAsync(cleaningJob);
+
+            var checkingJob = BuildCheckingJobInfo(eventData.Id, eventData.Name);
+            await JobStore.StoreAsync(checkingJob);
+            await JobScheduler.QueueAsync(checkingJob);
         }
 
         private async Task MigrateAsync(CreateEventData eventData)
@@ -145,5 +153,27 @@ namespace LY.MicroService.TaskManagement.EventBus.Handlers
                 Type = typeof(BackgroundCleaningJob).AssemblyQualifiedName,
             };
         }
+
+        private JobInfo BuildCheckingJobInfo(Guid tenantId, string tenantName)
+        {
+            return new JobInfo
+            {
+                Id = tenantId.ToString() + "_Checking",
+                Name = nameof(BackgroundCheckingJob),
+                Group = "Checking",
+                Description = "Checking tasks to be executed",
+                Args = new Dictionary<string, object>() { { nameof(JobInfo.TenantId), tenantId } },
+                Status = JobStatus.Running,
+                BeginTime = DateTime.Now,
+                CreationTime = DateTime.Now,
+                Cron = Options.JobCheckCronExpression,
+                LockTimeOut = Options.JobCheckLockTimeOut,
+                JobType = JobType.Period,
+                Priority = JobPriority.High,
+                Source = JobSource.System,
+                TenantId = tenantId,
+                Type = typeof(BackgroundCheckingJob).AssemblyQualifiedName,
+            };
+        }
     }
 }