Housekeeper: reprocessing improvements

application/src/main/java/org/thingsboard/server/service/housekeeper/DefaultHousekeeperService.java

@@ -99,7 +99,7 @@ public class DefaultHousekeeperService implements HousekeeperService {
                             processTask(msg);
                         } catch (Throwable e) {
                             log.error("Unexpected error during message processing [{}]", msg, e);
-                            reprocessingService.submitForReprocessing(msg);
+                            reprocessingService.submitForReprocessing(msg, e);
                         }
                     }
                     consumer.commit();
@@ -131,23 +131,23 @@ public class DefaultHousekeeperService implements HousekeeperService {
                 return null;
             });
             future.get(taskProcessingTimeout, TimeUnit.SECONDS);
-        } catch (ExecutionException executionException) {
+        } catch (Throwable e) {
-            Throwable error = executionException.getCause();
+            Throwable error = e;
+            if (e instanceof ExecutionException) {
+                error = error.getCause();
+            } else if (e instanceof TimeoutException) {
+                error = new TimeoutException("Timeout after " + taskProcessingTimeout + " seconds");
+            }
             log.error("[{}][{}][{}] {} task processing failed, submitting for reprocessing (attempt {}): {}",
                     task.getTenantId(), task.getEntityId().getEntityType(), task.getEntityId(),
                     task.getTaskType(), msg.getTask().getAttempt(), task, error);
-            reprocessingService.submitForReprocessing(queueMsg);
+            reprocessingService.submitForReprocessing(queueMsg, error);
-        } catch (TimeoutException timeoutException) {
-            log.error("[{}][{}][{}] {} task processing timeout after {} seconds, submitting for reprocessing (attempt {}): {}",
-                    task.getTenantId(), task.getEntityId().getEntityType(), task.getEntityId(),
-                    task.getTaskType(), taskProcessingTimeout, msg.getTask().getAttempt(), task);
-            reprocessingService.submitForReprocessing(queueMsg); // fixme: we should schedule such task for later (separate queue?)
         }
     }
 
     @Override
     public void submitTask(HousekeeperTask task) {
-        submitTask(UUID.randomUUID(), task);
+        submitTask(task.getEntityId().getId(), task);
     }
 
     @Override

application/src/main/java/org/thingsboard/server/service/housekeeper/HousekeeperReprocessingService.java

@@ -16,18 +16,23 @@
 package org.thingsboard.server.service.housekeeper;
 
 import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.lang3.exception.ExceptionUtils;
 import org.springframework.beans.factory.annotation.Value;
 import org.springframework.context.annotation.Lazy;
+import org.springframework.scheduling.annotation.Scheduled;
 import org.springframework.stereotype.Service;
 import org.thingsboard.common.util.ThingsBoardThreadFactory;
+import org.thingsboard.server.common.data.StringUtils;
+import org.thingsboard.server.common.data.id.TenantId;
+import org.thingsboard.server.common.msg.queue.ServiceType;
 import org.thingsboard.server.common.msg.queue.TopicPartitionInfo;
 import org.thingsboard.server.gen.transport.TransportProtos.HousekeeperTaskProto;
 import org.thingsboard.server.gen.transport.TransportProtos.ToHousekeeperServiceMsg;
 import org.thingsboard.server.queue.TbQueueConsumer;
 import org.thingsboard.server.queue.common.TbProtoQueueMsg;
+import org.thingsboard.server.queue.discovery.PartitionService;
 import org.thingsboard.server.queue.provider.TbCoreQueueFactory;
 import org.thingsboard.server.queue.provider.TbQueueProducerProvider;
-import org.thingsboard.server.queue.util.AfterStartUp;
 import org.thingsboard.server.queue.util.TbCoreComponent;
 
 import javax.annotation.PreDestroy;
@@ -42,35 +47,46 @@ import java.util.concurrent.TimeUnit;
 public class HousekeeperReprocessingService {
 
     private final DefaultHousekeeperService housekeeperService;
+    private final PartitionService partitionService;
+    private final TbCoreQueueFactory queueFactory;
     private final TbQueueProducerProvider producerProvider;
-    private final TbQueueConsumer<TbProtoQueueMsg<ToHousekeeperServiceMsg>> consumer;
+
     private final ExecutorService consumerExecutor = Executors.newSingleThreadExecutor(ThingsBoardThreadFactory.forName("housekeeper-reprocessing-consumer"));
 
+    private static final int startDelay = 15; // fixme - to 5 minutes
+    private static final int reprocessingDelay = 30; // seconds
+    private static final int maxReprocessingAttempts = 5;
     @Value("${queue.core.housekeeper.poll-interval-ms:500}")
     private int pollInterval;
 
-    private final long startTs = System.currentTimeMillis(); // fixme: some other tb-core might start earlier and submit for reprocessing
     private boolean stopped;
     // todo: stats
 
     public HousekeeperReprocessingService(@Lazy DefaultHousekeeperService housekeeperService,
-                                          TbCoreQueueFactory queueFactory,
+                                          PartitionService partitionService, TbCoreQueueFactory queueFactory,
                                           TbQueueProducerProvider producerProvider) {
         this.housekeeperService = housekeeperService;
-        this.consumer = queueFactory.createHousekeeperDelayedMsgConsumer();
+        this.partitionService = partitionService;
+        this.queueFactory = queueFactory;
         this.producerProvider = producerProvider;
     }
 
-    @AfterStartUp(order = AfterStartUp.REGULAR_SERVICE)
+    @Scheduled(initialDelay = startDelay, fixedDelay = reprocessingDelay, timeUnit = TimeUnit.SECONDS)
-    public void afterStartUp() {
+    public void startReprocessing() {
+        if (!partitionService.isMyPartition(ServiceType.TB_CORE, TenantId.SYS_TENANT_ID, TenantId.SYS_TENANT_ID)) {
+            return;
+        }
+
+        var consumer = queueFactory.createHousekeeperReprocessingMsgConsumer();
         consumer.subscribe();
         consumerExecutor.submit(() -> {
-            while (!stopped && !consumer.isStopped()) {
+            log.info("Starting Housekeeper tasks reprocessing");
+            long startTs = System.currentTimeMillis();
+            while (!stopped) {
                 try {
                     List<TbProtoQueueMsg<ToHousekeeperServiceMsg>> msgs = consumer.poll(pollInterval);
                     if (msgs.isEmpty() || msgs.stream().anyMatch(msg -> msg.getValue().getTask().getTs() >= startTs)) { // msg batch size should be 1. otherwise some tasks won't be reprocessed immediately
-                        stop();
+                        break;
-                        return;
                     }
 
                     for (TbProtoQueueMsg<ToHousekeeperServiceMsg> msg : msgs) {
@@ -78,7 +94,7 @@ public class HousekeeperReprocessingService {
                             housekeeperService.processTask(msg);
                         } catch (Exception e) {
                             log.error("Unexpected error during message reprocessing [{}]", msg, e);
-                            submitForReprocessing(msg);
+                            submitForReprocessing(msg, e);
                         }
                     }
                     consumer.commit();
@@ -93,31 +109,33 @@ public class HousekeeperReprocessingService {
                     }
                 }
             }
+            consumer.unsubscribe();
+            log.info("Stopped Housekeeper tasks reprocessing");
         });
-        log.info("Started Housekeeper tasks reprocessing");
     }
 
-    public void submitForReprocessing(TbProtoQueueMsg<ToHousekeeperServiceMsg> queueMsg) {
+    // todo: dead letter queue if attempts count exceeds the configured maximum
+    public void submitForReprocessing(TbProtoQueueMsg<ToHousekeeperServiceMsg> queueMsg, Throwable error) {
         ToHousekeeperServiceMsg msg = queueMsg.getValue();
         HousekeeperTaskProto task = msg.getTask();
         int attempt = task.getAttempt() + 1;
         msg = msg.toBuilder()
                 .setTask(task.toBuilder()
                         .setAttempt(attempt)
-                        .setTs(System.currentTimeMillis() + TimeUnit.HOURS.toMillis(1)) // so that it is not reprocessed by anyone in the nearest hour
+                        .addErrors(StringUtils.truncate(ExceptionUtils.getStackTrace(error), 1024))
+                        .setTs(System.currentTimeMillis() + TimeUnit.SECONDS.toMillis(reprocessingDelay))
                         .build())
                 .build();
 
-        var producer = producerProvider.getHousekeeperDelayedMsgProducer();
+        log.trace("Submitting for reprocessing: {}", msg);
+        var producer = producerProvider.getHousekeeperReprocessingMsgProducer();
         TopicPartitionInfo tpi = TopicPartitionInfo.builder().topic(producer.getDefaultTopic()).build();
         producer.send(tpi, new TbProtoQueueMsg<>(queueMsg.getKey(), msg), null);
     }
 
     @PreDestroy
     private void stop() {
-        log.info("Stopped Housekeeper tasks reprocessing");
         stopped = true;
-        consumer.unsubscribe();
         consumerExecutor.shutdownNow();
     }
 

application/src/main/java/org/thingsboard/server/service/housekeeper/processor/EventsDeletionTaskProcessor.java

@@ -29,6 +29,7 @@ public class EventsDeletionTaskProcessor implements HousekeeperTaskProcessor<Hou
     @Override
     public void process(HousekeeperTask task) throws Exception {
         eventService.removeEvents(task.getTenantId(), task.getEntityId(), null, 0L, System.currentTimeMillis());
+        throw new RuntimeException("test error");
     }
 
     @Override

common/proto/src/main/proto/queue.proto

@@ -24,36 +24,36 @@ option java_outer_classname = "TransportProtos";
  * Common data structures
  */
 enum EntityTypeProto {
-  UNSPECIFIED           = 0;
+  UNSPECIFIED = 0;
-  TENANT                = 1;
+  TENANT = 1;
-  CUSTOMER              = 2;
+  CUSTOMER = 2;
-  USER                  = 3;
+  USER = 3;
-  DASHBOARD             = 4;
+  DASHBOARD = 4;
-  ASSET                 = 5;
+  ASSET = 5;
-  DEVICE                = 6;
+  DEVICE = 6;
-  ALARM                 = 7;
+  ALARM = 7;
   // next 3 reserved for PE;
-  RULE_CHAIN            = 11;
+  RULE_CHAIN = 11;
-  RULE_NODE             = 12;
+  RULE_NODE = 12;
   // next 2 reserved for PE;
-  ENTITY_VIEW           = 15;
+  ENTITY_VIEW = 15;
-  WIDGETS_BUNDLE        = 16;
+  WIDGETS_BUNDLE = 16;
-  WIDGET_TYPE           = 17;
+  WIDGET_TYPE = 17;
   // next 2 reserved for PE;
-  TENANT_PROFILE        = 20;
+  TENANT_PROFILE = 20;
-  DEVICE_PROFILE        = 21;
+  DEVICE_PROFILE = 21;
-  ASSET_PROFILE         = 22;
+  ASSET_PROFILE = 22;
-  API_USAGE_STATE       = 23;
+  API_USAGE_STATE = 23;
-  TB_RESOURCE           = 24;
+  TB_RESOURCE = 24;
-  OTA_PACKAGE           = 25;
+  OTA_PACKAGE = 25;
-  EDGE                  = 26;
+  EDGE = 26;
-  RPC                   = 27;
+  RPC = 27;
-  QUEUE                 = 28;
+  QUEUE = 28;
-  NOTIFICATION_TARGET   = 29;
+  NOTIFICATION_TARGET = 29;
   NOTIFICATION_TEMPLATE = 30;
-  NOTIFICATION_REQUEST  = 31;
+  NOTIFICATION_REQUEST = 31;
-  NOTIFICATION          = 32;
+  NOTIFICATION = 32;
-  NOTIFICATION_RULE     = 33;
+  NOTIFICATION_RULE = 33;
 }
 
 /**
@@ -133,9 +133,9 @@ message KeyValueProto {
 }
 
 enum AttributeScopeProto {
-    CLIENT_SCOPE = 0;
+  CLIENT_SCOPE = 0;
-    SERVER_SCOPE = 1;
+  SERVER_SCOPE = 1;
-    SHARED_SCOPE = 2;
+  SHARED_SCOPE = 2;
 }
 
 message AttributeKey {
@@ -259,7 +259,7 @@ message GetOrCreateDeviceFromGatewayResponseMsg {
 }
 
 enum TransportApiRequestErrorCode {
-  UNKNOWN_TRANSPORT_API_ERROR  = 0;
+  UNKNOWN_TRANSPORT_API_ERROR = 0;
   ENTITY_LIMIT = 1;
 }
 
@@ -894,15 +894,15 @@ message FromDeviceRPCResponseProto {
 }
 
 enum ComponentLifecycleEvent {
-    CREATED       = 0;
+  CREATED = 0;
-    STARTED       = 1;
+  STARTED = 1;
-    ACTIVATED     = 2;
+  ACTIVATED = 2;
-    SUSPENDED     = 3;
+  SUSPENDED = 3;
-    UPDATED       = 4;
+  UPDATED = 4;
-    STOPPED       = 5;
+  STOPPED = 5;
-    DELETED       = 6;
+  DELETED = 6;
-    FAILED        = 7;
+  FAILED = 7;
-    DEACTIVATED   = 8;
+  DEACTIVATED = 8;
 }
 
 message ComponentLifecycleMsgProto {
@@ -1399,5 +1399,8 @@ message ToHousekeeperServiceMsg {
 message HousekeeperTaskProto {
   bytes value = 1;
   int64 ts = 2;
-  int32 attempt = 3;
+
+  // reprocessing
+  int32 attempt = 50;
+  repeated string errors = 51;
 }

common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsMonolithQueueFactory.java

@@ -238,12 +238,12 @@ public class AwsSqsMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEng
     }
 
     @Override
-    public TbQueueProducer<TbProtoQueueMsg<ToHousekeeperServiceMsg>> createHousekeeperDelayedMsgProducer() {
+    public TbQueueProducer<TbProtoQueueMsg<ToHousekeeperServiceMsg>> createHousekeeperReprocessingMsgProducer() {
         return null;
     }
 
     @Override
-    public TbQueueConsumer<TbProtoQueueMsg<ToHousekeeperServiceMsg>> createHousekeeperDelayedMsgConsumer() {
+    public TbQueueConsumer<TbProtoQueueMsg<ToHousekeeperServiceMsg>> createHousekeeperReprocessingMsgConsumer() {
         return null;
     }
 

common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTbCoreQueueFactory.java

@@ -217,12 +217,12 @@ public class AwsSqsTbCoreQueueFactory implements TbCoreQueueFactory {
     }
 
     @Override
-    public TbQueueProducer<TbProtoQueueMsg<ToHousekeeperServiceMsg>> createHousekeeperDelayedMsgProducer() {
+    public TbQueueProducer<TbProtoQueueMsg<ToHousekeeperServiceMsg>> createHousekeeperReprocessingMsgProducer() {
         return null;
     }
 
     @Override
-    public TbQueueConsumer<TbProtoQueueMsg<ToHousekeeperServiceMsg>> createHousekeeperDelayedMsgConsumer() {
+    public TbQueueConsumer<TbProtoQueueMsg<ToHousekeeperServiceMsg>> createHousekeeperReprocessingMsgConsumer() {
         return null;
     }
 

common/queue/src/main/java/org/thingsboard/server/queue/provider/InMemoryMonolithQueueFactory.java

@@ -171,13 +171,13 @@ public class InMemoryMonolithQueueFactory implements TbCoreQueueFactory, TbRuleE
     }
 
     @Override
-    public TbQueueProducer<TbProtoQueueMsg<TransportProtos.ToHousekeeperServiceMsg>> createHousekeeperDelayedMsgProducer() {
+    public TbQueueProducer<TbProtoQueueMsg<TransportProtos.ToHousekeeperServiceMsg>> createHousekeeperReprocessingMsgProducer() {
-        return new InMemoryTbQueueProducer<>(storage, topicService.buildTopicName(coreSettings.getHousekeeperDelayedTopic()));
+        return new InMemoryTbQueueProducer<>(storage, topicService.buildTopicName(coreSettings.getHousekeeperReprocessingTopic()));
     }
 
     @Override
-    public TbQueueConsumer<TbProtoQueueMsg<TransportProtos.ToHousekeeperServiceMsg>> createHousekeeperDelayedMsgConsumer() {
+    public TbQueueConsumer<TbProtoQueueMsg<TransportProtos.ToHousekeeperServiceMsg>> createHousekeeperReprocessingMsgConsumer() {
-        return new InMemoryTbQueueConsumer<>(storage, topicService.buildTopicName(coreSettings.getHousekeeperDelayedTopic()));
+        return new InMemoryTbQueueConsumer<>(storage, topicService.buildTopicName(coreSettings.getHousekeeperReprocessingTopic()));
     }
 
     @Scheduled(fixedRateString = "${queue.in_memory.stats.print-interval-ms:60000}")

common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaMonolithQueueFactory.java

@@ -373,22 +373,22 @@ public class KafkaMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEngi
     }
 
     @Override
-    public TbQueueProducer<TbProtoQueueMsg<ToHousekeeperServiceMsg>> createHousekeeperDelayedMsgProducer() {
+    public TbQueueProducer<TbProtoQueueMsg<ToHousekeeperServiceMsg>> createHousekeeperReprocessingMsgProducer() {
         return TbKafkaProducerTemplate.<TbProtoQueueMsg<ToHousekeeperServiceMsg>>builder()
                 .settings(kafkaSettings)
-                .clientId("monolith-housekeeper-delayed-producer-" + serviceInfoProvider.getServiceId())
+                .clientId("monolith-housekeeper-reprocessing-producer-" + serviceInfoProvider.getServiceId())
-                .defaultTopic(topicService.buildTopicName(coreSettings.getHousekeeperDelayedTopic()))
+                .defaultTopic(topicService.buildTopicName(coreSettings.getHousekeeperReprocessingTopic()))
                 .admin(housekeeperAdmin)
                 .build();
     }
 
     @Override
-    public TbQueueConsumer<TbProtoQueueMsg<ToHousekeeperServiceMsg>> createHousekeeperDelayedMsgConsumer() {
+    public TbQueueConsumer<TbProtoQueueMsg<ToHousekeeperServiceMsg>> createHousekeeperReprocessingMsgConsumer() {
         return TbKafkaConsumerTemplate.<TbProtoQueueMsg<ToHousekeeperServiceMsg>>builder()
                 .settings(kafkaSettings)
-                .topic(topicService.buildTopicName(coreSettings.getHousekeeperDelayedTopic()))
+                .topic(topicService.buildTopicName(coreSettings.getHousekeeperReprocessingTopic()))
-                .clientId("monolith-housekeeper-delayed-consumer-" + serviceInfoProvider.getServiceId())
+                .clientId("monolith-housekeeper-reprocessing-consumer-" + serviceInfoProvider.getServiceId())
-                .groupId(topicService.buildTopicName("monolith-housekeeper-delayed-consumer"))
+                .groupId(topicService.buildTopicName("monolith-housekeeper-reprocessing-consumer"))
                 .decoder(msg -> new TbProtoQueueMsg<>(msg.getKey(), ToHousekeeperServiceMsg.parseFrom(msg.getData()), msg.getHeaders()))
                 .admin(housekeeperAdmin)
                 .statsService(consumerStatsService)

common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbCoreQueueFactory.java

@@ -330,22 +330,22 @@ public class KafkaTbCoreQueueFactory implements TbCoreQueueFactory {
     }
 
     @Override
-    public TbQueueProducer<TbProtoQueueMsg<TransportProtos.ToHousekeeperServiceMsg>> createHousekeeperDelayedMsgProducer() {
+    public TbQueueProducer<TbProtoQueueMsg<TransportProtos.ToHousekeeperServiceMsg>> createHousekeeperReprocessingMsgProducer() {
         return TbKafkaProducerTemplate.<TbProtoQueueMsg<TransportProtos.ToHousekeeperServiceMsg>>builder()
                 .settings(kafkaSettings)
-                .clientId("tb-core-housekeeper-delayed-producer-" + serviceInfoProvider.getServiceId())
+                .clientId("tb-core-housekeeper-reprocessing-producer-" + serviceInfoProvider.getServiceId())
-                .defaultTopic(topicService.buildTopicName(coreSettings.getHousekeeperDelayedTopic()))
+                .defaultTopic(topicService.buildTopicName(coreSettings.getHousekeeperReprocessingTopic()))
                 .admin(housekeeperAdmin)
                 .build();
     }
 
     @Override
-    public TbQueueConsumer<TbProtoQueueMsg<TransportProtos.ToHousekeeperServiceMsg>> createHousekeeperDelayedMsgConsumer() {
+    public TbQueueConsumer<TbProtoQueueMsg<TransportProtos.ToHousekeeperServiceMsg>> createHousekeeperReprocessingMsgConsumer() {
         return TbKafkaConsumerTemplate.<TbProtoQueueMsg<TransportProtos.ToHousekeeperServiceMsg>>builder()
                 .settings(kafkaSettings)
-                .topic(topicService.buildTopicName(coreSettings.getHousekeeperDelayedTopic()))
+                .topic(topicService.buildTopicName(coreSettings.getHousekeeperReprocessingTopic()))
-                .clientId("tb-core-housekeeper-delayed-consumer-" + serviceInfoProvider.getServiceId())
+                .clientId("tb-core-housekeeper-reprocessing-consumer-" + serviceInfoProvider.getServiceId())
-                .groupId(topicService.buildTopicName("tb-core-housekeeper-delayed-consumer"))
+                .groupId(topicService.buildTopicName("tb-core-housekeeper-reprocessing-consumer"))
                 .decoder(msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportProtos.ToHousekeeperServiceMsg.parseFrom(msg.getData()), msg.getHeaders()))
                 .admin(housekeeperAdmin)
                 .statsService(consumerStatsService)

common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubMonolithQueueFactory.java

@@ -239,12 +239,12 @@ public class PubSubMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEng
     }
 
     @Override
-    public TbQueueProducer<TbProtoQueueMsg<ToHousekeeperServiceMsg>> createHousekeeperDelayedMsgProducer() {
+    public TbQueueProducer<TbProtoQueueMsg<ToHousekeeperServiceMsg>> createHousekeeperReprocessingMsgProducer() {
         return null;
     }
 
     @Override
-    public TbQueueConsumer<TbProtoQueueMsg<ToHousekeeperServiceMsg>> createHousekeeperDelayedMsgConsumer() {
+    public TbQueueConsumer<TbProtoQueueMsg<ToHousekeeperServiceMsg>> createHousekeeperReprocessingMsgConsumer() {
         return null;
     }
 

common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTbCoreQueueFactory.java

@@ -210,12 +210,12 @@ public class PubSubTbCoreQueueFactory implements TbCoreQueueFactory {
     }
 
     @Override
-    public TbQueueProducer<TbProtoQueueMsg<ToHousekeeperServiceMsg>> createHousekeeperDelayedMsgProducer() {
+    public TbQueueProducer<TbProtoQueueMsg<ToHousekeeperServiceMsg>> createHousekeeperReprocessingMsgProducer() {
         return null;
     }
 
     @Override
-    public TbQueueConsumer<TbProtoQueueMsg<ToHousekeeperServiceMsg>> createHousekeeperDelayedMsgConsumer() {
+    public TbQueueConsumer<TbProtoQueueMsg<ToHousekeeperServiceMsg>> createHousekeeperReprocessingMsgConsumer() {
         return null;
     }
 

common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqMonolithQueueFactory.java

@@ -236,12 +236,12 @@ public class RabbitMqMonolithQueueFactory implements TbCoreQueueFactory, TbRuleE
     }
 
     @Override
-    public TbQueueProducer<TbProtoQueueMsg<ToHousekeeperServiceMsg>> createHousekeeperDelayedMsgProducer() {
+    public TbQueueProducer<TbProtoQueueMsg<ToHousekeeperServiceMsg>> createHousekeeperReprocessingMsgProducer() {
         return null;
     }
 
     @Override
-    public TbQueueConsumer<TbProtoQueueMsg<ToHousekeeperServiceMsg>> createHousekeeperDelayedMsgConsumer() {
+    public TbQueueConsumer<TbProtoQueueMsg<ToHousekeeperServiceMsg>> createHousekeeperReprocessingMsgConsumer() {
         return null;
     }
 

common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqTbCoreQueueFactory.java

@@ -209,12 +209,12 @@ public class RabbitMqTbCoreQueueFactory implements TbCoreQueueFactory {
     }
 
     @Override
-    public TbQueueProducer<TbProtoQueueMsg<TransportProtos.ToHousekeeperServiceMsg>> createHousekeeperDelayedMsgProducer() {
+    public TbQueueProducer<TbProtoQueueMsg<TransportProtos.ToHousekeeperServiceMsg>> createHousekeeperReprocessingMsgProducer() {
         return null;
     }
 
     @Override
-    public TbQueueConsumer<TbProtoQueueMsg<TransportProtos.ToHousekeeperServiceMsg>> createHousekeeperDelayedMsgConsumer() {
+    public TbQueueConsumer<TbProtoQueueMsg<TransportProtos.ToHousekeeperServiceMsg>> createHousekeeperReprocessingMsgConsumer() {
         return null;
     }
 

common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusMonolithQueueFactory.java

@@ -235,12 +235,12 @@ public class ServiceBusMonolithQueueFactory implements TbCoreQueueFactory, TbRul
     }
 
     @Override
-    public TbQueueProducer<TbProtoQueueMsg<ToHousekeeperServiceMsg>> createHousekeeperDelayedMsgProducer() {
+    public TbQueueProducer<TbProtoQueueMsg<ToHousekeeperServiceMsg>> createHousekeeperReprocessingMsgProducer() {
         return null;
     }
 
     @Override
-    public TbQueueConsumer<TbProtoQueueMsg<ToHousekeeperServiceMsg>> createHousekeeperDelayedMsgConsumer() {
+    public TbQueueConsumer<TbProtoQueueMsg<ToHousekeeperServiceMsg>> createHousekeeperReprocessingMsgConsumer() {
         return null;
     }
 

common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusTbCoreQueueFactory.java

@@ -210,12 +210,12 @@ public class ServiceBusTbCoreQueueFactory implements TbCoreQueueFactory {
     }
 
     @Override
-    public TbQueueProducer<TbProtoQueueMsg<ToHousekeeperServiceMsg>> createHousekeeperDelayedMsgProducer() {
+    public TbQueueProducer<TbProtoQueueMsg<ToHousekeeperServiceMsg>> createHousekeeperReprocessingMsgProducer() {
         return null;
     }
 
     @Override
-    public TbQueueConsumer<TbProtoQueueMsg<ToHousekeeperServiceMsg>> createHousekeeperDelayedMsgConsumer() {
+    public TbQueueConsumer<TbProtoQueueMsg<ToHousekeeperServiceMsg>> createHousekeeperReprocessingMsgConsumer() {
         return null;
     }
 

common/queue/src/main/java/org/thingsboard/server/queue/provider/TbCoreQueueFactory.java

@@ -16,7 +16,6 @@
 package org.thingsboard.server.queue.provider;
 
 import org.thingsboard.server.gen.js.JsInvokeProtos;
-import org.thingsboard.server.gen.transport.TransportProtos;
 import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg;
 import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg;
 import org.thingsboard.server.gen.transport.TransportProtos.ToHousekeeperServiceMsg;
@@ -137,8 +136,8 @@ public interface TbCoreQueueFactory extends TbUsageStatsClientQueueFactory {
 
     TbQueueConsumer<TbProtoQueueMsg<ToHousekeeperServiceMsg>> createHousekeeperMsgConsumer();
 
-    TbQueueProducer<TbProtoQueueMsg<ToHousekeeperServiceMsg>> createHousekeeperDelayedMsgProducer();
+    TbQueueProducer<TbProtoQueueMsg<ToHousekeeperServiceMsg>> createHousekeeperReprocessingMsgProducer();
 
-    TbQueueConsumer<TbProtoQueueMsg<ToHousekeeperServiceMsg>> createHousekeeperDelayedMsgConsumer();
+    TbQueueConsumer<TbProtoQueueMsg<ToHousekeeperServiceMsg>> createHousekeeperReprocessingMsgConsumer();
 
 }

common/queue/src/main/java/org/thingsboard/server/queue/provider/TbCoreQueueProducerProvider.java

@@ -16,7 +16,6 @@
 package org.thingsboard.server.queue.provider;
 
 import org.springframework.stereotype.Service;
-import org.thingsboard.server.gen.transport.TransportProtos;
 import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg;
 import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg;
 import org.thingsboard.server.gen.transport.TransportProtos.ToHousekeeperServiceMsg;
@@ -44,7 +43,7 @@ public class TbCoreQueueProducerProvider implements TbQueueProducerProvider {
     private TbQueueProducer<TbProtoQueueMsg<ToUsageStatsServiceMsg>> toUsageStats;
     private TbQueueProducer<TbProtoQueueMsg<ToVersionControlServiceMsg>> toVersionControl;
     private TbQueueProducer<TbProtoQueueMsg<ToHousekeeperServiceMsg>> toHousekeeper;
-    private TbQueueProducer<TbProtoQueueMsg<ToHousekeeperServiceMsg>> toHousekeeperDelayed;
+    private TbQueueProducer<TbProtoQueueMsg<ToHousekeeperServiceMsg>> toHousekeeperReprocessing;
 
     public TbCoreQueueProducerProvider(TbCoreQueueFactory tbQueueProvider) {
         this.tbQueueProvider = tbQueueProvider;
@@ -60,7 +59,7 @@ public class TbCoreQueueProducerProvider implements TbQueueProducerProvider {
         this.toUsageStats = tbQueueProvider.createToUsageStatsServiceMsgProducer();
         this.toVersionControl = tbQueueProvider.createVersionControlMsgProducer();
         this.toHousekeeper = tbQueueProvider.createHousekeeperMsgProducer();
-        this.toHousekeeperDelayed = tbQueueProvider.createHousekeeperDelayedMsgProducer();
+        this.toHousekeeperReprocessing = tbQueueProvider.createHousekeeperReprocessingMsgProducer();
     }
 
     @Override
@@ -104,8 +103,8 @@ public class TbCoreQueueProducerProvider implements TbQueueProducerProvider {
     }
 
     @Override
-    public TbQueueProducer<TbProtoQueueMsg<ToHousekeeperServiceMsg>> getHousekeeperDelayedMsgProducer() {
+    public TbQueueProducer<TbProtoQueueMsg<ToHousekeeperServiceMsg>> getHousekeeperReprocessingMsgProducer() {
-        return toHousekeeperDelayed;
+        return toHousekeeperReprocessing;
     }
 
 }

common/queue/src/main/java/org/thingsboard/server/queue/provider/TbQueueProducerProvider.java

@@ -82,6 +82,6 @@ public interface TbQueueProducerProvider {
 
     TbQueueProducer<TbProtoQueueMsg<ToHousekeeperServiceMsg>> getHousekeeperMsgProducer();
 
-    TbQueueProducer<TbProtoQueueMsg<ToHousekeeperServiceMsg>> getHousekeeperDelayedMsgProducer();
+    TbQueueProducer<TbProtoQueueMsg<ToHousekeeperServiceMsg>> getHousekeeperReprocessingMsgProducer();
 
 }

common/queue/src/main/java/org/thingsboard/server/queue/provider/TbRuleEngineProducerProvider.java

@@ -96,7 +96,7 @@ public class TbRuleEngineProducerProvider implements TbQueueProducerProvider {
     }
 
     @Override
-    public TbQueueProducer<TbProtoQueueMsg<TransportProtos.ToHousekeeperServiceMsg>> getHousekeeperDelayedMsgProducer() {
+    public TbQueueProducer<TbProtoQueueMsg<TransportProtos.ToHousekeeperServiceMsg>> getHousekeeperReprocessingMsgProducer() {
         throw new RuntimeException("Not Implemented! Should not be used by Rule Engine!");
     }
 

common/queue/src/main/java/org/thingsboard/server/queue/provider/TbTransportQueueProducerProvider.java

@@ -92,7 +92,7 @@ public class TbTransportQueueProducerProvider implements TbQueueProducerProvider
     }
 
     @Override
-    public TbQueueProducer<TbProtoQueueMsg<TransportProtos.ToHousekeeperServiceMsg>> getHousekeeperDelayedMsgProducer() {
+    public TbQueueProducer<TbProtoQueueMsg<TransportProtos.ToHousekeeperServiceMsg>> getHousekeeperReprocessingMsgProducer() {
         throw new RuntimeException("Not Implemented! Should not be used by Transport!");
     }
 }

common/queue/src/main/java/org/thingsboard/server/queue/provider/TbVersionControlProducerProvider.java

@@ -88,7 +88,7 @@ public class TbVersionControlProducerProvider implements TbQueueProducerProvider
     }
 
     @Override
-    public TbQueueProducer<TbProtoQueueMsg<TransportProtos.ToHousekeeperServiceMsg>> getHousekeeperDelayedMsgProducer() {
+    public TbQueueProducer<TbProtoQueueMsg<TransportProtos.ToHousekeeperServiceMsg>> getHousekeeperReprocessingMsgProducer() {
         throw new RuntimeException("Not Implemented! Should not be used by Version Control Service!");
     }
 }

common/queue/src/main/java/org/thingsboard/server/queue/settings/TbQueueCoreSettings.java

@@ -38,7 +38,7 @@ public class TbQueueCoreSettings {
     private String housekeeperTopic;
 
     @Value("${queue.core.housekeeper.reprocessing-topic:tb_housekeeper.reprocessing}")
-    private String housekeeperDelayedTopic;
+    private String housekeeperReprocessingTopic;
 
     @Value("${queue.core.partitions}")
     private int partitions;