Merge pull request #10201 from thingsboard/feature/housekeeper

Housekeeping service

application/src/main/java/org/thingsboard/server/install/ThingsboardInstallService.java

@@ -110,7 +110,7 @@ public class ThingsboardInstallService {
                             log.info("Upgrading ThingsBoard from version 3.5.1 to 3.6.0 ...");
                             databaseEntitiesUpgradeService.upgradeDatabase("3.5.1");
                             dataUpdateService.updateData("3.5.1");
-                            systemDataLoaderService.updateDefaultNotificationConfigs();
+                            systemDataLoaderService.updateDefaultNotificationConfigs(true);
                         case "3.6.0":
                             log.info("Upgrading ThingsBoard from version 3.6.0 to 3.6.1 ...");
                             databaseEntitiesUpgradeService.upgradeDatabase("3.6.0");
@@ -126,13 +126,14 @@ public class ThingsboardInstallService {
                         case "3.6.2":
                             log.info("Upgrading ThingsBoard from version 3.6.2 to 3.6.3 ...");
                             databaseEntitiesUpgradeService.upgradeDatabase("3.6.2");
-                            systemDataLoaderService.updateDefaultNotificationConfigs();
+                            systemDataLoaderService.updateDefaultNotificationConfigs(true);
                         case "3.6.3":
                             log.info("Upgrading ThingsBoard from version 3.6.3 to 3.6.4 ...");
                             databaseEntitiesUpgradeService.upgradeDatabase("3.6.3");
                         case "3.6.4":
                             log.info("Upgrading ThingsBoard from version 3.6.4 to 3.7.0 ...");
                             databaseEntitiesUpgradeService.upgradeDatabase("3.6.4");
+                            systemDataLoaderService.updateDefaultNotificationConfigs(false);
                             //TODO DON'T FORGET to update switch statement in the CacheCleanupService if you need to clear the cache
                             break;
                         default:

application/src/main/java/org/thingsboard/server/service/edge/EdgeEventSourcingListener.java

@@ -33,6 +33,7 @@ import org.thingsboard.server.common.data.alarm.AlarmComment;
 import org.thingsboard.server.common.data.audit.ActionType;
 import org.thingsboard.server.common.data.edge.EdgeEventActionType;
 import org.thingsboard.server.common.data.edge.EdgeEventType;
+import org.thingsboard.server.common.data.id.TenantId;
 import org.thingsboard.server.common.data.oauth2.OAuth2Info;
 import org.thingsboard.server.common.data.relation.EntityRelation;
 import org.thingsboard.server.common.data.relation.RelationTypeGroup;
@@ -44,6 +45,7 @@ import org.thingsboard.server.dao.eventsourcing.ActionEntityEvent;
 import org.thingsboard.server.dao.eventsourcing.DeleteEntityEvent;
 import org.thingsboard.server.dao.eventsourcing.RelationActionEvent;
 import org.thingsboard.server.dao.eventsourcing.SaveEntityEvent;
+import org.thingsboard.server.dao.tenant.TenantService;
 import org.thingsboard.server.dao.user.UserServiceImpl;
 
 import javax.annotation.PostConstruct;
@@ -69,6 +71,7 @@ public class EdgeEventSourcingListener {
 
     private final TbClusterService tbClusterService;
     private final EdgeSynchronizationManager edgeSynchronizationManager;
+    private final TenantService tenantService;
 
     @PostConstruct
     public void init() {
@@ -95,19 +98,24 @@ public class EdgeEventSourcingListener {
 
     @TransactionalEventListener(fallbackExecution = true)
     public void handleEvent(DeleteEntityEvent<?> event) {
+        TenantId tenantId = event.getTenantId();
         EntityType entityType = event.getEntityId().getEntityType();
+        if (!tenantId.isSysTenantId() && !tenantService.tenantExists(tenantId)) {
+            log.debug("[{}] Ignoring DeleteEntityEvent because tenant does not exist: {}", tenantId, event);
+            return;
+        }
         try {
             if (EntityType.EDGE.equals(entityType) || EntityType.TENANT.equals(entityType)) {
                 return;
             }
-            log.trace("[{}] DeleteEntityEvent called: {}", event.getTenantId(), event);
+            log.trace("[{}] DeleteEntityEvent called: {}", tenantId, event);
             EdgeEventType type = getEdgeEventTypeForEntityEvent(event.getEntity());
             EdgeEventActionType actionType = getEdgeEventActionTypeForEntityEvent(event.getEntity());
-            tbClusterService.sendNotificationMsgToEdge(event.getTenantId(), null, event.getEntityId(),
+            tbClusterService.sendNotificationMsgToEdge(tenantId, null, event.getEntityId(),
                     JacksonUtil.toString(event.getEntity()), type, actionType,
                     edgeSynchronizationManager.getEdgeId().get());
         } catch (Exception e) {
-            log.error("[{}] failed to process DeleteEntityEvent: {}", event.getTenantId(), event, e);
+            log.error("[{}] failed to process DeleteEntityEvent: {}", tenantId, event, e);
         }
     }
 

application/src/main/java/org/thingsboard/server/service/entitiy/AbstractTbEntityService.java

@@ -27,12 +27,8 @@ import org.thingsboard.server.common.data.EntityType;
 import org.thingsboard.server.common.data.User;
 import org.thingsboard.server.common.data.exception.ThingsboardErrorCode;
 import org.thingsboard.server.common.data.exception.ThingsboardException;
-import org.thingsboard.server.common.data.id.AlarmId;
 import org.thingsboard.server.common.data.id.EntityId;
 import org.thingsboard.server.common.data.id.EntityIdFactory;
-import org.thingsboard.server.common.data.id.TenantId;
-import org.thingsboard.server.common.data.page.PageData;
-import org.thingsboard.server.common.data.page.TimePageLink;
 import org.thingsboard.server.dao.alarm.AlarmService;
 import org.thingsboard.server.dao.customer.CustomerService;
 import org.thingsboard.server.dao.edge.EdgeService;
@@ -71,13 +67,6 @@ public abstract class AbstractTbEntityService {
     @Lazy
     private EntitiesVersionControlService vcService;
 
-    protected void removeAlarmsByOriginatorId(TenantId tenantId, EntityId entityId) {
-        PageData<AlarmId> alarms =
-                alarmService.findAlarmIdsByOriginatorId(tenantId, entityId, new TimePageLink(Integer.MAX_VALUE));
-
-        alarms.getData().forEach(alarmId -> alarmService.delAlarm(tenantId, alarmId));
-    }
-
     protected <T> T checkNotNull(T reference) throws ThingsboardException {
         return checkNotNull(reference, "Requested item wasn't found!");
     }

application/src/main/java/org/thingsboard/server/service/entitiy/EntityStateSourcingListener.java

@@ -51,6 +51,7 @@ import org.thingsboard.server.common.msg.rule.engine.DeviceCredentialsUpdateNoti
 import org.thingsboard.server.dao.eventsourcing.ActionEntityEvent;
 import org.thingsboard.server.dao.eventsourcing.DeleteEntityEvent;
 import org.thingsboard.server.dao.eventsourcing.SaveEntityEvent;
+import org.thingsboard.server.dao.tenant.TenantService;
 
 import javax.annotation.PostConstruct;
 import java.util.Set;
@@ -61,6 +62,7 @@ import java.util.Set;
 public class EntityStateSourcingListener {
 
     private final TbClusterService tbClusterService;
+    private final TenantService tenantService;
 
     @PostConstruct
     public void init() {
@@ -121,6 +123,10 @@ public class EntityStateSourcingListener {
         TenantId tenantId = event.getTenantId();
         EntityId entityId = event.getEntityId();
         EntityType entityType = entityId.getEntityType();
+        if (!tenantId.isSysTenantId() && entityType != EntityType.TENANT  && !tenantService.tenantExists(tenantId)) {
+            log.debug("[{}] Ignoring DeleteEntityEvent because tenant does not exist: {}", tenantId, event);
+            return;
+        }
         log.debug("[{}][{}][{}] Handling entity deletion event: {}", tenantId, entityType, entityId, event);
 
         switch (entityType) {

application/src/main/java/org/thingsboard/server/service/entitiy/alarm/DefaultTbAlarmService.java

@@ -176,16 +176,16 @@ public class DefaultTbAlarmService extends AbstractTbEntityService implements Tb
     }
 
     @Override
-    public List<AlarmId> unassignDeletedUserAlarms(TenantId tenantId, User user, long unassignTs) {
+    public List<AlarmId> unassignDeletedUserAlarms(TenantId tenantId, UserId userId, String userTitle, long unassignTs) {
         List<AlarmId> totalAlarmIds = new ArrayList<>();
         PageLink pageLink = new PageLink(100, 0, null, new SortOrder("id", SortOrder.Direction.ASC));
         while (true) {
-            PageData<AlarmId> pageData = alarmService.findAlarmIdsByAssigneeId(user.getTenantId(), user.getId(), pageLink);
+            PageData<AlarmId> pageData = alarmService.findAlarmIdsByAssigneeId(tenantId, userId, pageLink);
             List<AlarmId> alarmIds = pageData.getData();
             if (alarmIds.isEmpty()) {
                 break;
             }
-            processAlarmsUnassignment(tenantId, user, alarmIds, unassignTs);
+            processAlarmsUnassignment(tenantId, userId, userTitle, alarmIds, unassignTs);
             totalAlarmIds.addAll(alarmIds);
             pageLink = pageLink.nextPageLink();
         }
@@ -204,16 +204,16 @@ public class DefaultTbAlarmService extends AbstractTbEntityService implements Tb
         return ts > 0 ? ts : System.currentTimeMillis();
     }
 
-    private void processAlarmsUnassignment(TenantId tenantId, User user, List<AlarmId> alarmIds, long unassignTs) {
+    private void processAlarmsUnassignment(TenantId tenantId, UserId userId, String userTitle, List<AlarmId> alarmIds, long unassignTs) {
         for (AlarmId alarmId : alarmIds) {
-            log.trace("[{}] Unassigning alarm {} userId {}", tenantId, alarmId, user.getId());
-            AlarmApiCallResult result = alarmSubscriptionService.unassignAlarm(user.getTenantId(), alarmId, unassignTs);
+            log.trace("[{}] Unassigning alarm {} userId {}", tenantId, alarmId, userId);
+            AlarmApiCallResult result = alarmSubscriptionService.unassignAlarm(tenantId, alarmId, unassignTs);
             if (!result.isSuccessful()) {
-                log.error("[{}] Cannot unassign alarm {} userId {}", tenantId, alarmId, user.getId());
+                log.error("[{}] Cannot unassign alarm {} userId {}", tenantId, alarmId, userId);
                 continue;
             }
             if (result.isModified()) {
-                String comment = String.format("Alarm was unassigned because user %s - was deleted", user.getTitle());
+                String comment = String.format("Alarm was unassigned because user %s - was deleted", userTitle);
                 addSystemAlarmComment(result.getAlarm(), null, "ASSIGN", comment);
                 logEntityActionService.logEntityAction(result.getAlarm().getTenantId(), result.getAlarm().getOriginator(), result.getAlarm(), result.getAlarm().getCustomerId(), ActionType.ALARM_UNASSIGNED, null);
             }

application/src/main/java/org/thingsboard/server/service/entitiy/alarm/TbAlarmService.java

@@ -41,7 +41,7 @@ public interface TbAlarmService {
 
     AlarmInfo unassign(Alarm alarm, long unassignTs, User user) throws ThingsboardException;
 
-    List<AlarmId> unassignDeletedUserAlarms(TenantId tenantId, User user, long unassignTs);
+    List<AlarmId> unassignDeletedUserAlarms(TenantId tenantId, UserId userId, String userTitle, long unassignTs);
 
     Boolean delete(Alarm alarm, User user);
 }

application/src/main/java/org/thingsboard/server/service/entitiy/asset/DefaultTbAssetService.java

@@ -61,7 +61,6 @@ public class DefaultTbAssetService extends AbstractTbEntityService implements Tb
         TenantId tenantId = asset.getTenantId();
         AssetId assetId = asset.getId();
         try {
-            removeAlarmsByOriginatorId(tenantId, assetId);
             assetService.deleteAsset(tenantId, assetId);
             logEntityActionService.logEntityAction(tenantId, assetId, asset, asset.getCustomerId(), actionType, user, assetId.toString());
         } catch (Exception e) {

application/src/main/java/org/thingsboard/server/service/entitiy/device/DefaultTbDeviceService.java

@@ -94,7 +94,6 @@ public class DefaultTbDeviceService extends AbstractTbEntityService implements T
         TenantId tenantId = device.getTenantId();
         DeviceId deviceId = device.getId();
         try {
-            removeAlarmsByOriginatorId(tenantId, deviceId);
             deviceService.deleteDevice(tenantId, deviceId);
             logEntityActionService.logEntityAction(tenantId, deviceId, device, device.getCustomerId(), actionType,
                     user, deviceId.toString());

application/src/main/java/org/thingsboard/server/service/housekeeper/HousekeeperReprocessingService.java

@@ -0,0 +1,118 @@
+/**
+ * Copyright © 2016-2024 The Thingsboard Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.thingsboard.server.service.housekeeper;
+
+import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.lang3.exception.ExceptionUtils;
+import org.springframework.context.annotation.Lazy;
+import org.springframework.stereotype.Service;
+import org.thingsboard.common.util.ThingsBoardThreadFactory;
+import org.thingsboard.server.common.data.StringUtils;
+import org.thingsboard.server.common.msg.queue.TopicPartitionInfo;
+import org.thingsboard.server.gen.transport.TransportProtos.HousekeeperTaskProto;
+import org.thingsboard.server.gen.transport.TransportProtos.ToHousekeeperServiceMsg;
+import org.thingsboard.server.queue.TbQueueConsumer;
+import org.thingsboard.server.queue.TbQueueProducer;
+import org.thingsboard.server.queue.common.TbProtoQueueMsg;
+import org.thingsboard.server.queue.housekeeper.HousekeeperConfig;
+import org.thingsboard.server.queue.provider.TbCoreQueueFactory;
+import org.thingsboard.server.queue.util.AfterStartUp;
+import org.thingsboard.server.queue.util.TbCoreComponent;
+import org.thingsboard.server.service.queue.consumer.QueueConsumerManager;
+
+import javax.annotation.PreDestroy;
+import java.util.LinkedHashSet;
+import java.util.List;
+import java.util.Set;
+import java.util.UUID;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+
+@TbCoreComponent
+@Service
+@Slf4j
+public class HousekeeperReprocessingService {
+
+    private final HousekeeperConfig config;
+    private final HousekeeperService housekeeperService;
+    private final QueueConsumerManager<TbProtoQueueMsg<ToHousekeeperServiceMsg>> consumer;
+    private final TbQueueProducer<TbProtoQueueMsg<ToHousekeeperServiceMsg>> producer;
+    private final TopicPartitionInfo submitTpi;
+
+    private final ExecutorService consumerExecutor = Executors.newSingleThreadExecutor(ThingsBoardThreadFactory.forName("housekeeper-reprocessing-consumer"));
+
+    public HousekeeperReprocessingService(HousekeeperConfig config,
+                                          @Lazy HousekeeperService housekeeperService,
+                                          TbCoreQueueFactory queueFactory) {
+        this.config = config;
+        this.housekeeperService = housekeeperService;
+        this.consumer = QueueConsumerManager.<TbProtoQueueMsg<ToHousekeeperServiceMsg>>builder()
+                .name("Housekeeper reprocessing")
+                .msgPackProcessor(this::processMsgs)
+                .pollInterval(config.getPollInterval())
+                .consumerCreator(queueFactory::createHousekeeperReprocessingMsgConsumer)
+                .consumerExecutor(consumerExecutor)
+                .build();
+        this.producer = queueFactory.createHousekeeperReprocessingMsgProducer();
+        this.submitTpi = TopicPartitionInfo.builder().topic(producer.getDefaultTopic()).build();
+    }
+
+    @AfterStartUp(order = AfterStartUp.REGULAR_SERVICE)
+    public void afterStartUp() {
+        consumer.subscribe(); // Kafka topic for tasks reprocessing has only 1 partition, so only one TB Core will reprocess tasks
+        consumer.launch();
+    }
+
+    private void processMsgs(List<TbProtoQueueMsg<ToHousekeeperServiceMsg>> msgs, TbQueueConsumer<TbProtoQueueMsg<ToHousekeeperServiceMsg>> consumer) throws Exception {
+        Thread.sleep(config.getTaskReprocessingDelay());
+
+        for (TbProtoQueueMsg<ToHousekeeperServiceMsg> msg : msgs) {
+            log.trace("Reprocessing task: {}", msg);
+            try {
+                housekeeperService.processTask(msg.getValue());
+            } catch (InterruptedException e) {
+                return;
+            } catch (Throwable e) {
+                log.error("Unexpected error during message reprocessing [{}]", msg, e);
+                submitForReprocessing(msg.getValue(), e);
+            }
+        }
+        consumer.commit();
+    }
+
+    public void submitForReprocessing(ToHousekeeperServiceMsg msg, Throwable error) {
+        HousekeeperTaskProto task = msg.getTask();
+        Set<String> errors = new LinkedHashSet<>(task.getErrorsList());
+        errors.add(StringUtils.truncate(ExceptionUtils.getStackTrace(error), 1024));
+        msg = msg.toBuilder()
+                .setTask(task.toBuilder()
+                        .setAttempt(task.getAttempt() + 1)
+                        .clearErrors().addAllErrors(errors)
+                        .build())
+                .build();
+
+        log.trace("Submitting for reprocessing: {}", msg);
+        producer.send(submitTpi, new TbProtoQueueMsg<>(UUID.randomUUID(), msg), null);
+    }
+
+    @PreDestroy
+    private void stop() {
+        consumer.stop();
+        consumerExecutor.shutdownNow();
+        log.info("Stopped Housekeeper reprocessing service");
+    }
+
+}

application/src/main/java/org/thingsboard/server/service/housekeeper/HousekeeperService.java

@@ -0,0 +1,166 @@
+/**
+ * Copyright © 2016-2024 The Thingsboard Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.thingsboard.server.service.housekeeper;
+
+import lombok.extern.slf4j.Slf4j;
+import org.springframework.context.annotation.Lazy;
+import org.springframework.stereotype.Service;
+import org.thingsboard.common.util.JacksonUtil;
+import org.thingsboard.common.util.ThingsBoardThreadFactory;
+import org.thingsboard.server.common.data.housekeeper.HousekeeperTask;
+import org.thingsboard.server.common.data.housekeeper.HousekeeperTaskType;
+import org.thingsboard.server.common.data.notification.rule.trigger.TaskProcessingFailureTrigger;
+import org.thingsboard.server.common.msg.notification.NotificationRuleProcessor;
+import org.thingsboard.server.gen.transport.TransportProtos.ToHousekeeperServiceMsg;
+import org.thingsboard.server.queue.TbQueueConsumer;
+import org.thingsboard.server.queue.common.TbProtoQueueMsg;
+import org.thingsboard.server.queue.housekeeper.HousekeeperConfig;
+import org.thingsboard.server.queue.provider.TbCoreQueueFactory;
+import org.thingsboard.server.queue.util.AfterStartUp;
+import org.thingsboard.server.queue.util.TbCoreComponent;
+import org.thingsboard.server.service.housekeeper.processor.HousekeeperTaskProcessor;
+import org.thingsboard.server.service.housekeeper.stats.HousekeeperStatsService;
+import org.thingsboard.server.service.queue.consumer.QueueConsumerManager;
+
+import javax.annotation.PreDestroy;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
+import java.util.stream.Collectors;
+
+@TbCoreComponent
+@Service
+@Slf4j
+public class HousekeeperService {
+
+    private final Map<HousekeeperTaskType, HousekeeperTaskProcessor<?>> taskProcessors;
+
+    private final HousekeeperConfig config;
+    private final HousekeeperReprocessingService reprocessingService;
+    private final Optional<HousekeeperStatsService> statsService;
+    private final NotificationRuleProcessor notificationRuleProcessor;
+    private final QueueConsumerManager<TbProtoQueueMsg<ToHousekeeperServiceMsg>> consumer;
+
+    private final ExecutorService consumerExecutor = Executors.newSingleThreadExecutor(ThingsBoardThreadFactory.forName("housekeeper-consumer"));
+    private final ExecutorService taskExecutor = Executors.newSingleThreadExecutor(ThingsBoardThreadFactory.forName("housekeeper-task-processor"));
+
+    public HousekeeperService(HousekeeperConfig config,
+                              HousekeeperReprocessingService reprocessingService,
+                              TbCoreQueueFactory queueFactory,
+                              Optional<HousekeeperStatsService> statsService,
+                              NotificationRuleProcessor notificationRuleProcessor,
+                              @Lazy List<HousekeeperTaskProcessor<?>> taskProcessors) {
+        this.config = config;
+        this.reprocessingService = reprocessingService;
+        this.statsService = statsService;
+        this.notificationRuleProcessor = notificationRuleProcessor;
+        this.consumer = QueueConsumerManager.<TbProtoQueueMsg<ToHousekeeperServiceMsg>>builder()
+                .name("Housekeeper")
+                .msgPackProcessor(this::processMsgs)
+                .pollInterval(config.getPollInterval())
+                .consumerCreator(queueFactory::createHousekeeperMsgConsumer)
+                .consumerExecutor(consumerExecutor)
+                .build();
+        this.taskProcessors = taskProcessors.stream().collect(Collectors.toMap(HousekeeperTaskProcessor::getTaskType, p -> p));
+    }
+
+    @AfterStartUp(order = AfterStartUp.REGULAR_SERVICE)
+    public void afterStartUp() {
+        consumer.subscribe();
+        consumer.launch();
+    }
+
+    private void processMsgs(List<TbProtoQueueMsg<ToHousekeeperServiceMsg>> msgs, TbQueueConsumer<TbProtoQueueMsg<ToHousekeeperServiceMsg>> consumer) {
+        for (TbProtoQueueMsg<ToHousekeeperServiceMsg> msg : msgs) {
+            log.trace("Processing task: {}", msg);
+            try {
+                processTask(msg.getValue());
+            } catch (InterruptedException e) {
+                return;
+            } catch (Throwable e) {
+                log.error("Unexpected error during message processing [{}]", msg, e);
+                reprocessingService.submitForReprocessing(msg.getValue(), e);
+            }
+        }
+        consumer.commit();
+    }
+
+    @SuppressWarnings("unchecked")
+    protected <T extends HousekeeperTask> void processTask(ToHousekeeperServiceMsg msg) throws Exception {
+        HousekeeperTask task = JacksonUtil.fromString(msg.getTask().getValue(), HousekeeperTask.class);
+        HousekeeperTaskType taskType = task.getTaskType();
+        if (config.getDisabledTaskTypes().contains(taskType)) {
+            log.debug("Task type {} is disabled, ignoring {}", taskType, task);
+            return;
+        }
+        HousekeeperTaskProcessor<T> taskProcessor = (HousekeeperTaskProcessor<T>) taskProcessors.get(taskType);
+        if (taskProcessor == null) {
+            throw new IllegalArgumentException("Unsupported task type " + taskType);
+        }
+
+        try {
+            long startTs = System.currentTimeMillis();
+            Future<Object> future = taskExecutor.submit(() -> {
+                taskProcessor.process((T) task);
+                return null;
+            });
+            future.get(config.getTaskProcessingTimeout(), TimeUnit.MILLISECONDS);
+
+            long timing = System.currentTimeMillis() - startTs;
+            if (log.isDebugEnabled()) {
+                log.debug("[{}] Processed {} in {} ms (attempt {})", task.getTenantId(), task.getDescription(), timing, msg.getTask().getAttempt());
+            }
+            statsService.ifPresent(statsService -> statsService.reportProcessed(taskType, msg, timing));
+        } catch (InterruptedException e) {
+            throw e;
+        } catch (Throwable e) {
+            Throwable error = e;
+            if (e instanceof ExecutionException) {
+                error = e.getCause();
+            } else if (e instanceof TimeoutException) {
+                error = new TimeoutException("Timeout after " + config.getTaskProcessingTimeout() + " seconds");
+            }
+
+            if (msg.getTask().getAttempt() < config.getMaxReprocessingAttempts()) {
+                log.warn("[{}] Failed to process {} (attempt {}), submitting for reprocessing",
+                        task.getTenantId(), task.getDescription(), msg.getTask().getAttempt(), error);
+                reprocessingService.submitForReprocessing(msg, error);
+            } else {
+                log.error("[{}] Failed to process task in {} attempts: {}", task.getTenantId(), msg.getTask().getAttempt(), msg, e);
+                notificationRuleProcessor.process(TaskProcessingFailureTrigger.builder()
+                        .task(task)
+                        .error(error)
+                        .attempt(msg.getTask().getAttempt())
+                        .build());
+            }
+            statsService.ifPresent(statsService -> statsService.reportFailure(taskType, msg));
+        }
+    }
+
+    @PreDestroy
+    private void stop() {
+        consumer.stop();
+        consumerExecutor.shutdownNow();
+        log.info("Stopped Housekeeper service");
+    }
+
+}

application/src/main/java/org/thingsboard/server/service/housekeeper/InMemoryHouseKeeperServiceService.java

@@ -1,103 +0,0 @@
-/**
- * Copyright © 2016-2024 The Thingsboard Authors
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.thingsboard.server.service.housekeeper;
-
-import com.google.common.util.concurrent.FutureCallback;
-import com.google.common.util.concurrent.Futures;
-import com.google.common.util.concurrent.ListenableFuture;
-import com.google.common.util.concurrent.ListeningExecutorService;
-import com.google.common.util.concurrent.MoreExecutors;
-import lombok.RequiredArgsConstructor;
-import lombok.extern.slf4j.Slf4j;
-import org.springframework.stereotype.Component;
-import org.springframework.transaction.event.TransactionalEventListener;
-import org.thingsboard.common.util.ThingsBoardThreadFactory;
-import org.thingsboard.server.common.data.EntityType;
-import org.thingsboard.server.common.data.User;
-import org.thingsboard.server.common.data.id.AlarmId;
-import org.thingsboard.server.common.data.id.EntityId;
-import org.thingsboard.server.common.data.id.TenantId;
-import org.thingsboard.server.dao.eventsourcing.DeleteEntityEvent;
-import org.thingsboard.server.dao.housekeeper.HouseKeeperService;
-import org.thingsboard.server.service.entitiy.alarm.TbAlarmService;
-
-import javax.annotation.PostConstruct;
-import javax.annotation.PreDestroy;
-import java.util.List;
-import java.util.concurrent.Executors;
-import java.util.concurrent.atomic.AtomicInteger;
-
-@Component
-@RequiredArgsConstructor
-@Slf4j
-public class InMemoryHouseKeeperServiceService implements HouseKeeperService {
-
-    final TbAlarmService alarmService;
-
-    ListeningExecutorService executor;
-
-    AtomicInteger queueSize = new AtomicInteger();
-    AtomicInteger totalProcessedCounter = new AtomicInteger();
-
-    @PostConstruct
-    public void init() {
-        log.debug("Starting HouseKeeper service");
-        executor = MoreExecutors.listeningDecorator(Executors.newSingleThreadExecutor(ThingsBoardThreadFactory.forName("housekeeper")));
-    }
-
-    @PreDestroy
-    public void destroy() {
-        if (executor != null) {
-            log.debug("Stopping HouseKeeper service");
-            executor.shutdown();
-        }
-    }
-
-    @TransactionalEventListener(fallbackExecution = true)
-    public void handleEvent(DeleteEntityEvent<?> event) {
-        log.trace("[{}] DeleteEntityEvent handler: {}", event.getTenantId(), event);
-        EntityId entityId = event.getEntityId();
-        if (EntityType.USER.equals(entityId.getEntityType())) {
-            unassignDeletedUserAlarms(event.getTenantId(), (User) event.getEntity(), event.getTs());
-        }
-    }
-
-    @Override
-    public ListenableFuture<List<AlarmId>> unassignDeletedUserAlarms(TenantId tenantId, User user, long unassignTs) {
-        log.debug("[{}][{}] unassignDeletedUserAlarms submitting, pending queue size: {} ", tenantId, user.getId().getId(), queueSize.get());
-        queueSize.incrementAndGet();
-        ListenableFuture<List<AlarmId>> future = executor.submit(() -> alarmService.unassignDeletedUserAlarms(tenantId, user, unassignTs));
-        Futures.addCallback(future, new FutureCallback<>() {
-            @Override
-            public void onSuccess(List<AlarmId> alarmIds) {
-                queueSize.decrementAndGet();
-                totalProcessedCounter.incrementAndGet();
-                log.debug("[{}][{}] unassignDeletedUserAlarms finished, pending queue size: {}, total processed count: {} ",
-                        tenantId, user.getId().getId(), queueSize.get(), totalProcessedCounter.get());
-            }
-
-            @Override
-            public void onFailure(Throwable throwable) {
-                queueSize.decrementAndGet();
-                totalProcessedCounter.incrementAndGet();
-                log.error("[{}][{}] unassignDeletedUserAlarms failed, pending queue size: {}, total processed count: {}",
-                        tenantId, user.getId().getId(), queueSize.get(), totalProcessedCounter.get(), throwable);
-            }
-        }, MoreExecutors.directExecutor());
-        return future;
-    }
-
-}

application/src/main/java/org/thingsboard/server/service/housekeeper/processor/AlarmsDeletionTaskProcessor.java

@@ -0,0 +1,80 @@
+/**
+ * Copyright © 2016-2024 The Thingsboard Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.thingsboard.server.service.housekeeper.processor;
+
+import lombok.RequiredArgsConstructor;
+import lombok.extern.slf4j.Slf4j;
+import org.springframework.stereotype.Component;
+import org.thingsboard.server.common.data.EntityType;
+import org.thingsboard.server.common.data.housekeeper.AlarmsDeletionHousekeeperTask;
+import org.thingsboard.server.common.data.housekeeper.HousekeeperTaskType;
+import org.thingsboard.server.common.data.id.AlarmId;
+import org.thingsboard.server.common.data.id.EntityId;
+import org.thingsboard.server.common.data.id.TenantId;
+import org.thingsboard.server.common.data.util.TbPair;
+import org.thingsboard.server.dao.alarm.AlarmService;
+
+import java.util.List;
+import java.util.UUID;
+
+@Component
+@RequiredArgsConstructor
+@Slf4j
+public class AlarmsDeletionTaskProcessor extends HousekeeperTaskProcessor<AlarmsDeletionHousekeeperTask> {
+
+    private final AlarmService alarmService;
+
+    @Override
+    public void process(AlarmsDeletionHousekeeperTask task) throws Exception {
+        EntityId entityId = task.getEntityId();
+        EntityType entityType = entityId.getEntityType();
+        TenantId tenantId = task.getTenantId();
+
+        if (entityType == EntityType.DEVICE || entityType == EntityType.ASSET) {
+            if (task.getAlarms() == null) {
+                AlarmId lastId = null;
+                long lastCreatedTime = 0;
+                while (true) {
+                    List<TbPair<UUID, Long>> alarms = alarmService.findAlarmIdsByOriginatorId(tenantId, entityId, lastCreatedTime, lastId, 128);
+                    if (alarms.isEmpty()) {
+                        break;
+                    }
+
+                    housekeeperClient.submitTask(new AlarmsDeletionHousekeeperTask(tenantId, entityId, alarms.stream().map(TbPair::getFirst).toList()));
+
+                    TbPair<UUID, Long> last = alarms.get(alarms.size() - 1);
+                    lastId = new AlarmId(last.getFirst());
+                    lastCreatedTime = last.getSecond();
+                    log.debug("[{}][{}][{}] Submitted task for deleting {} alarms", tenantId, entityType, entityId, alarms.size());
+                }
+            } else {
+                for (UUID alarmId : task.getAlarms()) {
+                    alarmService.delAlarm(tenantId, new AlarmId(alarmId));
+                }
+                log.debug("[{}][{}][{}] Deleted {} alarms", tenantId, entityType, entityId, task.getAlarms().size());
+            }
+        }
+
+        int count = alarmService.deleteEntityAlarmRecords(tenantId, entityId);
+        log.debug("[{}][{}][{}] Deleted {} entity alarms", tenantId, entityType, entityId, count);
+    }
+
+    @Override
+    public HousekeeperTaskType getTaskType() {
+        return HousekeeperTaskType.DELETE_ALARMS;
+    }
+
+}

application/src/main/java/org/thingsboard/server/service/housekeeper/processor/AlarmsUnassignTaskProcessor.java

@@ -0,0 +1,47 @@
+/**
+ * Copyright © 2016-2024 The Thingsboard Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.thingsboard.server.service.housekeeper.processor;
+
+import lombok.RequiredArgsConstructor;
+import lombok.extern.slf4j.Slf4j;
+import org.springframework.stereotype.Component;
+import org.thingsboard.server.common.data.id.AlarmId;
+import org.thingsboard.server.common.data.id.UserId;
+import org.thingsboard.server.common.data.housekeeper.HousekeeperTaskType;
+import org.thingsboard.server.common.data.housekeeper.AlarmsUnassignHousekeeperTask;
+import org.thingsboard.server.service.entitiy.alarm.TbAlarmService;
+
+import java.util.List;
+
+@Component
+@RequiredArgsConstructor
+@Slf4j
+public class AlarmsUnassignTaskProcessor extends HousekeeperTaskProcessor<AlarmsUnassignHousekeeperTask> {
+
+    private final TbAlarmService alarmService;
+
+    @Override
+    public void process(AlarmsUnassignHousekeeperTask task) throws Exception {
+        List<AlarmId> alarms = alarmService.unassignDeletedUserAlarms(task.getTenantId(), (UserId) task.getEntityId(), task.getUserTitle(), task.getTs());
+        log.debug("[{}][{}] Unassigned {} alarms", task.getTenantId(), task.getEntityId(), alarms.size());
+    }
+
+    @Override
+    public HousekeeperTaskType getTaskType() {
+        return HousekeeperTaskType.UNASSIGN_ALARMS;
+    }
+
+}

application/src/main/java/org/thingsboard/server/service/housekeeper/processor/AttributesDeletionTaskProcessor.java

@@ -0,0 +1,43 @@
+/**
+ * Copyright © 2016-2024 The Thingsboard Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.thingsboard.server.service.housekeeper.processor;
+
+import lombok.RequiredArgsConstructor;
+import lombok.extern.slf4j.Slf4j;
+import org.springframework.stereotype.Component;
+import org.thingsboard.server.dao.attributes.AttributesService;
+import org.thingsboard.server.common.data.housekeeper.HousekeeperTask;
+import org.thingsboard.server.common.data.housekeeper.HousekeeperTaskType;
+
+@Component
+@RequiredArgsConstructor
+@Slf4j
+public class AttributesDeletionTaskProcessor extends HousekeeperTaskProcessor<HousekeeperTask> {
+
+    private final AttributesService attributesService;
+
+    @Override
+    public void process(HousekeeperTask task) throws Exception {
+        int deletedCount = attributesService.removeAllByEntityId(task.getTenantId(), task.getEntityId());
+        log.debug("[{}][{}][{}] Deleted {} attributes", task.getTenantId(), task.getEntityId().getEntityType(), task.getEntityId(), deletedCount);
+    }
+
+    @Override
+    public HousekeeperTaskType getTaskType() {
+        return HousekeeperTaskType.DELETE_ATTRIBUTES;
+    }
+
+}

application/src/main/java/org/thingsboard/server/service/housekeeper/processor/EntitiesDeletionTaskProcessor.java

@@ -0,0 +1,57 @@
+/**
+ * Copyright © 2016-2024 The Thingsboard Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.thingsboard.server.service.housekeeper.processor;
+
+import lombok.RequiredArgsConstructor;
+import lombok.extern.slf4j.Slf4j;
+import org.springframework.stereotype.Component;
+import org.thingsboard.server.common.data.EntityType;
+import org.thingsboard.server.common.data.housekeeper.EntitiesDeletionHousekeeperTask;
+import org.thingsboard.server.common.data.housekeeper.HousekeeperTaskType;
+import org.thingsboard.server.common.data.id.EntityId;
+import org.thingsboard.server.common.data.id.EntityIdFactory;
+import org.thingsboard.server.common.data.id.TenantId;
+import org.thingsboard.server.dao.entity.EntityDaoService;
+import org.thingsboard.server.dao.entity.EntityServiceRegistry;
+
+import java.util.UUID;
+
+@Component
+@RequiredArgsConstructor
+@Slf4j
+public class EntitiesDeletionTaskProcessor extends HousekeeperTaskProcessor<EntitiesDeletionHousekeeperTask> {
+
+    private final EntityServiceRegistry entityServiceRegistry;
+
+    @Override
+    public void process(EntitiesDeletionHousekeeperTask task) throws Exception {
+        EntityType entityType = task.getEntityType();
+        TenantId tenantId = task.getTenantId();
+        EntityDaoService entityService = entityServiceRegistry.getServiceByEntityType(entityType);
+
+        for (UUID entityUuid : task.getEntities()) {
+            EntityId entityId = EntityIdFactory.getByTypeAndUuid(entityType, entityUuid);
+            entityService.deleteEntity(tenantId, entityId, true);
+        }
+        log.debug("[{}] Deleted {} {}s", tenantId, task.getEntities().size(), entityType.getNormalName().toLowerCase());
+    }
+
+    @Override
+    public HousekeeperTaskType getTaskType() {
+        return HousekeeperTaskType.DELETE_ENTITIES;
+    }
+
+}

application/src/main/java/org/thingsboard/server/service/housekeeper/processor/EventsDeletionTaskProcessor.java

@@ -0,0 +1,40 @@
+/**
+ * Copyright © 2016-2024 The Thingsboard Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.thingsboard.server.service.housekeeper.processor;
+
+import lombok.RequiredArgsConstructor;
+import org.springframework.stereotype.Component;
+import org.thingsboard.server.dao.event.EventService;
+import org.thingsboard.server.common.data.housekeeper.HousekeeperTask;
+import org.thingsboard.server.common.data.housekeeper.HousekeeperTaskType;
+
+@Component
+@RequiredArgsConstructor
+public class EventsDeletionTaskProcessor extends HousekeeperTaskProcessor<HousekeeperTask> {
+
+    private final EventService eventService;
+
+    @Override
+    public void process(HousekeeperTask task) throws Exception {
+        eventService.removeEvents(task.getTenantId(), task.getEntityId(), null, 0L, System.currentTimeMillis());
+    }
+
+    @Override
+    public HousekeeperTaskType getTaskType() {
+        return HousekeeperTaskType.DELETE_EVENTS;
+    }
+
+}

application/src/main/java/org/thingsboard/server/service/housekeeper/processor/HousekeeperTaskProcessor.java

@@ -0,0 +1,32 @@
+/**
+ * Copyright © 2016-2024 The Thingsboard Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.thingsboard.server.service.housekeeper.processor;
+
+import org.springframework.beans.factory.annotation.Autowired;
+import org.thingsboard.server.common.data.housekeeper.HousekeeperTask;
+import org.thingsboard.server.common.data.housekeeper.HousekeeperTaskType;
+import org.thingsboard.server.common.msg.housekeeper.HousekeeperClient;
+
+public abstract class HousekeeperTaskProcessor<T extends HousekeeperTask> {
+
+    @Autowired
+    protected HousekeeperClient housekeeperClient;
+
+    public abstract void process(T task) throws Exception;
+
+    public abstract HousekeeperTaskType getTaskType();
+
+}

application/src/main/java/org/thingsboard/server/service/housekeeper/processor/LatestTsDeletionTaskProcessor.java

@@ -0,0 +1,45 @@
+/**
+ * Copyright © 2016-2024 The Thingsboard Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.thingsboard.server.service.housekeeper.processor;
+
+import lombok.RequiredArgsConstructor;
+import lombok.extern.slf4j.Slf4j;
+import org.springframework.stereotype.Component;
+import org.thingsboard.server.common.data.housekeeper.HousekeeperTaskType;
+import org.thingsboard.server.common.data.housekeeper.LatestTsDeletionHousekeeperTask;
+import org.thingsboard.server.dao.timeseries.TimeseriesService;
+
+import java.util.List;
+
+@Component
+@RequiredArgsConstructor
+@Slf4j
+public class LatestTsDeletionTaskProcessor extends HousekeeperTaskProcessor<LatestTsDeletionHousekeeperTask> {
+
+    private final TimeseriesService timeseriesService;
+
+    @Override
+    public void process(LatestTsDeletionHousekeeperTask task) throws Exception {
+        timeseriesService.removeLatest(task.getTenantId(), task.getEntityId(), List.of(task.getKey())).get();
+        log.debug("[{}][{}][{}] Deleted latest telemetry for key '{}'", task.getTenantId(), task.getEntityId().getEntityType(), task.getEntityId(), task.getKey());
+    }
+
+    @Override
+    public HousekeeperTaskType getTaskType() {
+        return HousekeeperTaskType.DELETE_LATEST_TS;
+    }
+
+}

application/src/main/java/org/thingsboard/server/service/housekeeper/processor/TelemetryDeletionTaskProcessor.java

@@ -0,0 +1,60 @@
+/**
+ * Copyright © 2016-2024 The Thingsboard Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.thingsboard.server.service.housekeeper.processor;
+
+import lombok.RequiredArgsConstructor;
+import lombok.extern.slf4j.Slf4j;
+import org.springframework.stereotype.Component;
+import org.thingsboard.server.common.data.housekeeper.HousekeeperTask;
+import org.thingsboard.server.common.data.housekeeper.HousekeeperTaskType;
+import org.thingsboard.server.common.data.housekeeper.LatestTsDeletionHousekeeperTask;
+import org.thingsboard.server.common.data.housekeeper.TsHistoryDeletionHousekeeperTask;
+import org.thingsboard.server.common.data.id.EntityId;
+import org.thingsboard.server.common.data.id.TenantId;
+import org.thingsboard.server.dao.timeseries.TimeseriesService;
+
+import java.util.List;
+
+@Component
+@RequiredArgsConstructor
+@Slf4j
+public class TelemetryDeletionTaskProcessor extends HousekeeperTaskProcessor<HousekeeperTask> {
+
+    private final TimeseriesService timeseriesService;
+
+    @Override
+    public void process(HousekeeperTask task) throws Exception {
+        TenantId tenantId = task.getTenantId();
+        EntityId entityId = task.getEntityId();
+        List<String> keys = timeseriesService.findAllKeysByEntityIds(tenantId, List.of(entityId));
+
+        for (String key : keys) {
+            var latestTsDeletionTask = new LatestTsDeletionHousekeeperTask(tenantId, entityId, key);
+            housekeeperClient.submitTask(latestTsDeletionTask);
+
+            var tsHistoryDeletionTask = new TsHistoryDeletionHousekeeperTask(tenantId, entityId, key);
+            housekeeperClient.submitTask(tsHistoryDeletionTask);
+        }
+
+        log.trace("[{}][{}][{}] Submitted latest and ts history deletion tasks for {} keys", tenantId, entityId.getEntityType(), entityId, keys.size());
+    }
+
+    @Override
+    public HousekeeperTaskType getTaskType() {
+        return HousekeeperTaskType.DELETE_TELEMETRY;
+    }
+
+}

application/src/main/java/org/thingsboard/server/service/housekeeper/processor/TenantEntitiesDeletionTaskProcessor.java

@@ -0,0 +1,63 @@
+/**
+ * Copyright © 2016-2024 The Thingsboard Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.thingsboard.server.service.housekeeper.processor;
+
+import lombok.RequiredArgsConstructor;
+import lombok.extern.slf4j.Slf4j;
+import org.springframework.stereotype.Component;
+import org.thingsboard.server.common.data.EntityType;
+import org.thingsboard.server.common.data.housekeeper.EntitiesDeletionHousekeeperTask;
+import org.thingsboard.server.common.data.housekeeper.HousekeeperTaskType;
+import org.thingsboard.server.common.data.housekeeper.TenantEntitiesDeletionHousekeeperTask;
+import org.thingsboard.server.common.data.id.TenantId;
+import org.thingsboard.server.dao.Dao;
+import org.thingsboard.server.dao.entity.EntityDaoRegistry;
+
+import java.util.List;
+import java.util.UUID;
+
+@Component
+@RequiredArgsConstructor
+@Slf4j
+public class TenantEntitiesDeletionTaskProcessor extends HousekeeperTaskProcessor<TenantEntitiesDeletionHousekeeperTask> {
+
+    private final EntityDaoRegistry entityDaoRegistry;
+
+    @Override
+    public void process(TenantEntitiesDeletionHousekeeperTask task) throws Exception {
+        EntityType entityType = task.getEntityType();
+        TenantId tenantId = task.getTenantId();
+        Dao<?> entityDao = entityDaoRegistry.getDao(entityType);
+
+        UUID last = null;
+        while (true) {
+            List<UUID> entities = entityDao.findIdsByTenantIdAndIdOffset(tenantId, last, 128);
+            if (entities.isEmpty()) {
+                break;
+            }
+
+            housekeeperClient.submitTask(new EntitiesDeletionHousekeeperTask(tenantId, entityType, entities));
+            last = entities.get(entities.size() - 1);
+            log.debug("[{}] Submitted task for deleting {} {}s", tenantId, entities.size(), entityType.getNormalName().toLowerCase());
+        }
+    }
+
+    @Override
+    public HousekeeperTaskType getTaskType() {
+        return HousekeeperTaskType.DELETE_TENANT_ENTITIES;
+    }
+
+}

application/src/main/java/org/thingsboard/server/service/housekeeper/processor/TsHistoryDeletionTaskProcessor.java

@@ -0,0 +1,48 @@
+/**
+ * Copyright © 2016-2024 The Thingsboard Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.thingsboard.server.service.housekeeper.processor;
+
+import lombok.RequiredArgsConstructor;
+import lombok.extern.slf4j.Slf4j;
+import org.springframework.stereotype.Component;
+import org.thingsboard.server.common.data.housekeeper.HousekeeperTaskType;
+import org.thingsboard.server.common.data.housekeeper.TsHistoryDeletionHousekeeperTask;
+import org.thingsboard.server.common.data.kv.BaseDeleteTsKvQuery;
+import org.thingsboard.server.common.data.kv.DeleteTsKvQuery;
+import org.thingsboard.server.dao.timeseries.TimeseriesService;
+
+import java.util.List;
+
+@Component
+@RequiredArgsConstructor
+@Slf4j
+public class TsHistoryDeletionTaskProcessor extends HousekeeperTaskProcessor<TsHistoryDeletionHousekeeperTask> {
+
+    private final TimeseriesService timeseriesService;
+
+    @Override
+    public void process(TsHistoryDeletionHousekeeperTask task) throws Exception {
+        DeleteTsKvQuery deleteQuery = new BaseDeleteTsKvQuery(task.getKey(), 0, System.currentTimeMillis(), false, false);
+        timeseriesService.remove(task.getTenantId(), task.getEntityId(), List.of(deleteQuery)).get();
+        log.debug("[{}][{}][{}] Deleted timeseries history for key '{}'", task.getTenantId(), task.getEntityId().getEntityType(), task.getEntityId(), task.getKey());
+    }
+
+    @Override
+    public HousekeeperTaskType getTaskType() {
+        return HousekeeperTaskType.DELETE_TS_HISTORY;
+    }
+
+}

application/src/main/java/org/thingsboard/server/service/housekeeper/stats/HousekeeperStatsService.java

@@ -0,0 +1,125 @@
+/**
+ * Copyright © 2016-2024 The Thingsboard Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.thingsboard.server.service.housekeeper.stats;
+
+import lombok.Getter;
+import lombok.extern.slf4j.Slf4j;
+import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
+import org.springframework.scheduling.annotation.Scheduled;
+import org.springframework.stereotype.Service;
+import org.thingsboard.server.common.data.housekeeper.HousekeeperTaskType;
+import org.thingsboard.server.common.stats.DefaultCounter;
+import org.thingsboard.server.common.stats.StatsCounter;
+import org.thingsboard.server.common.stats.StatsFactory;
+import org.thingsboard.server.common.stats.StatsTimer;
+import org.thingsboard.server.common.stats.StatsType;
+import org.thingsboard.server.gen.transport.TransportProtos.ToHousekeeperServiceMsg;
+
+import java.util.ArrayList;
+import java.util.EnumMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.stream.Collectors;
+
+@Service
+@Slf4j
+@ConditionalOnProperty(name = "queue.core.housekeeper.stats.enabled", havingValue = "true", matchIfMissing = true)
+public class HousekeeperStatsService {
+
+    private final Map<HousekeeperTaskType, HousekeeperStats> stats = new EnumMap<>(HousekeeperTaskType.class);
+
+    public HousekeeperStatsService(StatsFactory statsFactory) {
+        for (HousekeeperTaskType taskType : HousekeeperTaskType.values()) {
+            stats.put(taskType, new HousekeeperStats(taskType, statsFactory));
+        }
+    }
+
+    @Scheduled(initialDelayString = "${queue.core.housekeeper.stats.print-interval-ms:60000}",
+            fixedDelayString = "${queue.core.housekeeper.stats.print-interval-ms:60000}")
+    private void reportStats() {
+        String statsStr = stats.values().stream().map(stats -> {
+            String countersStr = stats.getCounters().stream()
+                    .filter(counter -> counter.get() > 0)
+                    .map(counter -> counter.getName() + " = [" + counter.get() + "]")
+                    .collect(Collectors.joining(" "));
+            if (countersStr.isEmpty()) {
+                return null;
+            } else {
+                return stats.getTaskType() + " " + countersStr + " avgProcessingTime [" + stats.getProcessingTimer().getAvg() + " ms]";
+            }
+        }).filter(Objects::nonNull).collect(Collectors.joining(", "));
+
+        if (!statsStr.isEmpty()) {
+            stats.values().forEach(HousekeeperStats::reset);
+            log.info("Housekeeper stats: {}", statsStr);
+        }
+    }
+
+    public void reportProcessed(HousekeeperTaskType taskType, ToHousekeeperServiceMsg msg, long timing) {
+        HousekeeperStats stats = this.stats.get(taskType);
+        if (msg.getTask().getErrorsCount() == 0) {
+            stats.getProcessedCounter().increment();
+        } else {
+            stats.getReprocessedCounter().increment();
+        }
+        stats.getProcessingTimer().record(timing);
+    }
+
+    public void reportFailure(HousekeeperTaskType taskType, ToHousekeeperServiceMsg msg) {
+        HousekeeperStats stats = this.stats.get(taskType);
+        if (msg.getTask().getErrorsCount() == 0) {
+            stats.getFailedProcessingCounter().increment();
+        } else {
+            stats.getFailedReprocessingCounter().increment();
+        }
+    }
+
+    @Getter
+    static class HousekeeperStats {
+        private final HousekeeperTaskType taskType;
+        private final List<StatsCounter> counters = new ArrayList<>();
+
+        private final StatsCounter processedCounter;
+        private final StatsCounter failedProcessingCounter;
+        private final StatsCounter reprocessedCounter;
+        private final StatsCounter failedReprocessingCounter;
+
+        private final StatsTimer processingTimer;
+
+        public HousekeeperStats(HousekeeperTaskType taskType, StatsFactory statsFactory) {
+            this.taskType = taskType;
+            this.processedCounter = register("processed", statsFactory);
+            this.failedProcessingCounter = register("failedProcessing", statsFactory);
+            this.reprocessedCounter = register("reprocessed", statsFactory);
+            this.failedReprocessingCounter = register("failedReprocessing", statsFactory);
+            this.processingTimer = statsFactory.createTimer(StatsType.HOUSEKEEPER, "processingTime", "taskType", taskType.name());
+        }
+
+        private StatsCounter register(String statsName, StatsFactory statsFactory) {
+            StatsCounter counter = statsFactory.createStatsCounter(StatsType.HOUSEKEEPER.getName(), statsName, "taskType", taskType.name());
+            counters.add(counter);
+            return counter;
+        }
+
+        public void reset() {
+            counters.forEach(DefaultCounter::clear);
+            processingTimer.reset();
+        }
+
+    }
+
+}

application/src/main/java/org/thingsboard/server/service/install/DefaultSystemDataLoaderService.java

@@ -549,7 +549,7 @@ public class DefaultSystemDataLoaderService implements SystemDataLoaderService {
         } else {
             ListenableFuture<List<String>> saveFuture = attributesService.save(TenantId.SYS_TENANT_ID, deviceId, AttributeScope.SERVER_SCOPE,
                     Collections.singletonList(new BaseAttributeKvEntry(new BooleanDataEntry(key, value)
-                    , System.currentTimeMillis())));
+                            , System.currentTimeMillis())));
             addTsCallback(saveFuture, new TelemetrySaveCallback<>(deviceId, key, value));
         }
     }
@@ -693,23 +693,26 @@ public class DefaultSystemDataLoaderService implements SystemDataLoaderService {
 
     @Override
     @SneakyThrows
-    public void updateDefaultNotificationConfigs() {
-        PageDataIterable<TenantId> tenants = new PageDataIterable<>(tenantService::findTenantsIds, 500);
-        ExecutorService executor = Executors.newFixedThreadPool(Math.max(Runtime.getRuntime().availableProcessors(), 4));
-        log.info("Updating default edge failure notification configs for all tenants");
-        AtomicInteger count = new AtomicInteger();
-        for (TenantId tenantId : tenants) {
-            executor.submit(() -> {
-                notificationSettingsService.updateDefaultNotificationConfigs(tenantId);
-                int n = count.incrementAndGet();
-                if (n % 500 == 0) {
-                    log.info("{} tenants processed", n);
-                }
-            });
-        }
-        executor.shutdown();
-        executor.awaitTermination(Integer.MAX_VALUE, TimeUnit.SECONDS);
+    public void updateDefaultNotificationConfigs(boolean updateTenants) {
+        log.info("Updating notification configs...");
         notificationSettingsService.updateDefaultNotificationConfigs(TenantId.SYS_TENANT_ID);
+
+        if (updateTenants) {
+            PageDataIterable<TenantId> tenants = new PageDataIterable<>(tenantService::findTenantsIds, 500);
+            ExecutorService executor = Executors.newFixedThreadPool(Math.max(Runtime.getRuntime().availableProcessors(), 4));
+            AtomicInteger count = new AtomicInteger();
+            for (TenantId tenantId : tenants) {
+                executor.submit(() -> {
+                    notificationSettingsService.updateDefaultNotificationConfigs(tenantId);
+                    int n = count.incrementAndGet();
+                    if (n % 500 == 0) {
+                        log.info("{} tenants processed", n);
+                    }
+                });
+            }
+            executor.shutdown();
+            executor.awaitTermination(Integer.MAX_VALUE, TimeUnit.SECONDS);
+        }
     }
 
 }

application/src/main/java/org/thingsboard/server/service/install/SystemDataLoaderService.java

@@ -35,6 +35,6 @@ public interface SystemDataLoaderService {
 
     void createDefaultNotificationConfigs();
 
-    void updateDefaultNotificationConfigs();
+    void updateDefaultNotificationConfigs(boolean updateTenants);
 
 }

application/src/main/java/org/thingsboard/server/service/notification/rule/trigger/RuleEngineComponentLifecycleEventTriggerProcessor.java

@@ -47,7 +47,7 @@ public class RuleEngineComponentLifecycleEventTriggerProcessor implements Notifi
                 return false;
             }
         }
-        if (!partitionService.resolve(ServiceType.TB_RULE_ENGINE, trigger.getTenantId(), trigger.getComponentId()).isMyPartition()) {
+        if (!partitionService.isMyPartition(ServiceType.TB_RULE_ENGINE, trigger.getTenantId(), trigger.getComponentId())) {
             return false;
         }
 

application/src/main/java/org/thingsboard/server/service/notification/rule/trigger/TaskProcessingFailureTriggerProcessor.java

@@ -0,0 +1,53 @@
+/**
+ * Copyright © 2016-2024 The Thingsboard Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.thingsboard.server.service.notification.rule.trigger;
+
+import org.apache.commons.lang3.exception.ExceptionUtils;
+import org.springframework.stereotype.Service;
+import org.thingsboard.server.common.data.StringUtils;
+import org.thingsboard.server.common.data.housekeeper.HousekeeperTask;
+import org.thingsboard.server.common.data.notification.info.TaskProcessingFailureNotificationInfo;
+import org.thingsboard.server.common.data.notification.rule.trigger.TaskProcessingFailureTrigger;
+import org.thingsboard.server.common.data.notification.rule.trigger.config.NotificationRuleTriggerType;
+import org.thingsboard.server.common.data.notification.rule.trigger.config.TaskProcessingFailureNotificationRuleTriggerConfig;
+
+@Service
+public class TaskProcessingFailureTriggerProcessor implements NotificationRuleTriggerProcessor<TaskProcessingFailureTrigger, TaskProcessingFailureNotificationRuleTriggerConfig> {
+
+    @Override
+    public boolean matchesFilter(TaskProcessingFailureTrigger trigger, TaskProcessingFailureNotificationRuleTriggerConfig triggerConfig) {
+        return true;
+    }
+
+    @Override
+    public TaskProcessingFailureNotificationInfo constructNotificationInfo(TaskProcessingFailureTrigger trigger) {
+        HousekeeperTask task = trigger.getTask();
+        return TaskProcessingFailureNotificationInfo.builder()
+                .tenantId(task.getTenantId())
+                .entityId(task.getEntityId())
+                .taskType(task.getTaskType())
+                .taskDescription(task.getDescription())
+                .error(StringUtils.truncate(ExceptionUtils.getStackTrace(trigger.getError()), 1024))
+                .attempt(trigger.getAttempt())
+                .build();
+    }
+
+    @Override
+    public NotificationRuleTriggerType getTriggerType() {
+        return NotificationRuleTriggerType.TASK_PROCESSING_FAILURE;
+    }
+
+}

application/src/main/java/org/thingsboard/server/service/queue/consumer/QueueConsumerManager.java

@@ -0,0 +1,109 @@
+/**
+ * Copyright © 2016-2024 The Thingsboard Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.thingsboard.server.service.queue.consumer;
+
+import lombok.Builder;
+import lombok.Getter;
+import lombok.extern.slf4j.Slf4j;
+import org.thingsboard.common.util.ThingsBoardThreadFactory;
+import org.thingsboard.server.common.msg.queue.TopicPartitionInfo;
+import org.thingsboard.server.queue.TbQueueConsumer;
+import org.thingsboard.server.queue.TbQueueMsg;
+
+import java.util.List;
+import java.util.Set;
+import java.util.concurrent.ExecutorService;
+import java.util.function.Supplier;
+
+@Slf4j
+public class QueueConsumerManager<M extends TbQueueMsg> {
+
+    private final String name;
+    private final MsgPackProcessor<M> msgPackProcessor;
+    private final long pollInterval;
+    private final ExecutorService consumerExecutor;
+    private final String threadPrefix;
+
+    @Getter
+    private final TbQueueConsumer<M> consumer;
+    private volatile boolean stopped;
+
+    @Builder
+    public QueueConsumerManager(String name, MsgPackProcessor<M> msgPackProcessor,
+                                long pollInterval, Supplier<TbQueueConsumer<M>> consumerCreator,
+                                ExecutorService consumerExecutor, String threadPrefix) {
+        this.name = name;
+        this.pollInterval = pollInterval;
+        this.msgPackProcessor = msgPackProcessor;
+        this.consumerExecutor = consumerExecutor;
+        this.threadPrefix = threadPrefix;
+        this.consumer = consumerCreator.get();
+    }
+
+    public void subscribe() {
+        consumer.subscribe();
+    }
+
+    public void subscribe(Set<TopicPartitionInfo> partitions) {
+        consumer.subscribe(partitions);
+    }
+
+    public void launch() {
+        log.info("[{}] Launching consumer", name);
+        consumerExecutor.submit(() -> {
+            if (threadPrefix != null) {
+                ThingsBoardThreadFactory.addThreadNamePrefix(threadPrefix);
+            }
+            try {
+                consumerLoop(consumer);
+            } catch (Throwable e) {
+                log.error("Failure in consumer loop", e);
+            }
+            log.info("[{}] Consumer stopped", name);
+        });
+    }
+
+    private void consumerLoop(TbQueueConsumer<M> consumer) {
+        while (!stopped && !consumer.isStopped()) {
+            try {
+                List<M> msgs = consumer.poll(pollInterval);
+                if (msgs.isEmpty()) {
+                    continue;
+                }
+                msgPackProcessor.process(msgs, consumer);
+            } catch (Exception e) {
+                if (!consumer.isStopped()) {
+                    log.warn("Failed to process messages from queue", e);
+                    try {
+                        Thread.sleep(pollInterval);
+                    } catch (InterruptedException interruptedException) {
+                        log.trace("Failed to wait until the server has capacity to handle new requests", interruptedException);
+                    }
+                }
+            }
+        }
+    }
+
+    public void stop() {
+        log.debug("[{}] Stopping consumer", name);
+        stopped = true;
+        consumer.unsubscribe();
+    }
+
+    public interface MsgPackProcessor<M extends TbQueueMsg> {
+        void process(List<M> msgs, TbQueueConsumer<M> consumer) throws Exception;
+    }
+}

application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/DefaultExportableEntitiesService.java

@@ -147,10 +147,7 @@ public class DefaultExportableEntitiesService implements ExportableEntitiesServi
     public <I extends EntityId> void removeById(TenantId tenantId, I id) {
         EntityType entityType = id.getEntityType();
         EntityDaoService entityService = entityServiceRegistry.getServiceByEntityType(entityType);
-        if (entityService == null) {
-            throw new IllegalArgumentException("Unsupported entity type " + entityType);
-        }
-        entityService.deleteEntity(tenantId, id);
+        entityService.deleteEntity(tenantId, id, false);
     }
 
     private <I extends EntityId, E extends ExportableEntity<I>> ExportableEntityDao<I, E> getExportableEntityDao(EntityType entityType) {

application/src/main/java/org/thingsboard/server/service/ttl/AlarmsCleanUpService.java

@@ -56,7 +56,6 @@ public class AlarmsCleanUpService {
     private final TenantService tenantService;
     private final AlarmDao alarmDao;
     private final AlarmService alarmService;
-    private final RelationService relationService;
     private final EntityActionService entityActionService;
     private final PartitionService partitionService;
     private final TbTenantProfileCache tenantProfileCache;

application/src/main/resources/thingsboard.yml

@@ -1445,6 +1445,16 @@ queue:
     #      tb_rule_engine.sq:
     #        - key: max.poll.records
     #          value: "${TB_QUEUE_KAFKA_SQ_MAX_POLL_RECORDS:1024}"
+      tb_housekeeper:
+        # Consumer properties for Housekeeper tasks topic
+        - key: max.poll.records
+          # Amount of records to be returned in a single poll. For Housekeeper tasks topic, we should consume messages (tasks) one by one
+          value: "${TB_QUEUE_KAFKA_HOUSEKEEPER_MAX_POLL_RECORDS:1}"
+      tb_housekeeper.reprocessing:
+        # Consumer properties for Housekeeper reprocessing topic
+        - key: max.poll.records
+          # Amount of records to be returned in a single poll. For Housekeeper reprocessing topic, we should consume messages (tasks) one by one
+          value: "${TB_QUEUE_KAFKA_HOUSEKEEPER_REPROCESSING_MAX_POLL_RECORDS:1}"
     other-inline: "${TB_QUEUE_KAFKA_OTHER_PROPERTIES:}" # In this section you can specify custom parameters (semicolon separated) for Kafka consumer/producer/admin # Example "metrics.recording.level:INFO;metrics.sample.window.ms:30000"
     other: # DEPRECATED. In this section, you can specify custom parameters for Kafka consumer/producer and expose the env variables to configure outside
     #  - key: "request.timeout.ms" # refer to https://docs.confluent.io/platform/current/installation/configuration/producer-configs.html#producerconfigs_request.timeout.ms
@@ -1466,6 +1476,10 @@ queue:
       ota-updates: "${TB_QUEUE_KAFKA_OTA_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:1048576000;partitions:10;min.insync.replicas:1}"
       # Kafka properties for Version Control topic
       version-control: "${TB_QUEUE_KAFKA_VC_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:1048576000;partitions:1;min.insync.replicas:1}"
+      # Kafka properties for Housekeeper tasks topic
+      housekeeper: "${TB_QUEUE_KAFKA_HOUSEKEEPER_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:1048576000;partitions:10;min.insync.replicas:1}"
+      # Kafka properties for Housekeeper reprocessing topic; retention.ms is set to 90 days; partitions is set to 1 since only one reprocessing service is running at a time
+      housekeeper-reprocessing: "${TB_QUEUE_KAFKA_HOUSEKEEPER_REPROCESSING_TOPIC_PROPERTIES:retention.ms:7776000000;segment.bytes:26214400;retention.bytes:1048576000;partitions:1;min.insync.replicas:1}"
     consumer-stats:
       # Prints lag between consumer group offset and last messages offset in Kafka topics
       enabled: "${TB_QUEUE_KAFKA_CONSUMER_STATS_ENABLED:true}"
@@ -1621,6 +1635,29 @@ queue:
       enabled: "${TB_QUEUE_CORE_STATS_ENABLED:true}"
       # Statistics printing interval for Core microservices
       print-interval-ms: "${TB_QUEUE_CORE_STATS_PRINT_INTERVAL_MS:60000}"
+    housekeeper:
+      # Topic name for Housekeeper tasks
+      topic: "${TB_HOUSEKEEPER_TOPIC:tb_housekeeper}"
+      # Topic name for Housekeeper tasks to be reprocessed
+      reprocessing-topic: "${TB_HOUSEKEEPER_REPROCESSING_TOPIC:tb_housekeeper.reprocessing}"
+      # Poll interval for topics related to Housekeeper
+      poll-interval-ms: "${TB_HOUSEKEEPER_POLL_INTERVAL_MS:500}"
+      # Timeout in milliseconds for task processing. Tasks that fail to finish on time will be submitted for reprocessing
+      task-processing-timeout-ms: "${TB_HOUSEKEEPER_TASK_PROCESSING_TIMEOUT_MS:120000}"
+      # Comma-separated list of task types that shouldn't be processed. Available task types:
+      # DELETE_ATTRIBUTES, DELETE_TELEMETRY (both DELETE_LATEST_TS and DELETE_TS_HISTORY will be disabled),
+      # DELETE_LATEST_TS, DELETE_TS_HISTORY, DELETE_EVENTS, DELETE_ALARMS, UNASSIGN_ALARMS
+      disabled-task-types: "${TB_HOUSEKEEPER_DISABLED_TASK_TYPES:}"
+      # Delay in milliseconds between tasks reprocessing
+      task-reprocessing-delay-ms: "${TB_HOUSEKEEPER_TASK_REPROCESSING_DELAY_MS:3000}"
+      # Maximum amount of task reprocessing attempts. After exceeding, the task will be dropped
+      max-reprocessing-attempts: "${TB_HOUSEKEEPER_MAX_REPROCESSING_ATTEMPTS:10}"
+      stats:
+        # Enable/disable statistics for Housekeeper
+        enabled: "${TB_HOUSEKEEPER_STATS_ENABLED:true}"
+        # Statistics printing interval for Housekeeper
+        print-interval-ms: "${TB_HOUSEKEEPER_STATS_PRINT_INTERVAL_MS:60000}"
+
   vc:
     # Default topic name for Kafka, RabbitMQ, etc.
     topic: "${TB_QUEUE_VC_TOPIC:tb_version_control}"

application/src/test/java/org/thingsboard/server/controller/AbstractWebTest.java

@@ -25,7 +25,6 @@ import io.jsonwebtoken.Header;
 import io.jsonwebtoken.Jwt;
 import io.jsonwebtoken.Jwts;
 import lombok.extern.slf4j.Slf4j;
-import org.apache.http.HttpStatus;
 import org.awaitility.Awaitility;
 import org.hamcrest.Matcher;
 import org.hibernate.exception.ConstraintViolationException;
@@ -114,9 +113,11 @@ import org.thingsboard.server.common.data.tenant.profile.TenantProfileData;
 import org.thingsboard.server.common.msg.session.FeatureType;
 import org.thingsboard.server.config.ThingsboardSecurityConfiguration;
 import org.thingsboard.server.dao.Dao;
+import org.thingsboard.server.dao.attributes.AttributesService;
 import org.thingsboard.server.dao.device.ClaimDevicesService;
 import org.thingsboard.server.dao.tenant.TenantProfileService;
 import org.thingsboard.server.dao.timeseries.TimeseriesService;
+import org.thingsboard.server.queue.memory.InMemoryStorage;
 import org.thingsboard.server.service.entitiy.tenant.profile.TbTenantProfileService;
 import org.thingsboard.server.service.security.auth.jwt.RefreshTokenRequest;
 import org.thingsboard.server.service.security.auth.rest.LoginRequest;
@@ -229,7 +230,10 @@ public abstract class AbstractWebTest extends AbstractInMemoryStorageTest {
     private TbTenantProfileService tbTenantProfileService;
 
     @Autowired
-    public TimeseriesService tsService;
+    protected TimeseriesService tsService;
+
+    @Autowired
+    protected AttributesService attributesService;
 
     @Autowired
     protected DefaultActorService actorService;
@@ -240,6 +244,9 @@ public abstract class AbstractWebTest extends AbstractInMemoryStorageTest {
     @SpyBean
     protected MailService mailService;
 
+    @Autowired
+    protected InMemoryStorage storage;
+
     @Rule
     public TestRule watcher = new TestWatcher() {
         protected void starting(Description description) {
@@ -282,7 +289,7 @@ public abstract class AbstractWebTest extends AbstractInMemoryStorageTest {
 
         Tenant tenant = new Tenant();
         tenant.setTitle(TEST_TENANT_NAME);
-        Tenant savedTenant = doPost("/api/tenant", tenant, Tenant.class);
+        Tenant savedTenant = saveTenant(tenant);
         Assert.assertNotNull(savedTenant);
         tenantId = savedTenant.getId();
         tenantProfileId = savedTenant.getTenantProfileId();
@@ -360,22 +367,14 @@ public abstract class AbstractWebTest extends AbstractInMemoryStorageTest {
         assertThat(loadedTenants).as("All tenants expected to be deleted, but some tenants left in the database").isEmpty();
     }
 
-    private void deleteTenant(TenantId tenantId) {
-        int status = 0;
-        int retries = 0;
-        while (status != HttpStatus.SC_OK && retries < CLEANUP_TENANT_RETRIES_COUNT) {
-            retries++;
-            try {
-                status = doDelete("/api/tenant/" + tenantId.getId().toString())
-                        .andReturn().getResponse().getStatus();
-                if (status != HttpStatus.SC_OK) {
-                    log.warn("Tenant deletion failed, tenantId: {}", tenantId.getId().toString());
-                    Thread.sleep(1000L);
-                }
-            } catch (Exception e) {
-                throw new RuntimeException(e);
-            }
+    protected void deleteTenant(TenantId tenantId) {
+        try {
+            doDelete("/api/tenant/" + tenantId.getId()).andExpect(status().isOk());
+        } catch (Exception e) {
+            throw new RuntimeException(e);
         }
+        Awaitility.await("all tasks processed").atMost(60, TimeUnit.SECONDS).during(300, TimeUnit.MILLISECONDS)
+                .until(() -> storage.getLag("tb_housekeeper") == 0);
     }
 
     private List<Tenant> getAllTenants() throws Exception {
@@ -428,7 +427,7 @@ public abstract class AbstractWebTest extends AbstractInMemoryStorageTest {
         loginSysAdmin();
         Tenant tenant = new Tenant();
         tenant.setTitle(TEST_DIFFERENT_TENANT_NAME);
-        savedDifferentTenant = doPost("/api/tenant", tenant, Tenant.class);
+        savedDifferentTenant = saveTenant(tenant);
         differentTenantId = savedDifferentTenant.getId();
         Assert.assertNotNull(savedDifferentTenant);
         User differentTenantAdmin = new User();
@@ -438,6 +437,10 @@ public abstract class AbstractWebTest extends AbstractInMemoryStorageTest {
         savedDifferentTenantUser = createUserAndLogin(differentTenantAdmin, DIFFERENT_TENANT_ADMIN_PASSWORD);
     }
 
+    protected Tenant saveTenant(Tenant tenant) throws Exception {
+        return doPost("/api/tenant", tenant, Tenant.class);
+    }
+
     protected void loginDifferentCustomer() throws Exception {
         if (savedDifferentCustomer != null) {
             login(savedDifferentCustomer.getEmail(), CUSTOMER_USER_PASSWORD);

application/src/test/java/org/thingsboard/server/controller/AlarmControllerTest.java

@@ -53,9 +53,8 @@ import java.util.Collections;
 import java.util.Comparator;
 import java.util.LinkedList;
 import java.util.List;
-import java.util.stream.Collectors;
-import java.util.Objects;
 import java.util.concurrent.TimeUnit;
+import java.util.stream.Collectors;
 
 import static org.hamcrest.Matchers.containsString;
 import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status;
@@ -584,7 +583,7 @@ public class AlarmControllerTest extends AbstractControllerTest {
         Assert.assertEquals(savedUser.getId(), foundAlarm.getAssigneeId());
         Assert.assertTrue(foundAlarm.getAssignTs() >= beforeAssignmentTs);
 
-        beforeAssignmentTs = System.currentTimeMillis();
+        long afterAssignmentTs = System.currentTimeMillis();
 
         loginSysAdmin();
 
@@ -592,12 +591,12 @@ public class AlarmControllerTest extends AbstractControllerTest {
 
         loginDifferentTenant();
 
-        foundAlarm = Awaitility.await().atMost(TIMEOUT, TimeUnit.SECONDS)
-                .until(() -> doGet("/api/alarm/info/" + alarmId.getId(), AlarmInfo.class), Objects::nonNull);
-
-        Assert.assertNotNull(foundAlarm);
-        Assert.assertNull(foundAlarm.getAssigneeId());
-        Assert.assertTrue(foundAlarm.getAssignTs() >= beforeAssignmentTs);
+        Awaitility.await().atMost(TIMEOUT, TimeUnit.SECONDS).untilAsserted(() -> {
+            AlarmInfo alarmInfo = doGet("/api/alarm/info/" + alarmId.getId(), AlarmInfo.class);
+            Assert.assertNotNull(alarmInfo);
+            Assert.assertNull(alarmInfo.getAssigneeId());
+            Assert.assertTrue(alarmInfo.getAssignTs() >= afterAssignmentTs);
+        });
     }
 
     @Test
@@ -631,71 +630,16 @@ public class AlarmControllerTest extends AbstractControllerTest {
         Assert.assertEquals(savedUser.getId(), foundAlarm.getAssigneeId());
         Assert.assertTrue(foundAlarm.getAssignTs() >= beforeAssignmentTs);
 
-        beforeAssignmentTs = System.currentTimeMillis();
+        long afterAssignmentTs = System.currentTimeMillis();
 
         doDelete("/api/user/" + savedUser.getId().getId()).andExpect(status().isOk());
 
-        foundAlarm = Awaitility.await().atMost(TIMEOUT, TimeUnit.SECONDS)
-                .until(() -> doGet("/api/alarm/info/" + alarmId.getId(), AlarmInfo.class), Objects::nonNull);
-
-        Assert.assertNotNull(foundAlarm);
-        Assert.assertNull(foundAlarm.getAssigneeId());
-        Assert.assertTrue(foundAlarm.getAssignTs() >= beforeAssignmentTs);
-    }
-
-    @Test
-    public void testUnassignAlarmOnCustomerRemoving() throws Exception {
-        createDifferentTenantCustomer();
-        loginDifferentTenant();
-
-        User user = new User();
-        user.setAuthority(Authority.CUSTOMER_USER);
-        user.setTenantId(tenantId);
-        user.setCustomerId(differentTenantCustomerId);
-        user.setEmail("customerForAssign@thingsboard.org");
-        User savedUser = createUser(user, "password");
-
-        Device device = createDevice("Different customer device", "default", "differentTenantTest");
-
-        Device assignedDevice = doPost("/api/customer/" + differentTenantCustomerId.getId()
-                + "/device/" + device.getId().getId(), Device.class);
-        Assert.assertEquals(differentTenantCustomerId, assignedDevice.getCustomerId());
-
-        Alarm alarm = Alarm.builder()
-                .type(TEST_ALARM_TYPE)
-                .tenantId(savedDifferentTenant.getId())
-                .customerId(differentTenantCustomerId)
-                .originator(device.getId())
-                .severity(AlarmSeverity.MAJOR)
-                .build();
-        alarm = doPost("/api/alarm", alarm, Alarm.class);
-        Assert.assertNotNull(alarm);
-        AlarmId alarmId = alarm.getId();
-
-        alarm = doGet("/api/alarm/info/" + alarmId.getId(), AlarmInfo.class);
-        Assert.assertNotNull(alarm);
-
-        Mockito.reset(tbClusterService, auditLogService);
-        long beforeAssignmentTs = System.currentTimeMillis();
-
-        doPost("/api/alarm/" + alarmId.getId() + "/assign/" + savedUser.getId().getId()).andExpect(status().isOk());
-        AlarmInfo foundAlarm = doGet("/api/alarm/info/" + alarmId.getId(), AlarmInfo.class);
-        Assert.assertNotNull(foundAlarm);
-        Assert.assertEquals(savedUser.getId(), foundAlarm.getAssigneeId());
-        Assert.assertTrue(foundAlarm.getAssignTs() >= beforeAssignmentTs);
-
-        beforeAssignmentTs = System.currentTimeMillis();
-
-        Mockito.reset(tbClusterService, auditLogService);
-
-        doDelete("/api/customer/" + differentTenantCustomerId.getId()).andExpect(status().isOk());
-
-        foundAlarm = Awaitility.await().atMost(TIMEOUT, TimeUnit.SECONDS)
-                .until(() -> doGet("/api/alarm/info/" + alarmId.getId(), AlarmInfo.class), Objects::nonNull);
-
-        Assert.assertNotNull(foundAlarm);
-        Assert.assertNull(foundAlarm.getAssigneeId());
-        Assert.assertTrue(foundAlarm.getAssignTs() >= beforeAssignmentTs);
+         Awaitility.await().atMost(TIMEOUT, TimeUnit.SECONDS).untilAsserted(() -> {
+             AlarmInfo alarmInfo = doGet("/api/alarm/info/" + alarmId.getId(), AlarmInfo.class);
+             Assert.assertNotNull(alarmInfo);
+             Assert.assertNull(alarmInfo.getAssigneeId());
+             Assert.assertTrue(alarmInfo.getAssignTs() >= afterAssignmentTs);
+         });
     }
 
     @Test
@@ -1003,27 +947,20 @@ public class AlarmControllerTest extends AbstractControllerTest {
             types.remove(0);
         }
 
-        foundTypes = doGetTyped("/api/alarm/types?pageSize=1024&page=0", new TypeReference<PageData<EntitySubtype>>() {
-        })
-                .getData()
-                .stream()
-                .map(EntitySubtype::getType)
-                .collect(Collectors.toList());
-
-        Assert.assertEquals(types.size(), foundTypes.size());
-        Assert.assertEquals(types, foundTypes);
+        Awaitility.await().atMost(TIMEOUT, TimeUnit.SECONDS).untilAsserted(() -> {
+            List<String> actualTypes = doGetTyped("/api/alarm/types?pageSize=1024&page=0", new TypeReference<PageData<EntitySubtype>>() {})
+                    .getData().stream().map(EntitySubtype::getType).toList();
+            Assert.assertEquals(types.size(), actualTypes.size());
+            Assert.assertEquals(types, actualTypes);
+        });
 
         doDelete("/api/device/" + devices.get(0).getId()).andExpect(status().isOk());
 
-        foundTypes = doGetTyped("/api/alarm/types?pageSize=1024&page=0", new TypeReference<PageData<EntitySubtype>>() {
-        })
-                .getData()
-                .stream()
-                .map(EntitySubtype::getType)
-                .sorted()
-                .collect(Collectors.toList());
-
-        Assert.assertTrue(foundTypes.isEmpty());
+        Awaitility.await().atMost(TIMEOUT, TimeUnit.SECONDS).untilAsserted(() -> {
+            List<String> actualTypes = doGetTyped("/api/alarm/types?pageSize=1024&page=0", new TypeReference<PageData<EntitySubtype>>() {})
+                    .getData().stream().map(EntitySubtype::getType).sorted().toList();
+            Assert.assertTrue(actualTypes.isEmpty());
+        });
     }
 
 }

application/src/test/java/org/thingsboard/server/controller/AssetControllerTest.java

@@ -42,6 +42,7 @@ import org.thingsboard.server.common.data.asset.Asset;
 import org.thingsboard.server.common.data.asset.AssetProfile;
 import org.thingsboard.server.common.data.audit.ActionType;
 import org.thingsboard.server.common.data.edge.Edge;
+import org.thingsboard.server.common.data.id.AlarmId;
 import org.thingsboard.server.common.data.id.AssetId;
 import org.thingsboard.server.common.data.id.CustomerId;
 import org.thingsboard.server.common.data.page.PageData;
@@ -55,8 +56,10 @@ import org.thingsboard.server.dao.service.DaoSqlTest;
 
 import java.util.ArrayList;
 import java.util.List;
+import java.util.concurrent.TimeUnit;
 
 import static org.assertj.core.api.Assertions.assertThat;
+import static org.awaitility.Awaitility.await;
 import static org.hamcrest.Matchers.containsString;
 import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status;
 import static org.thingsboard.server.dao.model.ModelConstants.NULL_UUID;
@@ -87,7 +90,7 @@ public class AssetControllerTest extends AbstractControllerTest {
 
         Tenant tenant = new Tenant();
         tenant.setTitle("My tenant");
-        savedTenant = doPost("/api/tenant", tenant, Tenant.class);
+        savedTenant = saveTenant(tenant);
         Assert.assertNotNull(savedTenant);
 
         tenantAdmin = new User();
@@ -104,8 +107,7 @@ public class AssetControllerTest extends AbstractControllerTest {
     public void afterTest() throws Exception {
         loginSysAdmin();
 
-        doDelete("/api/tenant/" + savedTenant.getId().getId().toString())
-                .andExpect(status().isOk());
+        deleteTenant(savedTenant.getId());
     }
 
     @Test
@@ -312,8 +314,9 @@ public class AssetControllerTest extends AbstractControllerTest {
 
         alarm = doPost("/api/alarm", alarm, Alarm.class);
         Assert.assertNotNull(alarm);
+        AlarmId alarmId = alarm.getId();
 
-        AlarmInfo foundAlarm = doGet("/api/alarm/info/" + alarm.getId(), AlarmInfo.class);
+        AlarmInfo foundAlarm = doGet("/api/alarm/info/" + alarmId, AlarmInfo.class);
         Assert.assertNotNull(foundAlarm);
 
         doDelete("/api/asset/" + savedAsset.getId().getId().toString())
@@ -324,9 +327,11 @@ public class AssetControllerTest extends AbstractControllerTest {
                 .andExpect(status().isNotFound())
                 .andExpect(statusReason(containsString(msgErrorNoFound("Asset", assetIdStr))));
 
-        doGet("/api/alarm/info/" + alarm.getId())
-                .andExpect(status().isNotFound())
-                .andExpect(statusReason(containsString(msgErrorNoFound("Alarm", alarm.getId().getId().toString()))));
+        await().atMost(30, TimeUnit.SECONDS).untilAsserted(() -> {
+            doGet("/api/alarm/info/" + alarmId)
+                    .andExpect(status().isNotFound())
+                    .andExpect(statusReason(containsString(msgErrorNoFound("Alarm", alarmId.getId().toString()))));
+        });
     }
 
     @Test
@@ -548,7 +553,7 @@ public class AssetControllerTest extends AbstractControllerTest {
 
         Tenant tenant2 = new Tenant();
         tenant2.setTitle("Different tenant");
-        Tenant savedTenant2 = doPost("/api/tenant", tenant2, Tenant.class);
+        Tenant savedTenant2 = saveTenant(tenant2);
         Assert.assertNotNull(savedTenant2);
 
         User tenantAdmin2 = new User();
@@ -582,8 +587,7 @@ public class AssetControllerTest extends AbstractControllerTest {
 
         loginSysAdmin();
 
-        doDelete("/api/tenant/" + savedTenant2.getId().getId().toString())
-                .andExpect(status().isOk());
+        deleteTenant(savedTenant2.getId());
     }
 
     @Test

application/src/test/java/org/thingsboard/server/controller/AssetProfileControllerTest.java

@@ -82,7 +82,7 @@ public class AssetProfileControllerTest extends AbstractControllerTest {
 
         Tenant tenant = new Tenant();
         tenant.setTitle("My tenant");
-        savedTenant = doPost("/api/tenant", tenant, Tenant.class);
+        savedTenant = saveTenant(tenant);
         Assert.assertNotNull(savedTenant);
 
         tenantAdmin = new User();
@@ -99,8 +99,7 @@ public class AssetProfileControllerTest extends AbstractControllerTest {
     public void afterTest() throws Exception {
         loginSysAdmin();
 
-        doDelete("/api/tenant/" + savedTenant.getId().getId().toString())
-                .andExpect(status().isOk());
+        deleteTenant(savedTenant.getId());
     }
 
     @Test

application/src/test/java/org/thingsboard/server/controller/AuditLogControllerTest.java

@@ -49,7 +49,6 @@ import static org.junit.jupiter.api.Assertions.assertDoesNotThrow;
 import static org.mockito.ArgumentMatchers.eq;
 import static org.mockito.Mockito.reset;
 import static org.mockito.Mockito.verify;
-import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status;
 
 @DaoSqlTest
 public class AuditLogControllerTest extends AbstractControllerTest {
@@ -75,7 +74,7 @@ public class AuditLogControllerTest extends AbstractControllerTest {
 
         Tenant tenant = new Tenant();
         tenant.setTitle("My tenant");
-        savedTenant = doPost("/api/tenant", tenant, Tenant.class);
+        savedTenant = saveTenant(tenant);
         Assert.assertNotNull(savedTenant);
 
         tenantAdmin = new User();
@@ -92,8 +91,7 @@ public class AuditLogControllerTest extends AbstractControllerTest {
     public void afterTest() throws Exception {
         loginSysAdmin();
 
-        doDelete("/api/tenant/" + savedTenant.getId().getId().toString())
-                .andExpect(status().isOk());
+        deleteTenant(savedTenant.getId());
     }
 
     @Test

application/src/test/java/org/thingsboard/server/controller/ComponentDescriptorControllerTest.java

@@ -47,7 +47,7 @@ public class ComponentDescriptorControllerTest extends AbstractControllerTest {
 
         Tenant tenant = new Tenant();
         tenant.setTitle("My tenant");
-        savedTenant = doPost("/api/tenant", tenant, Tenant.class);
+        savedTenant = saveTenant(tenant);
         Assert.assertNotNull(savedTenant);
 
         tenantAdmin = new User();
@@ -64,8 +64,7 @@ public class ComponentDescriptorControllerTest extends AbstractControllerTest {
     public void afterTest() throws Exception {
         loginSysAdmin();
 
-        doDelete("/api/tenant/" + savedTenant.getId().getId().toString())
-                .andExpect(status().isOk());
+        deleteTenant(savedTenant.getId());
     }
 
     @Test

application/src/test/java/org/thingsboard/server/controller/CustomerControllerTest.java

@@ -85,7 +85,7 @@ public class CustomerControllerTest extends AbstractControllerTest {
 
         Tenant tenant = new Tenant();
         tenant.setTitle("My tenant");
-        savedTenant = doPost("/api/tenant", tenant, Tenant.class);
+        savedTenant = saveTenant(tenant);
         Assert.assertNotNull(savedTenant);
 
         tenantAdmin = new User();
@@ -104,8 +104,7 @@ public class CustomerControllerTest extends AbstractControllerTest {
 
         loginSysAdmin();
 
-        doDelete("/api/tenant/" + savedTenant.getId().getId().toString())
-                .andExpect(status().isOk());
+        deleteTenant(savedTenant.getId());
     }
 
     @Test
@@ -154,7 +153,7 @@ public class CustomerControllerTest extends AbstractControllerTest {
                 .andExpect(statusReason(containsString(msgError)));
 
         customer.setTenantId(savedTenant.getId());
-        testNotifyEntityEqualsOneTimeServiceNeverError(customer,savedTenant.getId(),
+        testNotifyEntityEqualsOneTimeServiceNeverError(customer, savedTenant.getId(),
                 tenantAdmin.getId(), tenantAdmin.getEmail(), ActionType.ADDED, new DataValidationException(msgError));
         Mockito.reset(tbClusterService, auditLogService);
 
@@ -165,7 +164,7 @@ public class CustomerControllerTest extends AbstractControllerTest {
                 .andExpect(status().isBadRequest())
                 .andExpect(statusReason(containsString(msgError)));
 
-        testNotifyEntityEqualsOneTimeServiceNeverError(customer,savedTenant.getId(),
+        testNotifyEntityEqualsOneTimeServiceNeverError(customer, savedTenant.getId(),
                 tenantAdmin.getId(), tenantAdmin.getEmail(), ActionType.ADDED, new DataValidationException(msgError));
         Mockito.reset(tbClusterService, auditLogService);
 
@@ -176,7 +175,7 @@ public class CustomerControllerTest extends AbstractControllerTest {
                 .andExpect(status().isBadRequest())
                 .andExpect(statusReason(containsString(msgError)));
 
-        testNotifyEntityEqualsOneTimeServiceNeverError(customer,savedTenant.getId(),
+        testNotifyEntityEqualsOneTimeServiceNeverError(customer, savedTenant.getId(),
                 tenantAdmin.getId(), tenantAdmin.getEmail(), ActionType.ADDED, new DataValidationException(msgError));
         Mockito.reset(tbClusterService, auditLogService);
 
@@ -187,7 +186,7 @@ public class CustomerControllerTest extends AbstractControllerTest {
                 .andExpect(status().isBadRequest())
                 .andExpect(statusReason(containsString(msgError)));
 
-        testNotifyEntityEqualsOneTimeServiceNeverError(customer,savedTenant.getId(),
+        testNotifyEntityEqualsOneTimeServiceNeverError(customer, savedTenant.getId(),
                 tenantAdmin.getId(), tenantAdmin.getEmail(), ActionType.ADDED, new DataValidationException(msgError));
         Mockito.reset(tbClusterService, auditLogService);
 
@@ -198,7 +197,7 @@ public class CustomerControllerTest extends AbstractControllerTest {
                 .andExpect(status().isBadRequest())
                 .andExpect(statusReason(containsString(msgError)));
 
-        testNotifyEntityEqualsOneTimeServiceNeverError(customer,savedTenant.getId(),
+        testNotifyEntityEqualsOneTimeServiceNeverError(customer, savedTenant.getId(),
                 tenantAdmin.getId(), tenantAdmin.getEmail(), ActionType.ADDED, new DataValidationException(msgError));
         Mockito.reset(tbClusterService, auditLogService);
 
@@ -209,7 +208,7 @@ public class CustomerControllerTest extends AbstractControllerTest {
                 .andExpect(status().isBadRequest())
                 .andExpect(statusReason(containsString(msgError)));
 
-        testNotifyEntityEqualsOneTimeServiceNeverError(customer,savedTenant.getId(),
+        testNotifyEntityEqualsOneTimeServiceNeverError(customer, savedTenant.getId(),
                 tenantAdmin.getId(), tenantAdmin.getEmail(), ActionType.ADDED, new DataValidationException(msgError));
     }
 
@@ -293,7 +292,7 @@ public class CustomerControllerTest extends AbstractControllerTest {
                 .andExpect(status().isBadRequest())
                 .andExpect(statusReason(containsString(msgError)));
 
-        testNotifyEntityEqualsOneTimeServiceNeverError(customer,savedTenant.getId(),
+        testNotifyEntityEqualsOneTimeServiceNeverError(customer, savedTenant.getId(),
                 tenantAdmin.getId(), tenantAdmin.getEmail(), ActionType.ADDED, new DataValidationException(msgError));
     }
 

application/src/test/java/org/thingsboard/server/controller/DashboardControllerTest.java

@@ -77,7 +77,7 @@ public class DashboardControllerTest extends AbstractControllerTest {
 
         Tenant tenant = new Tenant();
         tenant.setTitle("My tenant");
-        savedTenant = doPost("/api/tenant", tenant, Tenant.class);
+        savedTenant = saveTenant(tenant);
         Assert.assertNotNull(savedTenant);
 
         tenantAdmin = new User();
@@ -94,8 +94,7 @@ public class DashboardControllerTest extends AbstractControllerTest {
     public void afterTest() throws Exception {
         loginSysAdmin();
 
-        doDelete("/api/tenant/" + savedTenant.getId().getId().toString())
-                .andExpect(status().isOk());
+        deleteTenant(savedTenant.getId());
     }
 
     @Test
@@ -166,7 +165,7 @@ public class DashboardControllerTest extends AbstractControllerTest {
     @Test
     public void testSaveDashboardWithEmptyTitle() throws Exception {
         Dashboard dashboard = new Dashboard();
-        String msgError = "Dashboard title " + msgErrorShouldBeSpecified;;
+        String msgError = "Dashboard title " + msgErrorShouldBeSpecified;
 
         Mockito.reset(tbClusterService, auditLogService);
 
@@ -196,7 +195,7 @@ public class DashboardControllerTest extends AbstractControllerTest {
         Assert.assertTrue(assignedDashboard.getAssignedCustomers().contains(savedCustomer.toShortCustomerInfo()));
 
         testNotifyEntityAllOneTimeLogEntityActionEntityEqClass(assignedDashboard, assignedDashboard.getId(), assignedDashboard.getId(),
-                savedTenant.getId(),  savedCustomer.getId(), tenantAdmin.getId(), tenantAdmin.getEmail(), ActionType.ASSIGNED_TO_CUSTOMER,
+                savedTenant.getId(), savedCustomer.getId(), tenantAdmin.getId(), tenantAdmin.getEmail(), ActionType.ASSIGNED_TO_CUSTOMER,
                 ActionType.UPDATED, assignedDashboard.getId().getId().toString(), savedCustomer.getId().getId().toString(), savedCustomer.getTitle());
 
         Dashboard foundDashboard = doGet("/api/dashboard/" + savedDashboard.getId().getId().toString(), Dashboard.class);
@@ -239,8 +238,8 @@ public class DashboardControllerTest extends AbstractControllerTest {
         Assert.assertTrue(publicCustomer.isPublic());
 
         testNotifyEntityAllOneTimeLogEntityActionEntityEqClass(assignedDashboard, assignedDashboard.getId(), assignedDashboard.getId(),
-                savedTenant.getId(),  publicCustomer.getId(), tenantAdmin.getId(), tenantAdmin.getEmail(), ActionType.ASSIGNED_TO_CUSTOMER,
-                ActionType.UPDATED, assignedDashboard .getId().getId().toString(), publicCustomer.getId().getId().toString(), publicCustomer.getTitle());
+                savedTenant.getId(), publicCustomer.getId(), tenantAdmin.getId(), tenantAdmin.getEmail(), ActionType.ASSIGNED_TO_CUSTOMER,
+                ActionType.UPDATED, assignedDashboard.getId().getId().toString(), publicCustomer.getId().getId().toString(), publicCustomer.getTitle());
 
         Dashboard foundDashboard = doGet("/api/dashboard/" + savedDashboard.getId().getId().toString(), Dashboard.class);
         Assert.assertTrue(foundDashboard.getAssignedCustomers().contains(publicCustomer.toShortCustomerInfo()));
@@ -283,7 +282,7 @@ public class DashboardControllerTest extends AbstractControllerTest {
 
         Tenant tenant2 = new Tenant();
         tenant2.setTitle("Different tenant");
-        Tenant savedTenant2 = doPost("/api/tenant", tenant2, Tenant.class);
+        Tenant savedTenant2 = saveTenant(tenant2);
         Assert.assertNotNull(savedTenant2);
 
         User tenantAdmin2 = new User();
@@ -321,8 +320,7 @@ public class DashboardControllerTest extends AbstractControllerTest {
 
         loginSysAdmin();
 
-        doDelete("/api/tenant/" + savedTenant2.getId().getId().toString())
-                .andExpect(status().isOk());
+        deleteTenant(savedTenant2.getId());
     }
 
     @Test
@@ -482,7 +480,7 @@ public class DashboardControllerTest extends AbstractControllerTest {
 
         testNotifyManyEntityManyTimeMsgToEdgeServiceEntityEqAny(new Dashboard(), new Dashboard(),
                 savedTenant.getId(), tenantAdmin.getCustomerId(), tenantAdmin.getId(), tenantAdmin.getEmail(),
-                ActionType.ADDED, cntEntity, cntEntity, cntEntity*2);
+                ActionType.ADDED, cntEntity, cntEntity, cntEntity * 2);
 
         List<DashboardInfo> loadedDashboards = new ArrayList<>();
         PageLink pageLink = new PageLink(21);

application/src/test/java/org/thingsboard/server/controller/DeviceConnectivityControllerTest.java

@@ -142,7 +142,7 @@ public class DeviceConnectivityControllerTest extends AbstractControllerTest {
 
         Tenant tenant = new Tenant();
         tenant.setTitle("My tenant");
-        savedTenant = doPost("/api/tenant", tenant, Tenant.class);
+        savedTenant = saveTenant(tenant);
         Assert.assertNotNull(savedTenant);
 
         tenantAdmin = new User();
@@ -187,8 +187,7 @@ public class DeviceConnectivityControllerTest extends AbstractControllerTest {
     public void afterTest() throws Exception {
         loginSysAdmin();
 
-        doDelete("/api/tenant/" + savedTenant.getId().getId())
-                .andExpect(status().isOk());
+        deleteTenant(savedTenant.getId());
     }
 
     @Test

application/src/test/java/org/thingsboard/server/controller/DeviceControllerTest.java

@@ -132,7 +132,7 @@ public class DeviceControllerTest extends AbstractControllerTest {
 
         Tenant tenant = new Tenant();
         tenant.setTitle("My tenant");
-        savedTenant = doPost("/api/tenant", tenant, Tenant.class);
+        savedTenant = saveTenant(tenant);
         Assert.assertNotNull(savedTenant);
 
         tenantAdmin = new User();
@@ -151,8 +151,7 @@ public class DeviceControllerTest extends AbstractControllerTest {
 
         loginSysAdmin();
 
-        doDelete("/api/tenant/" + savedTenant.getId().getId())
-                .andExpect(status().isOk());
+        deleteTenant(savedTenant.getId());
     }
 
     @Test
@@ -687,7 +686,7 @@ public class DeviceControllerTest extends AbstractControllerTest {
 
         Tenant tenant2 = new Tenant();
         tenant2.setTitle("Different tenant");
-        Tenant savedTenant2 = doPost("/api/tenant", tenant2, Tenant.class);
+        Tenant savedTenant2 = saveTenant(tenant2);
         Assert.assertNotNull(savedTenant2);
 
         User tenantAdmin2 = new User();
@@ -721,9 +720,7 @@ public class DeviceControllerTest extends AbstractControllerTest {
         testNotificationUpdateGatewayNever();
 
         loginSysAdmin();
-
-        doDelete("/api/tenant/" + savedTenant2.getId().getId())
-                .andExpect(status().isOk());
+        deleteTenant(savedTenant2.getId());
     }
 
     @Test
@@ -914,7 +911,7 @@ public class DeviceControllerTest extends AbstractControllerTest {
 
         testNotifyManyEntityManyTimeMsgToEdgeServiceEntityEqAnyAdditionalInfoAny(new Device(), new Device(),
                 savedTenant.getId(), tenantAdmin.getCustomerId(), tenantAdmin.getId(), tenantAdmin.getEmail(),
-                ActionType.DELETED, ActionType.DELETED, cntEntity, cntEntity,1);
+                ActionType.DELETED, ActionType.DELETED, cntEntity, cntEntity, 1);
         testNotificationUpdateGatewayNever();
     }
 
@@ -1319,7 +1316,7 @@ public class DeviceControllerTest extends AbstractControllerTest {
         loginSysAdmin();
         Tenant tenant = new Tenant();
         tenant.setTitle("Different tenant");
-        Tenant savedDifferentTenant = doPost("/api/tenant", tenant, Tenant.class);
+        Tenant savedDifferentTenant = saveTenant(tenant);
         Assert.assertNotNull(savedDifferentTenant);
 
         User user = new User();
@@ -1380,8 +1377,7 @@ public class DeviceControllerTest extends AbstractControllerTest {
                 .andExpect(statusReason(containsString(msgErrorNoFound("Device", savedAnotherDevice.getId().getId().toString()))));
 
         loginSysAdmin();
-        doDelete("/api/tenant/" + savedDifferentTenant.getId().getId())
-                .andExpect(status().isOk());
+        deleteTenant(savedDifferentTenant.getId());
     }
 
     @Test

application/src/test/java/org/thingsboard/server/controller/DeviceProfileControllerTest.java

@@ -93,7 +93,7 @@ public class DeviceProfileControllerTest extends AbstractControllerTest {
 
         Tenant tenant = new Tenant();
         tenant.setTitle("My tenant");
-        savedTenant = doPost("/api/tenant", tenant, Tenant.class);
+        savedTenant = saveTenant(tenant);
         Assert.assertNotNull(savedTenant);
 
         tenantAdmin = new User();
@@ -110,8 +110,7 @@ public class DeviceProfileControllerTest extends AbstractControllerTest {
     public void afterTest() throws Exception {
         loginSysAdmin();
 
-        doDelete("/api/tenant/" + savedTenant.getId().getId().toString())
-                .andExpect(status().isOk());
+        deleteTenant(savedTenant.getId());
     }
 
     @Test

application/src/test/java/org/thingsboard/server/controller/EdgeControllerTest.java

@@ -361,7 +361,7 @@ public class EdgeControllerTest extends AbstractControllerTest {
 
         Tenant tenant2 = new Tenant();
         tenant2.setTitle("Different tenant");
-        Tenant savedTenant2 = doPost("/api/tenant", tenant2, Tenant.class);
+        Tenant savedTenant2 = saveTenant(tenant2);
         Assert.assertNotNull(savedTenant2);
 
         User tenantAdmin2 = new User();
@@ -394,8 +394,7 @@ public class EdgeControllerTest extends AbstractControllerTest {
 
         loginSysAdmin();
 
-        doDelete("/api/tenant/" + savedTenant2.getId().getId().toString())
-                .andExpect(status().isOk());
+        deleteTenant(savedTenant2.getId());
     }
 
     @Test

application/src/test/java/org/thingsboard/server/controller/EntityQueryControllerTest.java

@@ -83,7 +83,7 @@ public class EntityQueryControllerTest extends AbstractControllerTest {
 
         Tenant tenant = new Tenant();
         tenant.setTitle("My tenant");
-        savedTenant = doPost("/api/tenant", tenant, Tenant.class);
+        savedTenant = saveTenant(tenant);
         Assert.assertNotNull(savedTenant);
 
         tenantAdmin = new User();
@@ -99,9 +99,7 @@ public class EntityQueryControllerTest extends AbstractControllerTest {
     @After
     public void afterTest() throws Exception {
         loginSysAdmin();
-
-        doDelete("/api/tenant/" + savedTenant.getId().getId().toString())
-                .andExpect(status().isOk());
+        deleteTenant(savedTenant.getId());
     }
 
     @Test

application/src/test/java/org/thingsboard/server/controller/EntityRelationControllerTest.java

@@ -72,7 +72,7 @@ public class EntityRelationControllerTest extends AbstractControllerTest {
         Tenant tenant = new Tenant();
         tenant.setTitle("Test tenant");
 
-        savedTenant = doPost("/api/tenant", tenant, Tenant.class);
+        savedTenant = saveTenant(tenant);
         Assert.assertNotNull(savedTenant);
 
         tenantAdmin = new User();
@@ -93,8 +93,7 @@ public class EntityRelationControllerTest extends AbstractControllerTest {
     public void afterTest() throws Exception {
         loginSysAdmin();
 
-        doDelete("/api/tenant/" + savedTenant.getId().getId().toString())
-                .andExpect(status().isOk());
+        deleteTenant(savedTenant.getId());
     }
 
     @Test

application/src/test/java/org/thingsboard/server/controller/EntityViewControllerTest.java

@@ -370,7 +370,7 @@ public class EntityViewControllerTest extends AbstractControllerTest {
         loginSysAdmin();
 
         Tenant tenant2 = getNewTenant("Different tenant");
-        Tenant savedTenant2 = doPost("/api/tenant", tenant2, Tenant.class);
+        Tenant savedTenant2 = saveTenant(tenant2);
         Assert.assertNotNull(savedTenant2);
 
         User tenantAdmin2 = new User();
@@ -398,8 +398,7 @@ public class EntityViewControllerTest extends AbstractControllerTest {
 
         loginSysAdmin();
 
-        doDelete("/api/tenant/" + savedTenant2.getId().getId().toString())
-                .andExpect(status().isOk());
+        deleteTenant(savedTenant2.getId());
     }
 
     @Test
@@ -431,7 +430,7 @@ public class EntityViewControllerTest extends AbstractControllerTest {
         testNotifyEntityBroadcastEntityStateChangeEventMany(new EntityView(), new EntityView(),
                 tenantId, customerId, tenantAdminUserId, TENANT_ADMIN_EMAIL,
                 ActionType.ASSIGNED_TO_CUSTOMER, ActionType.UPDATED, cntEntity, cntEntity,
-                cntEntity*2, 3);
+                cntEntity * 2, 3);
     }
 
     @Test

application/src/test/java/org/thingsboard/server/controller/HomePageApiTest.java

@@ -103,7 +103,7 @@ public class HomePageApiTest extends AbstractControllerTest {
         for (int i = 0; i < 100; i++) {
             Tenant tenant = new Tenant();
             tenant.setTitle("tenant" + i);
-            tenants.add(doPost("/api/tenant", tenant, Tenant.class));
+            tenants.add(saveTenant(tenant));
         }
 
         EntityTypeFilter ef = new EntityTypeFilter();
@@ -115,7 +115,7 @@ public class HomePageApiTest extends AbstractControllerTest {
         Assert.assertEquals(initialCount + 100, update.getCount());
 
         for (Tenant tenant : tenants) {
-            doDelete("/api/tenant/" + tenant.getId().toString());
+            deleteTenant(tenant.getId());
         }
     }
 

application/src/test/java/org/thingsboard/server/controller/OtaPackageControllerTest.java

@@ -72,7 +72,7 @@ public class OtaPackageControllerTest extends AbstractControllerTest {
 
         Tenant tenant = new Tenant();
         tenant.setTitle("My tenant");
-        savedTenant = doPost("/api/tenant", tenant, Tenant.class);
+        savedTenant = saveTenant(tenant);
         Assert.assertNotNull(savedTenant);
 
         tenantAdmin = new User();
@@ -94,8 +94,7 @@ public class OtaPackageControllerTest extends AbstractControllerTest {
     public void afterTest() throws Exception {
         loginSysAdmin();
 
-        doDelete("/api/tenant/" + savedTenant.getId().getId().toString())
-                .andExpect(status().isOk());
+        deleteTenant(savedTenant.getId());
     }
 
     @Test
@@ -339,7 +338,7 @@ public class OtaPackageControllerTest extends AbstractControllerTest {
 
         testNotifyManyEntityManyTimeMsgToEdgeServiceEntityEqAny(new OtaPackageInfo(), new OtaPackageInfo(),
                 savedTenant.getId(), tenantAdmin.getCustomerId(), tenantAdmin.getId(), tenantAdmin.getEmail(),
-                ActionType.ADDED, cntEntity, 0, (cntEntity*2 - startIndexSaveData));
+                ActionType.ADDED, cntEntity, 0, (cntEntity * 2 - startIndexSaveData));
 
         List<OtaPackageInfo> loadedFirmwares = new ArrayList<>();
         PageLink pageLink = new PageLink(24);

application/src/test/java/org/thingsboard/server/controller/RpcControllerTest.java

@@ -49,7 +49,7 @@ public class RpcControllerTest extends AbstractControllerTest {
 
         Tenant tenant = new Tenant();
         tenant.setTitle("My tenant");
-        savedTenant = doPost("/api/tenant", tenant, Tenant.class);
+        savedTenant = saveTenant(tenant);
         Assert.assertNotNull(savedTenant);
 
         tenantAdmin = new User();
@@ -66,8 +66,7 @@ public class RpcControllerTest extends AbstractControllerTest {
     public void afterTest() throws Exception {
         loginSysAdmin();
 
-        doDelete("/api/tenant/" + savedTenant.getId().getId().toString())
-                .andExpect(status().isOk());
+        deleteTenant(savedTenant.getId());
     }
 
     private Device createDefaultDevice() {

application/src/test/java/org/thingsboard/server/controller/RuleChainControllerTest.java

@@ -82,7 +82,7 @@ public class RuleChainControllerTest extends AbstractControllerTest {
 
         Tenant tenant = new Tenant();
         tenant.setTitle("My tenant");
-        savedTenant = doPost("/api/tenant", tenant, Tenant.class);
+        savedTenant = saveTenant(tenant);
         Assert.assertNotNull(savedTenant);
 
         tenantAdmin = new User();
@@ -99,8 +99,7 @@ public class RuleChainControllerTest extends AbstractControllerTest {
     public void afterTest() throws Exception {
         loginSysAdmin();
 
-        doDelete("/api/tenant/" + savedTenant.getId().getId().toString())
-                .andExpect(status().isOk());
+        deleteTenant(savedTenant.getId());
     }
 
     @Test
@@ -147,9 +146,9 @@ public class RuleChainControllerTest extends AbstractControllerTest {
         int currentVersion = annotation.version();
 
         String oldConfig = "{\"attrMapping\":{\"serialNumber\":\"sn\"}," +
-                        "\"relationsQuery\":{\"direction\":\"FROM\",\"maxLevel\":1," +
-                        "\"filters\":[{\"relationType\":\"Contains\",\"entityTypes\":[]}]," +
-                        "\"fetchLastLevelOnly\":false},\"telemetry\":false}";
+                "\"relationsQuery\":{\"direction\":\"FROM\",\"maxLevel\":1," +
+                "\"filters\":[{\"relationType\":\"Contains\",\"entityTypes\":[]}]," +
+                "\"fetchLastLevelOnly\":false},\"telemetry\":false}";
 
         TbGetRelatedDataNodeConfiguration defaultConfiguration = new TbGetRelatedDataNodeConfiguration().defaultConfiguration();
         String newConfig = JacksonUtil.toString(defaultConfiguration);

application/src/test/java/org/thingsboard/server/controller/TbResourceControllerTest.java

@@ -73,7 +73,7 @@ public class TbResourceControllerTest extends AbstractControllerTest {
 
         Tenant tenant = new Tenant();
         tenant.setTitle("My tenant");
-        savedTenant = doPost("/api/tenant", tenant, Tenant.class);
+        savedTenant = saveTenant(tenant);
         Assert.assertNotNull(savedTenant);
 
         tenantAdmin = new User();
@@ -90,8 +90,7 @@ public class TbResourceControllerTest extends AbstractControllerTest {
     public void afterTest() throws Exception {
         loginSysAdmin();
 
-        doDelete("/api/tenant/" + savedTenant.getId().getId().toString())
-                .andExpect(status().isOk());
+        deleteTenant(savedTenant.getId());
     }
 
     @Test

application/src/test/java/org/thingsboard/server/controller/TenantControllerTest.java

@@ -140,7 +140,7 @@ public class TenantControllerTest extends AbstractControllerTest {
 
         Mockito.reset(tbClusterService);
 
-        Tenant savedTenant = doPost("/api/tenant", tenant, Tenant.class);
+        Tenant savedTenant = saveTenant(tenant);
         Assert.assertNotNull(savedTenant);
         Assert.assertNotNull(savedTenant.getId());
         Assert.assertTrue(savedTenant.getCreatedTime() > 0);
@@ -149,14 +149,13 @@ public class TenantControllerTest extends AbstractControllerTest {
         testBroadcastEntityStateChangeEventTimeManyTimeTenant(savedTenant, ComponentLifecycleEvent.CREATED, 1);
 
         savedTenant.setTitle("My new tenant");
-        doPost("/api/tenant", savedTenant, Tenant.class);
+        saveTenant(savedTenant);
         Tenant foundTenant = doGet("/api/tenant/" + savedTenant.getId().getId().toString(), Tenant.class);
         Assert.assertEquals(foundTenant.getTitle(), savedTenant.getTitle());
 
         testBroadcastEntityStateChangeEventTimeManyTimeTenant(savedTenant, ComponentLifecycleEvent.UPDATED, 1);
 
-        doDelete("/api/tenant/" + savedTenant.getId().getId().toString())
-                .andExpect(status().isOk());
+        deleteTenant(savedTenant.getId());
 
         testBroadcastEntityStateChangeEventTimeManyTimeTenant(savedTenant, ComponentLifecycleEvent.DELETED, 1);
     }
@@ -181,12 +180,11 @@ public class TenantControllerTest extends AbstractControllerTest {
         loginSysAdmin();
         Tenant tenant = new Tenant();
         tenant.setTitle("My tenant");
-        Tenant savedTenant = doPost("/api/tenant", tenant, Tenant.class);
+        Tenant savedTenant = saveTenant(tenant);
         Tenant foundTenant = doGet("/api/tenant/" + savedTenant.getId().getId().toString(), Tenant.class);
         Assert.assertNotNull(foundTenant);
         Assert.assertEquals(savedTenant, foundTenant);
-        doDelete("/api/tenant/" + savedTenant.getId().getId().toString())
-                .andExpect(status().isOk());
+        deleteTenant(savedTenant.getId());
     }
 
     @Test
@@ -194,12 +192,11 @@ public class TenantControllerTest extends AbstractControllerTest {
         loginSysAdmin();
         Tenant tenant = new Tenant();
         tenant.setTitle("My tenant");
-        Tenant savedTenant = doPost("/api/tenant", tenant, Tenant.class);
+        Tenant savedTenant = saveTenant(tenant);
         TenantInfo foundTenant = doGet("/api/tenant/info/" + savedTenant.getId().getId().toString(), TenantInfo.class);
         Assert.assertNotNull(foundTenant);
         Assert.assertEquals(new TenantInfo(savedTenant, "Default"), foundTenant);
-        doDelete("/api/tenant/" + savedTenant.getId().getId().toString())
-                .andExpect(status().isOk());
+        deleteTenant(savedTenant.getId());
     }
 
     @Test
@@ -237,11 +234,10 @@ public class TenantControllerTest extends AbstractControllerTest {
         loginSysAdmin();
         Tenant tenant = new Tenant();
         tenant.setTitle("My tenant");
-        Tenant savedTenant = doPost("/api/tenant", tenant, Tenant.class);
+        Tenant savedTenant = saveTenant(tenant);
 
         String tenantIdStr = savedTenant.getId().getId().toString();
-        doDelete("/api/tenant/" + tenantIdStr)
-                .andExpect(status().isOk());
+        deleteTenant(savedTenant.getId());
         doGet("/api/tenant/" + tenantIdStr)
                 .andExpect(status().isNotFound())
                 .andExpect(statusReason(containsString(msgErrorNoFound("Tenant", tenantIdStr))));
@@ -264,8 +260,7 @@ public class TenantControllerTest extends AbstractControllerTest {
         for (int i = 0; i < cntEntity; i++) {
             Tenant tenant = new Tenant();
             tenant.setTitle("Tenant" + i);
-            createFutures.add(executor.submit(() ->
-                    doPost("/api/tenant", tenant, Tenant.class)));
+            createFutures.add(executor.submit(() -> saveTenant(tenant)));
         }
         tenants.addAll(Futures.allAsList(createFutures).get(TIMEOUT, TimeUnit.SECONDS));
 
@@ -308,8 +303,7 @@ public class TenantControllerTest extends AbstractControllerTest {
             String title = title1 + suffix;
             title = i % 2 == 0 ? title.toLowerCase() : title.toUpperCase();
             tenant.setTitle(title);
-            createFutures.add(executor.submit(() ->
-                    doPost("/api/tenant", tenant, Tenant.class)));
+            createFutures.add(executor.submit(() -> saveTenant(tenant)));
         }
 
         List<Tenant> tenantsTitle1 = Futures.allAsList(createFutures).get(TIMEOUT, TimeUnit.SECONDS);
@@ -323,8 +317,7 @@ public class TenantControllerTest extends AbstractControllerTest {
             String title = title2 + suffix;
             title = i % 2 == 0 ? title.toLowerCase() : title.toUpperCase();
             tenant.setTitle(title);
-            createFutures.add(executor.submit(() ->
-                    doPost("/api/tenant", tenant, Tenant.class)));
+            createFutures.add(executor.submit(() -> saveTenant(tenant)));
         }
 
         List<Tenant> tenantsTitle2 = Futures.allAsList(createFutures).get(TIMEOUT, TimeUnit.SECONDS);
@@ -396,7 +389,7 @@ public class TenantControllerTest extends AbstractControllerTest {
             Tenant tenant = new Tenant();
             tenant.setTitle("Tenant" + i);
             createFutures.add(executor.submit(() ->
-                    new TenantInfo(doPost("/api/tenant", tenant, Tenant.class), "Default")));
+                    new TenantInfo(saveTenant(tenant), "Default")));
         }
         tenants.addAll(Futures.allAsList(createFutures).get(TIMEOUT, TimeUnit.SECONDS));
 
@@ -448,7 +441,7 @@ public class TenantControllerTest extends AbstractControllerTest {
 
         Tenant tenant = new Tenant();
         tenant.setTitle("Isolated tenant");
-        tenant = doPost("/api/tenant", tenant, Tenant.class);
+        tenant = saveTenant(tenant);
 
         User tenantUser = new User();
         tenantUser.setAuthority(Authority.TENANT_ADMIN);
@@ -477,7 +470,7 @@ public class TenantControllerTest extends AbstractControllerTest {
         tenantProfile = doPost("/api/tenantProfile", tenantProfile, TenantProfile.class);
 
         tenant.setTenantProfileId(tenantProfile.getId());
-        doPost("/api/tenant", tenant, Tenant.class);
+        saveTenant(tenant);
 
         login(username, password);
 
@@ -507,7 +500,7 @@ public class TenantControllerTest extends AbstractControllerTest {
         tenantProfile2 = doPost("/api/tenantProfile", tenantProfile2, TenantProfile.class);
 
         tenant.setTenantProfileId(tenantProfile2.getId());
-        doPost("/api/tenant", tenant, Tenant.class);
+        saveTenant(tenant);
 
         login(username, password);
 
@@ -549,7 +542,7 @@ public class TenantControllerTest extends AbstractControllerTest {
         loginSysAdmin();
 
         tenant.setTenantProfileId(null);
-        doPost("/api/tenant", tenant, Tenant.class);
+        saveTenant(tenant);
 
         login(username, password);
         for (Queue queue : foundTenantQueues) {
@@ -559,7 +552,7 @@ public class TenantControllerTest extends AbstractControllerTest {
         }
 
         loginSysAdmin();
-        doDelete("/api/tenant/" + tenant.getId().getId().toString()).andExpect(status().isOk());
+        deleteTenant(tenant.getId());
     }
 
     @Test
@@ -576,7 +569,7 @@ public class TenantControllerTest extends AbstractControllerTest {
         createDifferentTenant();
         loginSysAdmin();
         savedDifferentTenant.setTenantProfileId(tenantProfile.getId());
-        savedDifferentTenant = doPost("/api/tenant", savedDifferentTenant, Tenant.class);
+        savedDifferentTenant = saveTenant(savedDifferentTenant);
         TenantId tenantId = differentTenantId;
 
         loginDifferentTenant();
@@ -661,7 +654,7 @@ public class TenantControllerTest extends AbstractControllerTest {
         createDifferentTenant();
         loginSysAdmin();
         savedDifferentTenant.setTenantProfileId(tenantProfile.getId());
-        savedDifferentTenant = doPost("/api/tenant", savedDifferentTenant, Tenant.class);
+        savedDifferentTenant = saveTenant(savedDifferentTenant);
         TenantId tenantId = differentTenantId;
         await().atMost(30, TimeUnit.SECONDS)
                 .until(() -> {
@@ -716,7 +709,7 @@ public class TenantControllerTest extends AbstractControllerTest {
         createDifferentTenant();
         loginSysAdmin();
         savedDifferentTenant.setTenantProfileId(tenantProfile.getId());
-        savedDifferentTenant = doPost("/api/tenant", savedDifferentTenant, Tenant.class);
+        savedDifferentTenant = saveTenant(savedDifferentTenant);
         TenantId tenantId = differentTenantId;
         await().atMost(30, TimeUnit.SECONDS).untilAsserted(() -> {
             assertThat(partitionService.getMyPartitions(new QueueKey(ServiceType.TB_RULE_ENGINE, tenantId))).isNotNull();

application/src/test/java/org/thingsboard/server/controller/TenantProfileControllerTest.java

@@ -75,7 +75,7 @@ public class TenantProfileControllerTest extends AbstractControllerTest {
 
         savedTenantProfile.setName("New tenant profile");
         doPost("/api/tenantProfile", savedTenantProfile, TenantProfile.class);
-        TenantProfile foundTenantProfile = doGet("/api/tenantProfile/"+savedTenantProfile.getId().getId().toString(), TenantProfile.class);
+        TenantProfile foundTenantProfile = doGet("/api/tenantProfile/" + savedTenantProfile.getId().getId().toString(), TenantProfile.class);
         Assert.assertEquals(foundTenantProfile.getName(), savedTenantProfile.getName());
 
         testBroadcastEntityStateChangeEventTimeManyTimeTenantProfile(savedTenantProfile, ComponentLifecycleEvent.UPDATED, 1);
@@ -100,7 +100,7 @@ public class TenantProfileControllerTest extends AbstractControllerTest {
         loginSysAdmin();
         TenantProfile tenantProfile = this.createTenantProfile("Tenant Profile");
         TenantProfile savedTenantProfile = doPost("/api/tenantProfile", tenantProfile, TenantProfile.class);
-        TenantProfile foundTenantProfile = doGet("/api/tenantProfile/"+savedTenantProfile.getId().getId().toString(), TenantProfile.class);
+        TenantProfile foundTenantProfile = doGet("/api/tenantProfile/" + savedTenantProfile.getId().getId().toString(), TenantProfile.class);
         Assert.assertNotNull(foundTenantProfile);
         Assert.assertEquals(savedTenantProfile, foundTenantProfile);
     }
@@ -110,7 +110,7 @@ public class TenantProfileControllerTest extends AbstractControllerTest {
         loginSysAdmin();
         TenantProfile tenantProfile = this.createTenantProfile("Tenant Profile");
         TenantProfile savedTenantProfile = doPost("/api/tenantProfile", tenantProfile, TenantProfile.class);
-        EntityInfo foundTenantProfileInfo = doGet("/api/tenantProfileInfo/"+savedTenantProfile.getId().getId().toString(), EntityInfo.class);
+        EntityInfo foundTenantProfileInfo = doGet("/api/tenantProfileInfo/" + savedTenantProfile.getId().getId().toString(), EntityInfo.class);
         Assert.assertNotNull(foundTenantProfileInfo);
         Assert.assertEquals(savedTenantProfile.getId(), foundTenantProfileInfo.getId());
         Assert.assertEquals(savedTenantProfile.getName(), foundTenantProfileInfo.getName());
@@ -129,7 +129,7 @@ public class TenantProfileControllerTest extends AbstractControllerTest {
         loginSysAdmin();
         TenantProfile tenantProfile = this.createTenantProfile("Tenant Profile 1");
         TenantProfile savedTenantProfile = doPost("/api/tenantProfile", tenantProfile, TenantProfile.class);
-        TenantProfile defaultTenantProfile = doPost("/api/tenantProfile/"+savedTenantProfile.getId().getId().toString()+"/default", TenantProfile.class);
+        TenantProfile defaultTenantProfile = doPost("/api/tenantProfile/" + savedTenantProfile.getId().getId().toString() + "/default", TenantProfile.class);
         Assert.assertNotNull(defaultTenantProfile);
         EntityInfo foundDefaultTenantProfile = doGet("/api/tenantProfileInfo/default", EntityInfo.class);
         Assert.assertNotNull(foundDefaultTenantProfile);
@@ -176,7 +176,7 @@ public class TenantProfileControllerTest extends AbstractControllerTest {
         Tenant tenant = new Tenant();
         tenant.setTitle("My tenant with tenant profile");
         tenant.setTenantProfileId(savedTenantProfile.getId());
-        Tenant savedTenant = doPost("/api/tenant", tenant, Tenant.class);
+        Tenant savedTenant = saveTenant(tenant);
 
         Mockito.reset(tbClusterService);
 
@@ -186,8 +186,7 @@ public class TenantProfileControllerTest extends AbstractControllerTest {
 
         testBroadcastEntityStateChangeEventNeverTenantProfile();
 
-        doDelete("/api/tenant/"+savedTenant.getId().getId().toString())
-                .andExpect(status().isOk());
+        deleteTenant(savedTenant.getId());
     }
 
     @Test
@@ -214,7 +213,7 @@ public class TenantProfileControllerTest extends AbstractControllerTest {
         List<TenantProfile> tenantProfiles = new ArrayList<>();
         PageLink pageLink = new PageLink(17);
         PageData<TenantProfile> pageData = doGetTypedWithPageLink("/api/tenantProfiles?",
-                new TypeReference<>(){}, pageLink);
+                new TypeReference<>() {}, pageLink);
         Assert.assertFalse(pageData.hasNext());
         Assert.assertEquals(1, pageData.getTotalElements());
         tenantProfiles.addAll(pageData.getData());
@@ -222,8 +221,8 @@ public class TenantProfileControllerTest extends AbstractControllerTest {
         Mockito.reset(tbClusterService);
 
         int cntEntity = 28;
-        for (int i=0;i<28;i++) {
-            TenantProfile tenantProfile = this.createTenantProfile("Tenant Profile"+i);
+        for (int i = 0; i < 28; i++) {
+            TenantProfile tenantProfile = this.createTenantProfile("Tenant Profile" + i);
             tenantProfiles.add(doPost("/api/tenantProfile", tenantProfile, TenantProfile.class));
         }
 
@@ -233,7 +232,7 @@ public class TenantProfileControllerTest extends AbstractControllerTest {
         pageLink = new PageLink(17);
         do {
             pageData = doGetTypedWithPageLink("/api/tenantProfiles?",
-                    new TypeReference<>(){}, pageLink);
+                    new TypeReference<>() {}, pageLink);
             loadedTenantProfiles.addAll(pageData.getData());
             if (pageData.hasNext()) {
                 pageLink = pageLink.nextPageLink();
@@ -256,7 +255,7 @@ public class TenantProfileControllerTest extends AbstractControllerTest {
 
         pageLink = new PageLink(17);
         pageData = doGetTypedWithPageLink("/api/tenantProfiles?",
-                new TypeReference<PageData<TenantProfile>>(){}, pageLink);
+                new TypeReference<PageData<TenantProfile>>() {}, pageLink);
         Assert.assertFalse(pageData.hasNext());
         Assert.assertEquals(1, pageData.getTotalElements());
 
@@ -269,13 +268,13 @@ public class TenantProfileControllerTest extends AbstractControllerTest {
         List<TenantProfile> tenantProfiles = new ArrayList<>();
         PageLink pageLink = new PageLink(17);
         PageData<TenantProfile> tenantProfilePageData = doGetTypedWithPageLink("/api/tenantProfiles?",
-                new TypeReference<>(){}, pageLink);
+                new TypeReference<>() {}, pageLink);
         Assert.assertFalse(tenantProfilePageData.hasNext());
         Assert.assertEquals(1, tenantProfilePageData.getTotalElements());
         tenantProfiles.addAll(tenantProfilePageData.getData());
 
-        for (int i=0;i<28;i++) {
-            TenantProfile tenantProfile = this.createTenantProfile("Tenant Profile"+i);
+        for (int i = 0; i < 28; i++) {
+            TenantProfile tenantProfile = this.createTenantProfile("Tenant Profile" + i);
             tenantProfiles.add(doPost("/api/tenantProfile", tenantProfile, TenantProfile.class));
         }
 
@@ -284,7 +283,7 @@ public class TenantProfileControllerTest extends AbstractControllerTest {
         PageData<EntityInfo> pageData;
         do {
             pageData = doGetTypedWithPageLink("/api/tenantProfileInfos?",
-                    new TypeReference<PageData<EntityInfo>>(){}, pageLink);
+                    new TypeReference<PageData<EntityInfo>>() {}, pageLink);
             loadedTenantProfileInfos.addAll(pageData.getData());
             if (pageData.hasNext()) {
                 pageLink = pageLink.nextPageLink();
@@ -308,7 +307,7 @@ public class TenantProfileControllerTest extends AbstractControllerTest {
 
         pageLink = new PageLink(17);
         pageData = doGetTypedWithPageLink("/api/tenantProfileInfos?",
-                new TypeReference<PageData<EntityInfo>>(){}, pageLink);
+                new TypeReference<PageData<EntityInfo>>() {}, pageLink);
         Assert.assertFalse(pageData.hasNext());
         Assert.assertEquals(1, pageData.getTotalElements());
     }
@@ -354,14 +353,14 @@ public class TenantProfileControllerTest extends AbstractControllerTest {
         ArgumentMatcher<TenantProfile> matcherTenantProfile = cntTime == 1 ? argument -> argument.equals(tenantProfile) :
                 argument -> argument.getClass().equals(TenantProfile.class);
         if (ComponentLifecycleEvent.DELETED.equals(event)) {
-            Mockito.verify(tbClusterService, times( cntTime)).onTenantProfileDelete(Mockito.argThat( matcherTenantProfile),
+            Mockito.verify(tbClusterService, times(cntTime)).onTenantProfileDelete(Mockito.argThat(matcherTenantProfile),
                     Mockito.isNull());
             testBroadcastEntityStateChangeEventNever(createEntityId_NULL_UUID(new Tenant()));
         } else {
-            Mockito.verify(tbClusterService, times( cntTime)).onTenantProfileChange(Mockito.argThat(matcherTenantProfile),
+            Mockito.verify(tbClusterService, times(cntTime)).onTenantProfileChange(Mockito.argThat(matcherTenantProfile),
                     Mockito.isNull());
             TenantProfileId tenantProfileIdId = cntTime == 1 ? tenantProfile.getId() : (TenantProfileId) createEntityId_NULL_UUID(tenantProfile);
-            testBroadcastEntityStateChangeEventTime(tenantProfileIdId, null,  cntTime);
+            testBroadcastEntityStateChangeEventTime(tenantProfileIdId, null, cntTime);
         }
         Mockito.reset(tbClusterService);
     }

application/src/test/java/org/thingsboard/server/controller/UserControllerTest.java

@@ -380,7 +380,7 @@ public class UserControllerTest extends AbstractControllerTest {
         //here created a new tenant despite already created on AbstractWebTest and then delete the tenant properly on the last line
         Tenant tenant = new Tenant();
         tenant.setTitle("My tenant with many admins");
-        Tenant savedTenant = doPost("/api/tenant", tenant, Tenant.class);
+        Tenant savedTenant = saveTenant(tenant);
         Assert.assertNotNull(savedTenant);
 
         TenantId tenantId = savedTenant.getId();
@@ -422,8 +422,7 @@ public class UserControllerTest extends AbstractControllerTest {
         assertThat(tenantAdmins).as("admins list size").hasSameSizeAs(loadedTenantAdmins);
         assertThat(tenantAdmins).as("admins list content").isEqualTo(loadedTenantAdmins);
 
-        doDelete("/api/tenant/" + tenantId.getId().toString())
-                .andExpect(status().isOk());
+        deleteTenant(tenantId);
 
         pageLink = new PageLink(33);
         pageData = doGetTypedWithPageLink("/api/tenant/" + tenantId.getId().toString() + "/users?",
@@ -669,7 +668,7 @@ public class UserControllerTest extends AbstractControllerTest {
         List<UserId> expectedCustomerUserIds = new ArrayList<>();
         expectedCustomerUserIds.add(customerUserId);
         for (int i = 0; i < 45; i++) {
-            User customerUser = createCustomerUser( customerId);
+            User customerUser = createCustomerUser(customerId);
             customerUser.setEmail(email + StringUtils.randomAlphanumeric((int) (5 + Math.random() * 10)) + "@thingsboard.org");
             User user = doPost("/api/user", customerUser, User.class);
             expectedCustomerUserIds.add(user.getId());
@@ -750,7 +749,7 @@ public class UserControllerTest extends AbstractControllerTest {
 
         String email = "testEmail1";
         for (int i = 0; i < 45; i++) {
-            User customerUser = createCustomerUser( customerId);
+            User customerUser = createCustomerUser(customerId);
             customerUser.setEmail(email + StringUtils.randomAlphanumeric((int) (5 + Math.random() * 10)) + "@thingsboard.org");
             doPost("/api/user", customerUser, User.class);
         }
@@ -993,8 +992,8 @@ public class UserControllerTest extends AbstractControllerTest {
         List<UserEmailInfo> usersInfo = getUsersInfo(pageLink);
 
         List<UserEmailInfo> expectedUserInfos = customerUsersContainingWord.stream().map(customerUser -> new UserEmailInfo(customerUser.getId(),
-                customerUser.getEmail(), customerUser.getFirstName() == null ? "" : customerUser.getFirstName(),
-                customerUser.getLastName() == null ? "" : customerUser.getLastName()))
+                        customerUser.getEmail(), customerUser.getFirstName() == null ? "" : customerUser.getFirstName(),
+                        customerUser.getLastName() == null ? "" : customerUser.getLastName()))
                 .sorted(userDataIdComparator).collect(Collectors.toList());
         usersInfo.sort(userDataIdComparator);
 
@@ -1046,8 +1045,8 @@ public class UserControllerTest extends AbstractControllerTest {
         List<UserEmailInfo> usersInfo = getUsersInfo(pageLink);
 
         List<UserEmailInfo> expectedUserInfos = usersContainingWord.stream().map(customerUser -> new UserEmailInfo(customerUser.getId(),
-                customerUser.getEmail(), customerUser.getFirstName() == null ? "" : customerUser.getFirstName(),
-                customerUser.getLastName() == null ? "" : customerUser.getLastName()))
+                        customerUser.getEmail(), customerUser.getFirstName() == null ? "" : customerUser.getFirstName(),
+                        customerUser.getLastName() == null ? "" : customerUser.getLastName()))
                 .sorted(userDataIdComparator).collect(Collectors.toList());
         usersInfo.sort(userDataIdComparator);
 

application/src/test/java/org/thingsboard/server/controller/WidgetTypeControllerTest.java

@@ -24,7 +24,6 @@ import org.junit.Test;
 import org.thingsboard.common.util.JacksonUtil;
 import org.thingsboard.server.common.data.Tenant;
 import org.thingsboard.server.common.data.User;
-import org.thingsboard.server.common.data.id.WidgetTypeId;
 import org.thingsboard.server.common.data.security.Authority;
 import org.thingsboard.server.common.data.widget.WidgetType;
 import org.thingsboard.server.common.data.widget.WidgetTypeDetails;
@@ -53,7 +52,7 @@ public class WidgetTypeControllerTest extends AbstractControllerTest {
 
         Tenant tenant = new Tenant();
         tenant.setTitle("My tenant");
-        savedTenant = doPost("/api/tenant", tenant, Tenant.class);
+        savedTenant = saveTenant(tenant);
         Assert.assertNotNull(savedTenant);
 
         tenantAdmin = new User();
@@ -70,8 +69,7 @@ public class WidgetTypeControllerTest extends AbstractControllerTest {
     public void afterTest() throws Exception {
         loginSysAdmin();
 
-        doDelete("/api/tenant/"+savedTenant.getId().getId().toString())
-                .andExpect(status().isOk());
+        deleteTenant(savedTenant.getId());
     }
 
     @Test
@@ -127,10 +125,10 @@ public class WidgetTypeControllerTest extends AbstractControllerTest {
         widgetType.setDescriptor(JacksonUtil.fromString("{ \"someKey\": \"someValue\" }", JsonNode.class));
         WidgetTypeDetails savedWidgetType = doPost("/api/widgetType", widgetType, WidgetTypeDetails.class);
 
-        doDelete("/api/widgetType/"+savedWidgetType.getId().getId().toString())
+        doDelete("/api/widgetType/" + savedWidgetType.getId().getId().toString())
                 .andExpect(status().isOk());
 
-        doGet("/api/widgetType/"+savedWidgetType.getId().getId().toString())
+        doGet("/api/widgetType/" + savedWidgetType.getId().getId().toString())
                 .andExpect(status().isNotFound());
     }
 
@@ -173,7 +171,7 @@ public class WidgetTypeControllerTest extends AbstractControllerTest {
         widgetsBundle = doPost("/api/widgetsBundle", widgetsBundle, WidgetsBundle.class);
 
         List<WidgetType> widgetTypes = new ArrayList<>();
-        for (int i=0;i<89;i++) {
+        for (int i = 0; i < 89; i++) {
             WidgetTypeDetails widgetType = new WidgetTypeDetails();
             widgetType.setName("Widget Type " + i);
             widgetType.setDescriptor(JacksonUtil.fromString("{ \"someKey\": \"someValue\" }", JsonNode.class));
@@ -184,7 +182,7 @@ public class WidgetTypeControllerTest extends AbstractControllerTest {
         doPost("/api/widgetsBundle/" + widgetsBundle.getId().getId().toString() + "/widgetTypes", widgetTypeIds);
 
         List<WidgetType> loadedWidgetTypes = doGetTyped("/api/widgetTypes?widgetsBundleId={widgetsBundleId}",
-                new TypeReference<>(){}, widgetsBundle.getId().getId().toString());
+                new TypeReference<>() {}, widgetsBundle.getId().getId().toString());
 
         Collections.sort(widgetTypes, idComparator);
         Collections.sort(loadedWidgetTypes, idComparator);
@@ -194,12 +192,12 @@ public class WidgetTypeControllerTest extends AbstractControllerTest {
         loginCustomerUser();
 
         List<WidgetType> loadedWidgetTypesCustomer = doGetTyped("/api/widgetTypes?widgetsBundleId={widgetsBundleId}",
-                new TypeReference<>(){}, widgetsBundle.getId().getId().toString());
+                new TypeReference<>() {}, widgetsBundle.getId().getId().toString());
         Collections.sort(loadedWidgetTypesCustomer, idComparator);
         Assert.assertEquals(widgetTypes, loadedWidgetTypesCustomer);
 
         List<WidgetTypeDetails> customerLoadedWidgetTypesDetails = doGetTyped("/api/widgetTypesDetails?widgetsBundleId={widgetsBundleId}",
-                new TypeReference<>(){}, widgetsBundle.getId().getId().toString());
+                new TypeReference<>() {}, widgetsBundle.getId().getId().toString());
         List<WidgetType> widgetTypesFromDetailsListCustomer = customerLoadedWidgetTypesDetails.stream().map(WidgetType::new).collect(Collectors.toList());
         Collections.sort(widgetTypesFromDetailsListCustomer, idComparator);
         Assert.assertEquals(widgetTypesFromDetailsListCustomer, loadedWidgetTypes);
@@ -207,12 +205,12 @@ public class WidgetTypeControllerTest extends AbstractControllerTest {
         loginSysAdmin();
 
         List<WidgetType> sysAdminLoadedWidgetTypes = doGetTyped("/api/widgetTypes?widgetsBundleId={widgetsBundleId}",
-                new TypeReference<>(){}, widgetsBundle.getId().getId().toString());
+                new TypeReference<>() {}, widgetsBundle.getId().getId().toString());
         Collections.sort(sysAdminLoadedWidgetTypes, idComparator);
         Assert.assertEquals(widgetTypes, sysAdminLoadedWidgetTypes);
 
         List<WidgetTypeDetails> sysAdminLoadedWidgetTypesDetails = doGetTyped("/api/widgetTypesDetails?widgetsBundleId={widgetsBundleId}",
-                new TypeReference<>(){}, widgetsBundle.getId().getId().toString());
+                new TypeReference<>() {}, widgetsBundle.getId().getId().toString());
         List<WidgetType> widgetTypesFromDetailsListSysAdmin = sysAdminLoadedWidgetTypesDetails.stream().map(WidgetType::new).collect(Collectors.toList());
         Collections.sort(widgetTypesFromDetailsListSysAdmin, idComparator);
         Assert.assertEquals(widgetTypesFromDetailsListSysAdmin, loadedWidgetTypes);
@@ -225,7 +223,7 @@ public class WidgetTypeControllerTest extends AbstractControllerTest {
         widgetType.setDescriptor(JacksonUtil.fromString("{ \"someKey\": \"someValue\" }", JsonNode.class));
         WidgetTypeDetails savedWidgetType = doPost("/api/widgetType", widgetType, WidgetTypeDetails.class);
         WidgetType foundWidgetType = doGet("/api/widgetType?fqn={fqn}",
-                WidgetType.class, "tenant."+savedWidgetType.getFqn());
+                WidgetType.class, "tenant." + savedWidgetType.getFqn());
         Assert.assertNotNull(foundWidgetType);
         Assert.assertEquals(new WidgetType(savedWidgetType), foundWidgetType);
     }

application/src/test/java/org/thingsboard/server/controller/WidgetsBundleControllerTest.java

@@ -53,7 +53,7 @@ public class WidgetsBundleControllerTest extends AbstractControllerTest {
 
         Tenant tenant = new Tenant();
         tenant.setTitle("My tenant");
-        savedTenant = doPost("/api/tenant", tenant, Tenant.class);
+        savedTenant = saveTenant(tenant);
         Assert.assertNotNull(savedTenant);
 
         tenantAdmin = new User();
@@ -70,8 +70,7 @@ public class WidgetsBundleControllerTest extends AbstractControllerTest {
     public void afterTest() throws Exception {
         loginSysAdmin();
 
-        doDelete("/api/tenant/" + savedTenant.getId().getId().toString())
-                .andExpect(status().isOk());
+        deleteTenant(savedTenant.getId());
     }
 
     @Test

application/src/test/java/org/thingsboard/server/edge/TenantEdgeTest.java

@@ -40,7 +40,7 @@ public class TenantEdgeTest extends AbstractEdgeTest {
         // updated edge tenant
         savedTenant.setTitle("Updated Title for Tenant Edge Test");
         edgeImitator.expectMessageAmount(2); // expect tenant and tenant profile update msg
-        savedTenant = doPost("/api/tenant", savedTenant, Tenant.class);
+        savedTenant = saveTenant(savedTenant);
         Assert.assertTrue(edgeImitator.waitForMessages());
         Optional<TenantUpdateMsg> tenantUpdateMsgOpt = edgeImitator.findMessageByType(TenantUpdateMsg.class);
         Assert.assertTrue(tenantUpdateMsgOpt.isPresent());
@@ -61,7 +61,7 @@ public class TenantEdgeTest extends AbstractEdgeTest {
         TenantProfile tenantProfile = createTenantProfile();
         savedTenant.setTenantProfileId(tenantProfile.getId());
         edgeImitator.expectMessageAmount(2); // expect tenant and tenant profile update msg
-        savedTenant = doPost("/api/tenant", savedTenant, Tenant.class);
+        savedTenant = saveTenant(savedTenant);
         Assert.assertTrue(edgeImitator.waitForMessages());
         tenantUpdateMsgOpt = edgeImitator.findMessageByType(TenantUpdateMsg.class);
         Assert.assertTrue(tenantUpdateMsgOpt.isPresent());

application/src/test/java/org/thingsboard/server/rules/flow/AbstractRuleEngineFlowIntegrationTest.java

@@ -27,9 +27,9 @@ import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.test.util.ReflectionTestUtils;
 import org.thingsboard.common.util.JacksonUtil;
 import org.thingsboard.rule.engine.flow.TbRuleChainInputNodeConfiguration;
-import org.thingsboard.rule.engine.util.TbMsgSource;
 import org.thingsboard.rule.engine.metadata.TbGetAttributesNode;
 import org.thingsboard.rule.engine.metadata.TbGetAttributesNodeConfiguration;
+import org.thingsboard.rule.engine.util.TbMsgSource;
 import org.thingsboard.server.actors.ActorSystemContext;
 import org.thingsboard.server.common.data.AttributeScope;
 import org.thingsboard.server.common.data.DataConstants;
@@ -62,7 +62,6 @@ import java.util.List;
 import java.util.stream.Collectors;
 
 import static org.mockito.Mockito.spy;
-import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status;
 
 /**
  * @author Valerii Sosliuk
@@ -103,7 +102,7 @@ public abstract class AbstractRuleEngineFlowIntegrationTest extends AbstractRule
 
         Tenant tenant = new Tenant();
         tenant.setTitle("My tenant");
-        savedTenant = doPost("/api/tenant", tenant, Tenant.class);
+        savedTenant = saveTenant(tenant);
         Assert.assertNotNull(savedTenant);
         ruleChainService.deleteRuleChainsByTenantId(savedTenant.getId());
 
@@ -121,7 +120,7 @@ public abstract class AbstractRuleEngineFlowIntegrationTest extends AbstractRule
     public void afterTest() throws Exception {
         loginSysAdmin();
         if (savedTenant != null) {
-            doDelete("/api/tenant/" + savedTenant.getId().getId().toString()).andExpect(status().isOk());
+            deleteTenant(savedTenant.getId());
         }
     }
 

application/src/test/java/org/thingsboard/server/service/entitiy/alarm/DefaultTbAlarmServiceTest.java

@@ -173,7 +173,7 @@ public class DefaultTbAlarmServiceTest {
         User user = new User();
         user.setEmail("testEmail@gmail.com");
         user.setId(new UserId(UUID.randomUUID()));
-        service.unassignDeletedUserAlarms(new TenantId(UUID.randomUUID()), user, System.currentTimeMillis());
+        service.unassignDeletedUserAlarms(new TenantId(UUID.randomUUID()), user.getId(), user.getTitle(), System.currentTimeMillis());
 
         ObjectNode commentNode = JacksonUtil.newObjectNode();
         commentNode.put("subtype", "ASSIGN");

application/src/test/java/org/thingsboard/server/service/housekeeper/HousekeeperServiceTest.java

@@ -0,0 +1,478 @@
+/**
+ * Copyright © 2016-2024 The Thingsboard Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.thingsboard.server.service.housekeeper;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.node.TextNode;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+import org.mockito.ArgumentMatcher;
+import org.mockito.Mockito;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.boot.test.mock.mockito.SpyBean;
+import org.springframework.test.context.TestPropertySource;
+import org.testcontainers.shaded.org.apache.commons.lang3.RandomStringUtils;
+import org.thingsboard.common.util.JacksonUtil;
+import org.thingsboard.rule.engine.metadata.TbGetAttributesNode;
+import org.thingsboard.rule.engine.metadata.TbGetAttributesNodeConfiguration;
+import org.thingsboard.server.common.data.ApiUsageState;
+import org.thingsboard.server.common.data.AttributeScope;
+import org.thingsboard.server.common.data.Device;
+import org.thingsboard.server.common.data.EventInfo;
+import org.thingsboard.server.common.data.alarm.Alarm;
+import org.thingsboard.server.common.data.alarm.AlarmSeverity;
+import org.thingsboard.server.common.data.alarm.EntityAlarm;
+import org.thingsboard.server.common.data.asset.Asset;
+import org.thingsboard.server.common.data.event.EventType;
+import org.thingsboard.server.common.data.event.LifecycleEvent;
+import org.thingsboard.server.common.data.housekeeper.HousekeeperTask;
+import org.thingsboard.server.common.data.housekeeper.HousekeeperTaskType;
+import org.thingsboard.server.common.data.id.AlarmId;
+import org.thingsboard.server.common.data.id.AssetId;
+import org.thingsboard.server.common.data.id.DeviceId;
+import org.thingsboard.server.common.data.id.EntityId;
+import org.thingsboard.server.common.data.id.RuleChainId;
+import org.thingsboard.server.common.data.id.RuleNodeId;
+import org.thingsboard.server.common.data.id.TenantId;
+import org.thingsboard.server.common.data.id.UserId;
+import org.thingsboard.server.common.data.kv.BaseAttributeKvEntry;
+import org.thingsboard.server.common.data.kv.BaseReadTsKvQuery;
+import org.thingsboard.server.common.data.kv.BasicTsKvEntry;
+import org.thingsboard.server.common.data.kv.StringDataEntry;
+import org.thingsboard.server.common.data.kv.TsKvEntry;
+import org.thingsboard.server.common.data.msg.TbNodeConnectionType;
+import org.thingsboard.server.common.data.page.PageLink;
+import org.thingsboard.server.common.data.page.TimePageLink;
+import org.thingsboard.server.common.data.relation.EntityRelation;
+import org.thingsboard.server.common.data.relation.RelationTypeGroup;
+import org.thingsboard.server.common.data.rule.RuleChain;
+import org.thingsboard.server.common.data.rule.RuleChainMetaData;
+import org.thingsboard.server.common.data.rule.RuleChainType;
+import org.thingsboard.server.common.data.rule.RuleNode;
+import org.thingsboard.server.controller.AbstractControllerTest;
+import org.thingsboard.server.dao.alarm.AlarmDao;
+import org.thingsboard.server.dao.alarm.AlarmService;
+import org.thingsboard.server.dao.attributes.AttributesService;
+import org.thingsboard.server.dao.entity.EntityServiceRegistry;
+import org.thingsboard.server.dao.event.EventService;
+import org.thingsboard.server.dao.relation.RelationService;
+import org.thingsboard.server.dao.rule.RuleChainService;
+import org.thingsboard.server.dao.service.DaoSqlTest;
+import org.thingsboard.server.dao.timeseries.TimeseriesService;
+import org.thingsboard.server.dao.usagerecord.ApiUsageStateDao;
+import org.thingsboard.server.gen.transport.TransportProtos.HousekeeperTaskProto;
+import org.thingsboard.server.gen.transport.TransportProtos.ToHousekeeperServiceMsg;
+import org.thingsboard.server.service.housekeeper.processor.TsHistoryDeletionTaskProcessor;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+import java.util.Optional;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
+import java.util.function.Function;
+import java.util.function.Predicate;
+import java.util.stream.Collectors;
+
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.awaitility.Awaitility.await;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.argThat;
+import static org.mockito.Mockito.doCallRealMethod;
+import static org.mockito.Mockito.doThrow;
+import static org.mockito.Mockito.never;
+import static org.mockito.Mockito.verify;
+import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status;
+
+@DaoSqlTest
+@TestPropertySource(properties = {
+        "transport.http.enabled=true",
+        "queue.core.housekeeper.task-reprocessing-delay-ms=2000",
+        "queue.core.housekeeper.poll-interval-ms=1000",
+        "queue.core.housekeeper.max-reprocessing-attempts=5"
+})
+public class HousekeeperServiceTest extends AbstractControllerTest {
+
+    @SpyBean
+    private HousekeeperService housekeeperService;
+    @SpyBean
+    private HousekeeperReprocessingService housekeeperReprocessingService;
+    @Autowired
+    private EventService eventService;
+    @Autowired
+    private TimeseriesService timeseriesService;
+    @Autowired
+    private AttributesService attributesService;
+    @Autowired
+    private RuleChainService ruleChainService;
+    @Autowired
+    private AlarmService alarmService;
+    @Autowired
+    private AlarmDao alarmDao;
+    @Autowired
+    private RelationService relationService;
+    @Autowired
+    private ApiUsageStateDao apiUsageStateDao;
+    @Autowired
+    private EntityServiceRegistry entityServiceRegistry;
+    @SpyBean
+    private TsHistoryDeletionTaskProcessor tsHistoryDeletionTaskProcessor;
+
+    private TenantId tenantId;
+
+    private static final String TELEMETRY_KEY = "telemetry1";
+    private static final String ATTRIBUTE_KEY = "_attribute1";
+    private static final String KV_VALUE = "ewfewfwef";
+
+    @Before
+    public void setUp() throws Exception {
+        loginTenantAdmin();
+        this.tenantId = super.tenantId;
+    }
+
+    @After
+    public void tearDown() throws Exception {
+    }
+
+    @Test
+    public void whenDeviceIsDeleted_thenCleanUpRelatedData() throws Exception {
+        Device device = createDevice("test", "test");
+        createRelatedData(device.getId());
+
+        doDelete("/api/device/" + device.getId()).andExpect(status().isOk());
+
+        await().atMost(10, TimeUnit.SECONDS).untilAsserted(() -> {
+            verifyNoRelatedData(device.getId());
+        });
+    }
+
+    @Test
+    public void whenRuleChainIsDeleted_thenCleanUpRelatedData() throws Exception {
+        RuleChainMetaData ruleChainMetaData = createRuleChain();
+        RuleChainId ruleChainId = ruleChainMetaData.getRuleChainId();
+        RuleNodeId ruleNode1Id = ruleChainMetaData.getNodes().get(0).getId();
+        RuleNodeId ruleNode2Id = ruleChainMetaData.getNodes().get(1).getId();
+        createRelatedData(ruleChainId);
+        createRelatedData(ruleNode1Id);
+        createRelatedData(ruleNode2Id);
+
+        doDelete("/api/ruleChain/" + ruleChainId).andExpect(status().isOk());
+
+        await().atMost(10, TimeUnit.SECONDS).untilAsserted(() -> {
+            verifyNoRelatedData(ruleNode1Id);
+            verifyNoRelatedData(ruleNode2Id);
+            verifyNoRelatedData(ruleChainId);
+        });
+    }
+
+    @Test
+    public void whenUserIsDeleted_thenCleanUpRelatedData() throws Exception {
+        Device device = createDevice("test", "test");
+        UserId userId = customerUserId;
+        createRelatedData(userId);
+        Alarm alarm = Alarm.builder()
+                .type("test")
+                .tenantId(tenantId)
+                .originator(device.getId())
+                .severity(AlarmSeverity.MAJOR)
+                .build();
+        alarm = doPost("/api/alarm", alarm, Alarm.class);
+        AlarmId alarmId = alarm.getId();
+        alarm = doPost("/api/alarm/" + alarmId + "/assign/" + userId, "", Alarm.class);
+        assertThat(alarm.getAssigneeId()).isEqualTo(userId);
+        assertThat(alarmService.findAlarmIdsByAssigneeId(tenantId, userId, new PageLink(100)).getData()).isNotEmpty();
+
+        doDelete("/api/user/" + userId).andExpect(status().isOk());
+
+        await().atMost(10, TimeUnit.SECONDS).untilAsserted(() -> {
+            verifyNoRelatedData(userId);
+            assertThat(alarmService.findAlarmById(tenantId, alarmId).getAssigneeId()).isNull();
+        });
+    }
+
+    @Test
+    public void whenDeviceIsDeleted_thenDeleteAllAlarms() throws Exception {
+        Device device = createDevice("test", "test");
+        for (int i = 1; i <= 1000; i++) {
+            createAlarm(device.getId());
+        }
+
+        doDelete("/api/device/" + device.getId()).andExpect(status().isOk());
+
+        await().atMost(30, TimeUnit.SECONDS).untilAsserted(() -> {
+            verifyNoAlarms(device.getId());
+        });
+    }
+
+    @Test
+    public void whenTenantIsDeleted_thenDeleteAllEntitiesAndCleanUpRelatedData() throws Exception {
+        loginDifferentTenant();
+        tenantId = differentTenantId;
+
+        createRelatedData(tenantId);
+        createDifferentTenantCustomer();
+        createRelatedData(differentTenantCustomerId);
+        loginDifferentTenant();
+
+        List<DeviceId> devices = new ArrayList<>();
+        for (int i = 1; i <= 300; i++) {
+            Device device = createDevice("test" + i, "test" + i);
+            devices.add(device.getId());
+        }
+        DeviceId firstDevice = devices.get(0);
+        createRelatedData(firstDevice);
+        DeviceId lastDevice = devices.get(devices.size() - 1);
+        createRelatedData(lastDevice);
+
+        Asset asset = createAsset();
+        createRelatedData(asset.getId());
+        createRelation(firstDevice, asset.getId());
+        createAlarm(firstDevice, asset.getId());
+
+        RuleChainMetaData ruleChainMetaData = createRuleChain();
+        RuleChainId ruleChainId = ruleChainMetaData.getRuleChainId();
+        RuleNodeId ruleNode1Id = ruleChainMetaData.getNodes().get(0).getId();
+        RuleNodeId ruleNode2Id = ruleChainMetaData.getNodes().get(1).getId();
+        createRelatedData(ruleChainId);
+        createRelatedData(ruleNode1Id);
+        createRelatedData(ruleNode2Id);
+
+        UserId userId = savedDifferentTenantUser.getId();
+        createRelatedData(userId);
+
+        ApiUsageState tenantApiUsageState = apiUsageStateDao.findApiUsageStateByEntityId(differentTenantId);
+
+        loginSysAdmin();
+        deleteDifferentTenant();
+
+        await().atMost(60, TimeUnit.SECONDS).pollInterval(1, TimeUnit.SECONDS).untilAsserted(() -> {
+            for (DeviceId deviceId : devices) {
+                verifyNoRelatedData(deviceId);
+            }
+            verifyNoRelatedData(asset.getId());
+            verifyNoRelatedData(ruleNode1Id);
+            verifyNoRelatedData(ruleNode2Id);
+            verifyNoRelatedData(ruleChainId);
+            verifyNoRelatedData(userId);
+            verifyNoRelatedData(differentTenantCustomerId);
+            verifyNoRelatedData(tenantApiUsageState.getId());
+            verifyNoRelatedData(tenantId);
+        });
+    }
+
+    @Test
+    public void whenTaskProcessingFails_thenReprocess() throws Exception {
+        TimeoutException error = new TimeoutException("Test timeout");
+        doThrow(error).when(tsHistoryDeletionTaskProcessor).process(any());
+
+        Device device = createDevice("test", "test");
+        createRelatedData(device.getId());
+
+        doDelete("/api/device/" + device.getId()).andExpect(status().isOk());
+
+        int attempts = 2;
+        await().atMost(30, TimeUnit.SECONDS).pollInterval(1, TimeUnit.SECONDS).untilAsserted(() -> {
+            for (int i = 0; i <= attempts; i++) {
+                int attempt = i;
+                verify(housekeeperReprocessingService).submitForReprocessing(argThat(getTaskMatcher(device.getId(), HousekeeperTaskType.DELETE_TS_HISTORY,
+                        task -> task.getAttempt() == attempt)), argThat(e -> e.getMessage().equals(error.getMessage())));
+            }
+        });
+
+        assertThat(getTimeseriesHistory(device.getId())).isNotEmpty();
+        doCallRealMethod().when(tsHistoryDeletionTaskProcessor).process(any());
+        await().atMost(30, TimeUnit.SECONDS).untilAsserted(() -> {
+            assertThat(getTimeseriesHistory(device.getId())).isEmpty();
+        });
+    }
+
+    @Test
+    public void whenReprocessingAttemptsExceeded_thenDropTheTask() throws Exception {
+        TimeoutException error = new TimeoutException("Test timeout");
+        doThrow(error).when(tsHistoryDeletionTaskProcessor).process(any());
+
+        Device device = createDevice("test", "test");
+        createRelatedData(device.getId());
+
+        doDelete("/api/device/" + device.getId()).andExpect(status().isOk());
+
+        int maxAttempts = 5;
+        await().atMost(30, TimeUnit.SECONDS).untilAsserted(() -> {
+            for (int i = 1; i <= maxAttempts; i++) {
+                verifyTaskProcessing(device.getId(), HousekeeperTaskType.DELETE_TS_HISTORY, i);
+            }
+        });
+
+        Mockito.clearInvocations(housekeeperService);
+        doCallRealMethod().when(tsHistoryDeletionTaskProcessor).process(any());
+        TimeUnit.SECONDS.sleep(2);
+        verify(housekeeperService, never()).processTask(argThat(getTaskMatcher(device.getId(), HousekeeperTaskType.DELETE_TS_HISTORY, null)));
+    }
+
+    private void verifyTaskProcessing(EntityId entityId, HousekeeperTaskType taskType, int expectedAttempt) throws Exception {
+        verify(housekeeperService).processTask(argThat(getTaskMatcher(entityId, taskType, task -> task.getAttempt() == expectedAttempt)));
+    }
+
+    private ArgumentMatcher<ToHousekeeperServiceMsg> getTaskMatcher(EntityId entityId, HousekeeperTaskType taskType,
+                                                                    Predicate<HousekeeperTaskProto> additionalCheck) {
+        return msg -> {
+            HousekeeperTask task = JacksonUtil.fromString(msg.getTask().getValue(), HousekeeperTask.class);
+            return task.getEntityId().equals(entityId) && task.getTaskType() == taskType && (additionalCheck == null || additionalCheck.test(msg.getTask()));
+        };
+    }
+
+    private void createRelatedData(EntityId entityId) throws Exception {
+        createTelemetry(entityId);
+        for (AttributeScope scope : AttributeScope.values()) {
+            createAttribute(entityId, scope, scope + ATTRIBUTE_KEY);
+        }
+        createEvent(entityId);
+    }
+
+    private void verifyNoRelatedData(EntityId entityId) throws Exception {
+        assertThat(entityServiceRegistry.getServiceByEntityType(entityId.getEntityType()).findEntity(tenantId, entityId)).isEmpty();
+
+        assertThat(getLatestTelemetry(entityId)).isNull();
+        assertThat(getTimeseriesHistory(entityId)).isEmpty();
+        for (AttributeScope scope : AttributeScope.values()) {
+            assertThat(attributesService.findAll(tenantId, entityId, scope).get()).isEmpty();
+        }
+        assertThat(getEvents(entityId)).isEmpty();
+        assertThat(alarmDao.findEntityAlarmRecordsByEntityId(tenantId, entityId)).isEmpty();
+        verifyNoAlarms(entityId);
+        assertThat(relationService.findByTo(tenantId, entityId, RelationTypeGroup.COMMON)).isEmpty();
+        assertThat(relationService.findByFrom(tenantId, entityId, RelationTypeGroup.COMMON)).isEmpty();
+    }
+
+    private void verifyNoAlarms(EntityId entityId) {
+        assertThat(alarmService.findAlarmIdsByOriginatorId(tenantId, entityId, 0, null, 10)).isEmpty();
+    }
+
+    private void createAttribute(EntityId entityId, AttributeScope scope, String key) throws Exception {
+        attributesService.save(tenantId, entityId, scope, new BaseAttributeKvEntry(System.currentTimeMillis(), new StringDataEntry(key, KV_VALUE))).get();
+    }
+
+    private void createTelemetry(EntityId entityId) throws Exception {
+        timeseriesService.save(tenantId, entityId, new BasicTsKvEntry(System.currentTimeMillis(), new StringDataEntry(TELEMETRY_KEY, KV_VALUE))).get();
+    }
+
+    private void createEvent(EntityId entityId) {
+        LifecycleEvent event = LifecycleEvent.builder()
+                .tenantId(tenantId)
+                .entityId(entityId.getId())
+                .serviceId("test")
+                .lcEventType("test")
+                .success(true)
+                .build();
+        eventService.saveAsync(event);
+        await().atMost(10, TimeUnit.SECONDS)
+                .until(() -> !getEvents(entityId).isEmpty());
+
+    }
+
+    private void createRelation(DeviceId to, AssetId from) {
+        EntityRelation relation = new EntityRelation(from, to, EntityRelation.CONTAINS_TYPE, RelationTypeGroup.COMMON);
+        relationService.saveRelation(tenantId, relation);
+    }
+
+    private void createAlarm(DeviceId deviceId, EntityId propagatedEntityId) {
+        Alarm alarm = doPost("/api/alarm", Alarm.builder()
+                .tenantId(tenantId)
+                .originator(deviceId)
+                .severity(AlarmSeverity.CRITICAL)
+                .type("test alarm for " + deviceId)
+                .propagate(true)
+                .build(), Alarm.class);
+
+        List<EntityAlarm> entityAlarms = alarmDao.findEntityAlarmRecords(tenantId, alarm.getId());
+        assertThat(entityAlarms).anyMatch(entityAlarm -> entityAlarm.getEntityId().equals(deviceId) && entityAlarm.getAlarmType().equals(alarm.getType()));
+        assertThat(entityAlarms).anyMatch(entityAlarm -> entityAlarm.getEntityId().equals(propagatedEntityId) && entityAlarm.getAlarmType().equals(alarm.getType()));
+        assertThat(alarmService.findAlarmIdsByOriginatorId(tenantId, deviceId, 0, null, 10)).isNotEmpty();
+    }
+
+    private void createAlarm(DeviceId deviceId) {
+        Alarm alarm = doPost("/api/alarm", Alarm.builder()
+                .tenantId(tenantId)
+                .originator(deviceId)
+                .severity(AlarmSeverity.CRITICAL)
+                .type("test alarm for " + deviceId + " " + RandomStringUtils.randomAlphabetic(10))
+                .build(), Alarm.class);
+        assertThat(alarmService.findAlarmIdsByOriginatorId(tenantId, deviceId, 0, null, 10)).isNotEmpty();
+    }
+
+    private TsKvEntry getLatestTelemetry(EntityId entityId) throws Exception {
+        return timeseriesService.findLatest(tenantId, entityId, HousekeeperServiceTest.TELEMETRY_KEY).get().orElse(null);
+    }
+
+    private List<TsKvEntry> getTimeseriesHistory(EntityId entityId) throws Exception {
+        return timeseriesService.findAll(tenantId, entityId, List.of(new BaseReadTsKvQuery(HousekeeperServiceTest.TELEMETRY_KEY, 0, System.currentTimeMillis(), 10, "DESC"))).get();
+    }
+
+    private List<EventInfo> getEvents(EntityId entityId) {
+        return eventService.findEvents(tenantId, entityId, EventType.LC_EVENT, new TimePageLink(100)).getData()
+                .stream().filter(event -> Optional.ofNullable(event.getBody()).map(body -> body.get("event"))
+                        .map(JsonNode::asText).orElse("").equals("test"))
+                .collect(Collectors.toList());
+    }
+
+    private Asset createAsset() {
+        Asset asset = new Asset();
+        asset.setName("test");
+        asset.setType("test");
+        return doPost("/api/asset", asset, Asset.class);
+    }
+
+    private RuleChainMetaData createRuleChain() {
+        RuleChain ruleChain = new RuleChain();
+        ruleChain.setTenantId(tenantId);
+        ruleChain.setName("Test");
+        ruleChain.setType(RuleChainType.CORE);
+        ruleChain.setDebugMode(true);
+        ruleChain.setConfiguration(JacksonUtil.newObjectNode().set("a", new TextNode("b")));
+        ruleChain = ruleChainService.saveRuleChain(ruleChain);
+        RuleChainId ruleChainId = ruleChain.getId();
+
+        RuleChainMetaData metaData = new RuleChainMetaData();
+        metaData.setRuleChainId(ruleChainId);
+
+        RuleNode ruleNode1 = new RuleNode();
+        ruleNode1.setName("Simple Rule Node 1");
+        ruleNode1.setType(org.thingsboard.rule.engine.metadata.TbGetAttributesNode.class.getName());
+        ruleNode1.setConfigurationVersion(TbGetAttributesNode.class.getAnnotation(org.thingsboard.rule.engine.api.RuleNode.class).version());
+        ruleNode1.setDebugMode(true);
+        TbGetAttributesNodeConfiguration configuration1 = new TbGetAttributesNodeConfiguration();
+        configuration1.setServerAttributeNames(Collections.singletonList("serverAttributeKey1"));
+        ruleNode1.setConfiguration(JacksonUtil.valueToTree(configuration1));
+
+        RuleNode ruleNode2 = new RuleNode();
+        ruleNode2.setName("Simple Rule Node 2");
+        ruleNode2.setType(org.thingsboard.rule.engine.metadata.TbGetAttributesNode.class.getName());
+        ruleNode2.setConfigurationVersion(TbGetAttributesNode.class.getAnnotation(org.thingsboard.rule.engine.api.RuleNode.class).version());
+        ruleNode2.setDebugMode(true);
+        TbGetAttributesNodeConfiguration configuration2 = new TbGetAttributesNodeConfiguration();
+        configuration2.setServerAttributeNames(Collections.singletonList("serverAttributeKey2"));
+        ruleNode2.setConfiguration(JacksonUtil.valueToTree(configuration2));
+
+        metaData.setNodes(Arrays.asList(ruleNode1, ruleNode2));
+        metaData.setFirstNodeIndex(0);
+        metaData.addConnectionInfo(0, 1, TbNodeConnectionType.SUCCESS);
+        ruleChainService.saveRuleChainMetaData(tenantId, metaData, Function.identity());
+        return ruleChainService.loadRuleChainMetaData(tenantId, ruleChainId);
+    }
+
+}

application/src/test/java/org/thingsboard/server/service/notification/NotificationApiTest.java

@@ -323,8 +323,10 @@ public class NotificationApiTest extends AbstractNotificationApiTest {
 
         deleteDifferentTenant();
 
-        assertThat(notificationRequestService.findNotificationRequestsByTenantIdAndOriginatorType(tenantId, EntityType.USER, new PageLink(1)).getTotalElements())
-                .isZero();
+        await().atMost(TIMEOUT, TimeUnit.SECONDS).untilAsserted(() -> {
+            assertThat(notificationRequestService.findNotificationRequestsByTenantIdAndOriginatorType(tenantId, EntityType.USER, new PageLink(1)).getTotalElements())
+                    .isZero();
+        });
     }
 
     @Test

application/src/test/java/org/thingsboard/server/service/notification/NotificationTargetApiTest.java

@@ -38,8 +38,10 @@ import org.thingsboard.server.dao.service.DaoSqlTest;
 
 import java.util.Collections;
 import java.util.List;
+import java.util.concurrent.TimeUnit;
 
 import static org.assertj.core.api.Assertions.assertThat;
+import static org.awaitility.Awaitility.await;
 import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status;
 
 @DaoSqlTest
@@ -148,7 +150,9 @@ public class NotificationTargetApiTest extends AbstractNotificationApiTest {
         assertThat(notificationTargetDao.findByTenantIdAndSupportedNotificationTypeAndPageLink(differentTenantId, NotificationType.GENERAL, new PageLink(10)).getData()).isNotEmpty();
 
         deleteDifferentTenant();
-        assertThat(notificationTargetDao.findByTenantIdAndPageLink(differentTenantId, new PageLink(10)).getData()).isEmpty();
+        await().atMost(TIMEOUT, TimeUnit.SECONDS).untilAsserted(() -> {
+            assertThat(notificationTargetDao.findByTenantIdAndPageLink(differentTenantId, new PageLink(10)).getData()).isEmpty();
+        });
     }
 
     @Test

application/src/test/java/org/thingsboard/server/service/resource/sql/BaseTbResourceServiceTest.java

@@ -51,7 +51,6 @@ import static org.assertj.core.api.Assertions.assertThatThrownBy;
 import static org.junit.Assert.assertArrayEquals;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertThrows;
-import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status;
 
 @DaoSqlTest
 public class BaseTbResourceServiceTest extends AbstractControllerTest {
@@ -129,7 +128,7 @@ public class BaseTbResourceServiceTest extends AbstractControllerTest {
 
         Tenant tenant = new Tenant();
         tenant.setTitle("My tenant");
-        savedTenant = doPost("/api/tenant", tenant, Tenant.class);
+        savedTenant = saveTenant(tenant);
         tenantId = savedTenant.getId();
         Assert.assertNotNull(savedTenant);
 
@@ -147,8 +146,7 @@ public class BaseTbResourceServiceTest extends AbstractControllerTest {
     public void afterTest() throws Exception {
         loginSysAdmin();
 
-        doDelete("/api/tenant/" + savedTenant.getId().getId().toString())
-                .andExpect(status().isOk());
+        deleteTenant(savedTenant.getId());
     }
 
     @Test
@@ -397,7 +395,7 @@ public class BaseTbResourceServiceTest extends AbstractControllerTest {
         loginSysAdmin();
         Tenant tenant = new Tenant();
         tenant.setTitle("Test tenant");
-        tenant = doPost("/api/tenant", tenant, Tenant.class);
+        tenant = saveTenant(tenant);
 
         TenantId tenantId = tenant.getId();
 
@@ -441,8 +439,7 @@ public class BaseTbResourceServiceTest extends AbstractControllerTest {
         Assert.assertFalse(pageData.hasNext());
         Assert.assertTrue(pageData.getData().isEmpty());
 
-        doDelete("/api/tenant/" + tenantId.getId().toString())
-                .andExpect(status().isOk());
+        deleteTenant(tenantId);
     }
 
     @Test
@@ -451,7 +448,7 @@ public class BaseTbResourceServiceTest extends AbstractControllerTest {
 
         Tenant tenant = new Tenant();
         tenant.setTitle("Test tenant");
-        tenant = doPost("/api/tenant", tenant, Tenant.class);
+        tenant = saveTenant(tenant);
 
         TenantId tenantId = tenant.getId();
 
@@ -518,8 +515,7 @@ public class BaseTbResourceServiceTest extends AbstractControllerTest {
         Assert.assertFalse(pageData.hasNext());
         Assert.assertTrue(pageData.getData().isEmpty());
 
-        doDelete("/api/tenant/" + tenantId.getId().toString())
-                .andExpect(status().isOk());
+        deleteTenant(tenantId);
     }
 
 }

application/src/test/java/org/thingsboard/server/service/sql/SequentialTimeseriesPersistenceTest.java

@@ -50,8 +50,6 @@ import java.util.concurrent.ExecutionException;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.TimeoutException;
 
-import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status;
-
 @DaoSqlTest
 public class SequentialTimeseriesPersistenceTest extends AbstractControllerTest {
 
@@ -77,7 +75,7 @@ public class SequentialTimeseriesPersistenceTest extends AbstractControllerTest
 
         Tenant tenant = new Tenant();
         tenant.setTitle("My tenant");
-        savedTenant = doPost("/api/tenant", tenant, Tenant.class);
+        savedTenant = saveTenant(tenant);
         Assert.assertNotNull(savedTenant);
 
         tenantAdmin = new User();
@@ -93,7 +91,7 @@ public class SequentialTimeseriesPersistenceTest extends AbstractControllerTest
     @After
     public void afterTest() throws Exception {
         loginSysAdmin();
-        doDelete("/api/tenant/" + savedTenant.getId().getId().toString()).andExpect(status().isOk());
+        deleteTenant(savedTenant.getId());
     }
 
     @Test

application/src/test/resources/logback-test.xml

@@ -31,6 +31,7 @@
     <!--    To reduce logs -->
     <logger name="org.apache.catalina.loader.WebappClassLoaderBase" level="ERROR" />
     <logger name="org.thingsboard.server.service.queue.DefaultTbClusterService" level="ERROR" />
+    <logger name="org.thingsboard.server.service.security.auth.jwt.settings.DefaultJwtSettingsService" level="ERROR" />
 
     <!-- Coap client context debug for the test scope -->
     <!--    <logger name="org.thingsboard.server.transport.coap.client.DefaultCoapClientContext" level="TRACE" />-->

common/dao-api/src/main/java/org/thingsboard/server/dao/alarm/AlarmService.java

@@ -38,10 +38,13 @@ import org.thingsboard.server.common.data.page.PageLink;
 import org.thingsboard.server.common.data.query.AlarmCountQuery;
 import org.thingsboard.server.common.data.query.AlarmData;
 import org.thingsboard.server.common.data.query.AlarmDataQuery;
+import org.thingsboard.server.common.data.util.TbPair;
 import org.thingsboard.server.dao.entity.EntityDaoService;
 
 import java.util.Collection;
+import java.util.List;
 import java.util.Set;
+import java.util.UUID;
 
 
 public interface AlarmService extends EntityDaoService {
@@ -106,9 +109,9 @@ public interface AlarmService extends EntityDaoService {
 
     PageData<AlarmId> findAlarmIdsByAssigneeId(TenantId tenantId, UserId userId, PageLink pageLink);
 
-    PageData<AlarmId> findAlarmIdsByOriginatorId(TenantId tenantId, EntityId originatorId, PageLink pageLink);
+    List<TbPair<UUID, Long>> findAlarmIdsByOriginatorId(TenantId tenantId, EntityId originatorId, long createdTimeOffset, AlarmId idOffset, int limit);
 
-    void deleteEntityAlarmRelations(TenantId tenantId, EntityId entityId);
+    int deleteEntityAlarmRecords(TenantId tenantId, EntityId entityId);
 
     void deleteEntityAlarmRecordsByTenantId(TenantId tenantId);
 

common/dao-api/src/main/java/org/thingsboard/server/dao/attributes/AttributesService.java

@@ -71,4 +71,6 @@ public interface AttributesService {
 
     List<String> findAllKeysByEntityIds(TenantId tenantId, List<EntityId> entityIds, String scope);
 
+    int removeAllByEntityId(TenantId tenantId, EntityId entityId);
+
 }

common/dao-api/src/main/java/org/thingsboard/server/dao/entity/EntityDaoService.java

@@ -30,10 +30,14 @@ public interface EntityDaoService {
         throw new IllegalArgumentException("Not implemented for " + getEntityType());
     }
 
-    default void deleteEntity(TenantId tenantId, EntityId id) {
+    default void deleteEntity(TenantId tenantId, EntityId id, boolean force) {
         throw new IllegalArgumentException(getEntityType().getNormalName() + " deletion not supported");
     }
 
+    default void deleteByTenantId(TenantId tenantId) {
+        throw new IllegalArgumentException("Deletion by tenant id not supported for " + getEntityType().getNormalName());
+    }
+
     EntityType getEntityType();
 
 }

common/dao-api/src/main/java/org/thingsboard/server/dao/notification/NotificationTemplateService.java

@@ -32,6 +32,8 @@ public interface NotificationTemplateService {
 
     PageData<NotificationTemplate> findNotificationTemplatesByTenantIdAndNotificationTypes(TenantId tenantId, List<NotificationType> notificationTypes, PageLink pageLink);
 
+    int countNotificationTemplatesByTenantIdAndNotificationTypes(TenantId tenantId, List<NotificationType> notificationTypes);
+
     void deleteNotificationTemplateById(TenantId tenantId, NotificationTemplateId id);
 
     void deleteNotificationTemplatesByTenantId(TenantId tenantId);

common/dao-api/src/main/java/org/thingsboard/server/dao/queue/QueueStatsService.java

@@ -32,6 +32,4 @@ public interface QueueStatsService extends EntityDaoService {
 
     List<QueueStats> findByTenantId(TenantId tenantId);
 
-    void deleteByTenantId(TenantId tenantId);
-
-}
+}

common/dao-api/src/main/java/org/thingsboard/server/dao/usagerecord/ApiUsageStateService.java

@@ -31,8 +31,6 @@ public interface ApiUsageStateService extends EntityDaoService {
 
     ApiUsageState findApiUsageStateByEntityId(EntityId entityId);
 
-    void deleteApiUsageStateByTenantId(TenantId tenantId);
-
     void deleteApiUsageStateByEntityId(EntityId entityId);
 
     ApiUsageState findApiUsageStateById(TenantId tenantId, ApiUsageStateId id);

common/dao-api/src/main/java/org/thingsboard/server/dao/user/UserService.java

@@ -77,6 +77,8 @@ public interface UserService extends EntityDaoService {
 
     void deleteTenantAdmins(TenantId tenantId);
 
+    void deleteAllByTenantId(TenantId tenantId);
+
     PageData<User> findCustomerUsers(TenantId tenantId, CustomerId customerId, PageLink pageLink);
 
     PageData<User> findUsersByCustomerIds(TenantId tenantId, List<CustomerId> customerIds, PageLink pageLink);

common/data/src/main/java/org/thingsboard/server/common/data/AttributeScope.java

@@ -17,6 +17,10 @@ package org.thingsboard.server.common.data;
 
 import lombok.Getter;
 
+import java.util.Arrays;
+import java.util.Map;
+import java.util.stream.Collectors;
+
 public enum AttributeScope {
 
     CLIENT_SCOPE(1),
@@ -25,8 +29,15 @@ public enum AttributeScope {
     @Getter
     private final int id;
 
+    private static final Map<Integer, AttributeScope> values = Arrays.stream(values())
+            .collect(Collectors.toMap(AttributeScope::getId, scope -> scope));
+
     AttributeScope(int id) {
         this.id = id;
     }
 
+    public static AttributeScope valueOf(int id) {
+        return values.get(id);
+    }
+
 }

common/data/src/main/java/org/thingsboard/server/common/data/EntityType.java

@@ -20,7 +20,6 @@ import org.apache.commons.lang3.StringUtils;
 
 import java.util.EnumSet;
 import java.util.List;
-import java.util.stream.Collectors;
 
 /**
  * @author Andrew Shvayka
@@ -28,51 +27,57 @@ import java.util.stream.Collectors;
 public enum EntityType {
     TENANT(1),
     CUSTOMER(2),
-    USER(3),
+    USER(3, "tb_user"),
     DASHBOARD(4),
     ASSET(5),
     DEVICE(6),
-    ALARM (7),
-    RULE_CHAIN (11),
-    RULE_NODE (12),
-
-    ENTITY_VIEW (15) {
+    ALARM(7),
+    RULE_CHAIN(11),
+    RULE_NODE(12),
+    ENTITY_VIEW(15) {
         // backward compatibility for TbOriginatorTypeSwitchNode to return correct rule node connection.
         @Override
-        public String getNormalName () {
+        public String getNormalName() {
             return "Entity View";
         }
     },
-    WIDGETS_BUNDLE (16),
-    WIDGET_TYPE (17),
-    TENANT_PROFILE (20),
-    DEVICE_PROFILE (21),
-    ASSET_PROFILE (22),
-    API_USAGE_STATE (23),
-    TB_RESOURCE (24),
-    OTA_PACKAGE (25),
-    EDGE (26),
-    RPC (27),
-    QUEUE (28),
-    NOTIFICATION_TARGET (29),
-    NOTIFICATION_TEMPLATE (30),
-    NOTIFICATION_REQUEST (31),
-    NOTIFICATION (32),
-    NOTIFICATION_RULE (33),
+    WIDGETS_BUNDLE(16),
+    WIDGET_TYPE(17),
+    TENANT_PROFILE(20),
+    DEVICE_PROFILE(21),
+    ASSET_PROFILE(22),
+    API_USAGE_STATE(23),
+    TB_RESOURCE(24, "resource"),
+    OTA_PACKAGE(25),
+    EDGE(26),
+    RPC(27),
+    QUEUE(28),
+    NOTIFICATION_TARGET(29),
+    NOTIFICATION_TEMPLATE(30),
+    NOTIFICATION_REQUEST(31),
+    NOTIFICATION(32),
+    NOTIFICATION_RULE(33),
     QUEUE_STATS(34);
 
     @Getter
     private final int protoNumber; // Corresponds to EntityTypeProto
+    @Getter
+    private final String tableName;
+    @Getter
+    private final String normalName = StringUtils.capitalize(StringUtils.removeStart(name(), "TB_")
+            .toLowerCase().replaceAll("_", " "));
+
+    public static final List<String> NORMAL_NAMES = EnumSet.allOf(EntityType.class).stream()
+            .map(EntityType::getNormalName).toList();
 
-    private EntityType(int protoNumber) {
+    EntityType(int protoNumber) {
         this.protoNumber = protoNumber;
+        this.tableName = name().toLowerCase();
     }
 
-    public static final List<String> NORMAL_NAMES = EnumSet.allOf(EntityType.class).stream()
-            .map(EntityType::getNormalName).collect(Collectors.toUnmodifiableList());
-
-    @Getter
-    private final String normalName = StringUtils.capitalize(StringUtils.removeStart(name(), "TB_")
-            .toLowerCase().replaceAll("_", " "));
+    EntityType(int protoNumber, String tableName) {
+        this.protoNumber = protoNumber;
+        this.tableName = tableName;
+    }
 
 }

common/data/src/main/java/org/thingsboard/server/common/data/housekeeper/AlarmsDeletionHousekeeperTask.java

@@ -0,0 +1,51 @@
+/**
+ * Copyright © 2016-2024 The Thingsboard Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.thingsboard.server.common.data.housekeeper;
+
+import lombok.AccessLevel;
+import lombok.Data;
+import lombok.EqualsAndHashCode;
+import lombok.NoArgsConstructor;
+import lombok.ToString;
+import org.thingsboard.server.common.data.id.EntityId;
+import org.thingsboard.server.common.data.id.TenantId;
+
+import java.util.List;
+import java.util.UUID;
+
+@Data
+@ToString(callSuper = true)
+@EqualsAndHashCode(callSuper = true)
+@NoArgsConstructor(access = AccessLevel.PROTECTED)
+public class AlarmsDeletionHousekeeperTask extends HousekeeperTask {
+
+    private List<UUID> alarms;
+
+    public AlarmsDeletionHousekeeperTask(TenantId tenantId, EntityId entityId) {
+        this(tenantId, entityId, null);
+    }
+
+    public AlarmsDeletionHousekeeperTask(TenantId tenantId, EntityId entityId, List<UUID> alarms) {
+        super(tenantId, entityId, HousekeeperTaskType.DELETE_ALARMS);
+        this.alarms = alarms;
+    }
+
+    @Override
+    public String getDescription() {
+        return super.getDescription() + (alarms != null ? " (" + alarms + ")" : "");
+    }
+
+}

common/data/src/main/java/org/thingsboard/server/common/data/housekeeper/AlarmsUnassignHousekeeperTask.java

@@ -0,0 +1,38 @@
+/**
+ * Copyright © 2016-2024 The Thingsboard Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.thingsboard.server.common.data.housekeeper;
+
+import lombok.AccessLevel;
+import lombok.Data;
+import lombok.EqualsAndHashCode;
+import lombok.NoArgsConstructor;
+import lombok.ToString;
+import org.thingsboard.server.common.data.User;
+
+@Data
+@ToString(callSuper = true)
+@EqualsAndHashCode(callSuper = true)
+@NoArgsConstructor(access = AccessLevel.PROTECTED)
+public class AlarmsUnassignHousekeeperTask extends HousekeeperTask {
+
+    private String userTitle;
+
+    protected AlarmsUnassignHousekeeperTask(User user) {
+        super(user.getTenantId(), user.getId(), HousekeeperTaskType.UNASSIGN_ALARMS);
+        this.userTitle = user.getTitle();
+    }
+
+}

common/data/src/main/java/org/thingsboard/server/common/data/housekeeper/EntitiesDeletionHousekeeperTask.java

@@ -0,0 +1,50 @@
+/**
+ * Copyright © 2016-2024 The Thingsboard Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.thingsboard.server.common.data.housekeeper;
+
+import com.fasterxml.jackson.annotation.JsonIgnore;
+import lombok.Data;
+import lombok.EqualsAndHashCode;
+import lombok.NoArgsConstructor;
+import lombok.ToString;
+import org.thingsboard.server.common.data.EntityType;
+import org.thingsboard.server.common.data.id.TenantId;
+
+import java.util.List;
+import java.util.UUID;
+
+@Data
+@ToString(callSuper = true)
+@EqualsAndHashCode(callSuper = true)
+@NoArgsConstructor
+public class EntitiesDeletionHousekeeperTask extends HousekeeperTask {
+
+    private EntityType entityType;
+    private List<UUID> entities;
+
+    public EntitiesDeletionHousekeeperTask(TenantId tenantId, EntityType entityType, List<UUID> entities) {
+        super(tenantId, tenantId, HousekeeperTaskType.DELETE_ENTITIES);
+        this.entityType = entityType;
+        this.entities = entities;
+    }
+
+    @JsonIgnore
+    @Override
+    public String getDescription() {
+        return entityType.getNormalName().toLowerCase() + "s deletion (" + entities + ")";
+    }
+
+}

common/data/src/main/java/org/thingsboard/server/common/data/housekeeper/HousekeeperTask.java

@@ -0,0 +1,89 @@
+/**
+ * Copyright © 2016-2024 The Thingsboard Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.thingsboard.server.common.data.housekeeper;
+
+import com.fasterxml.jackson.annotation.JsonIgnore;
+import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
+import com.fasterxml.jackson.annotation.JsonSubTypes;
+import com.fasterxml.jackson.annotation.JsonSubTypes.Type;
+import com.fasterxml.jackson.annotation.JsonTypeInfo;
+import lombok.AccessLevel;
+import lombok.Data;
+import lombok.NoArgsConstructor;
+import lombok.NonNull;
+import org.thingsboard.server.common.data.EntityType;
+import org.thingsboard.server.common.data.User;
+import org.thingsboard.server.common.data.id.EntityId;
+import org.thingsboard.server.common.data.id.TenantId;
+
+import java.io.Serializable;
+
+@JsonIgnoreProperties(ignoreUnknown = true)
+@JsonTypeInfo(use = JsonTypeInfo.Id.NAME, property = "taskType", visible = true, include = JsonTypeInfo.As.EXISTING_PROPERTY, defaultImpl = HousekeeperTask.class)
+@JsonSubTypes({
+        @Type(name = "DELETE_TS_HISTORY", value = TsHistoryDeletionHousekeeperTask.class),
+        @Type(name = "DELETE_LATEST_TS", value = LatestTsDeletionHousekeeperTask.class),
+        @Type(name = "DELETE_TENANT_ENTITIES", value = TenantEntitiesDeletionHousekeeperTask.class),
+        @Type(name = "DELETE_ENTITIES", value = EntitiesDeletionHousekeeperTask.class),
+        @Type(name = "DELETE_ALARMS", value = AlarmsDeletionHousekeeperTask.class),
+        @Type(name = "UNASSIGN_ALARMS", value = AlarmsUnassignHousekeeperTask.class)
+})
+@Data
+@NoArgsConstructor(access = AccessLevel.PROTECTED)
+public class HousekeeperTask implements Serializable {
+
+    private TenantId tenantId;
+    private EntityId entityId;
+    private HousekeeperTaskType taskType;
+    private long ts;
+
+    protected HousekeeperTask(@NonNull TenantId tenantId, @NonNull EntityId entityId, @NonNull HousekeeperTaskType taskType) {
+        this.tenantId = tenantId;
+        this.entityId = entityId;
+        this.taskType = taskType;
+        this.ts = System.currentTimeMillis();
+    }
+
+    public static HousekeeperTask deleteAttributes(TenantId tenantId, EntityId entityId) {
+        return new HousekeeperTask(tenantId, entityId, HousekeeperTaskType.DELETE_ATTRIBUTES);
+    }
+
+    public static HousekeeperTask deleteTelemetry(TenantId tenantId, EntityId entityId) {
+        return new HousekeeperTask(tenantId, entityId, HousekeeperTaskType.DELETE_TELEMETRY);
+    }
+
+    public static HousekeeperTask deleteEvents(TenantId tenantId, EntityId entityId) {
+        return new HousekeeperTask(tenantId, entityId, HousekeeperTaskType.DELETE_EVENTS);
+    }
+
+    public static HousekeeperTask unassignAlarms(User user) {
+        return new AlarmsUnassignHousekeeperTask(user);
+    }
+
+    public static HousekeeperTask deleteAlarms(TenantId tenantId, EntityId entityId) {
+        return new AlarmsDeletionHousekeeperTask(tenantId, entityId);
+    }
+
+    public static HousekeeperTask deleteTenantEntities(TenantId tenantId, EntityType entityType) {
+        return new TenantEntitiesDeletionHousekeeperTask(tenantId, entityType);
+    }
+
+    @JsonIgnore
+    public String getDescription() {
+        return taskType.getDescription() + " for " + entityId.getEntityType().getNormalName().toLowerCase() + " " + entityId.getId();
+    }
+
+}

common/data/src/main/java/org/thingsboard/server/common/data/housekeeper/HousekeeperTaskType.java

@@ -0,0 +1,37 @@
+/**
+ * Copyright © 2016-2024 The Thingsboard Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.thingsboard.server.common.data.housekeeper;
+
+import lombok.Getter;
+import lombok.RequiredArgsConstructor;
+
+@RequiredArgsConstructor
+@Getter
+public enum HousekeeperTaskType {
+
+    DELETE_ATTRIBUTES("attributes deletion"),
+    DELETE_TELEMETRY("telemetry deletion"),
+    DELETE_LATEST_TS("latest telemetry deletion"),
+    DELETE_TS_HISTORY("timeseries history deletion"),
+    DELETE_EVENTS("events deletion"),
+    DELETE_ALARMS("alarms deletion"),
+    UNASSIGN_ALARMS("alarms unassigning"),
+    DELETE_TENANT_ENTITIES("tenant entities deletion"),
+    DELETE_ENTITIES("entities deletion");
+
+    private final String description;
+
+}

common/data/src/main/java/org/thingsboard/server/common/data/housekeeper/LatestTsDeletionHousekeeperTask.java

@@ -0,0 +1,44 @@
+/**
+ * Copyright © 2016-2024 The Thingsboard Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.thingsboard.server.common.data.housekeeper;
+
+import lombok.AccessLevel;
+import lombok.Data;
+import lombok.EqualsAndHashCode;
+import lombok.NoArgsConstructor;
+import lombok.ToString;
+import org.thingsboard.server.common.data.id.EntityId;
+import org.thingsboard.server.common.data.id.TenantId;
+
+@Data
+@ToString(callSuper = true)
+@EqualsAndHashCode(callSuper = true)
+@NoArgsConstructor(access = AccessLevel.PROTECTED)
+public class LatestTsDeletionHousekeeperTask extends HousekeeperTask {
+
+    private String key;
+
+    public LatestTsDeletionHousekeeperTask(TenantId tenantId, EntityId entityId, String key) {
+        super(tenantId, entityId, HousekeeperTaskType.DELETE_LATEST_TS);
+        this.key = key;
+    }
+
+    @Override
+    public String getDescription() {
+        return super.getDescription() + (key != null ? " for key '" + key + "'" : "");
+    }
+
+}

common/data/src/main/java/org/thingsboard/server/common/data/housekeeper/TenantEntitiesDeletionHousekeeperTask.java

@@ -0,0 +1,45 @@
+/**
+ * Copyright © 2016-2024 The Thingsboard Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.thingsboard.server.common.data.housekeeper;
+
+import com.fasterxml.jackson.annotation.JsonIgnore;
+import lombok.Data;
+import lombok.EqualsAndHashCode;
+import lombok.NoArgsConstructor;
+import lombok.ToString;
+import org.thingsboard.server.common.data.EntityType;
+import org.thingsboard.server.common.data.id.TenantId;
+
+@Data
+@ToString(callSuper = true)
+@EqualsAndHashCode(callSuper = true)
+@NoArgsConstructor
+public class TenantEntitiesDeletionHousekeeperTask extends HousekeeperTask {
+
+    private EntityType entityType;
+
+    public TenantEntitiesDeletionHousekeeperTask(TenantId tenantId, EntityType entityType) {
+        super(tenantId, tenantId, HousekeeperTaskType.DELETE_TENANT_ENTITIES);
+        this.entityType = entityType;
+    }
+
+    @JsonIgnore
+    @Override
+    public String getDescription() {
+        return entityType.getNormalName().toLowerCase() + "s deletion";
+    }
+
+}

common/data/src/main/java/org/thingsboard/server/common/data/housekeeper/TsHistoryDeletionHousekeeperTask.java

@@ -0,0 +1,44 @@
+/**
+ * Copyright © 2016-2024 The Thingsboard Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.thingsboard.server.common.data.housekeeper;
+
+import lombok.AccessLevel;
+import lombok.Data;
+import lombok.EqualsAndHashCode;
+import lombok.NoArgsConstructor;
+import lombok.ToString;
+import org.thingsboard.server.common.data.id.EntityId;
+import org.thingsboard.server.common.data.id.TenantId;
+
+@Data
+@ToString(callSuper = true)
+@EqualsAndHashCode(callSuper = true)
+@NoArgsConstructor(access = AccessLevel.PROTECTED)
+public class TsHistoryDeletionHousekeeperTask extends HousekeeperTask {
+
+    private String key;
+
+    public TsHistoryDeletionHousekeeperTask(TenantId tenantId, EntityId entityId, String key) {
+        super(tenantId, entityId, HousekeeperTaskType.DELETE_TS_HISTORY);
+        this.key = key;
+    }
+
+    @Override
+    public String getDescription() {
+        return super.getDescription() + (key != null ? " for key '" + key + "'" : "");
+    }
+
+}

common/data/src/main/java/org/thingsboard/server/common/data/notification/NotificationType.java

@@ -30,6 +30,6 @@ public enum NotificationType {
     RULE_NODE,
     RATE_LIMITS,
     EDGE_CONNECTION,
-    EDGE_COMMUNICATION_FAILURE
-
+    EDGE_COMMUNICATION_FAILURE,
+    TASK_PROCESSING_FAILURE
 }

common/data/src/main/java/org/thingsboard/server/common/data/notification/info/TaskProcessingFailureNotificationInfo.java

@@ -0,0 +1,61 @@
+/**
+ * Copyright © 2016-2024 The Thingsboard Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.thingsboard.server.common.data.notification.info;
+
+import lombok.AllArgsConstructor;
+import lombok.Builder;
+import lombok.Data;
+import lombok.NoArgsConstructor;
+import org.thingsboard.server.common.data.housekeeper.HousekeeperTaskType;
+import org.thingsboard.server.common.data.id.EntityId;
+import org.thingsboard.server.common.data.id.TenantId;
+
+import java.util.Map;
+
+import static org.thingsboard.server.common.data.util.CollectionsUtil.mapOf;
+
+@Data
+@NoArgsConstructor
+@AllArgsConstructor
+@Builder
+public class TaskProcessingFailureNotificationInfo implements RuleOriginatedNotificationInfo {
+
+    private TenantId tenantId;
+    private EntityId entityId;
+    private HousekeeperTaskType taskType;
+    private String taskDescription;
+    private String error;
+    private int attempt;
+
+    @Override
+    public Map<String, String> getTemplateData() {
+        return mapOf(
+                "tenantId", tenantId.toString(),
+                "entityType", entityId.getEntityType().getNormalName(),
+                "entityId", entityId.getId().toString(),
+                "taskType", taskType.getDescription(),
+                "taskDescription", taskDescription,
+                "error", error,
+                "attempt", String.valueOf(attempt)
+        );
+    }
+
+    @Override
+    public TenantId getAffectedTenantId() {
+        return tenantId;
+    }
+
+}

common/data/src/main/java/org/thingsboard/server/common/data/notification/rule/trigger/TaskProcessingFailureTrigger.java

@@ -0,0 +1,53 @@
+/**
+ * Copyright © 2016-2024 The Thingsboard Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.thingsboard.server.common.data.notification.rule.trigger;
+
+import lombok.Builder;
+import lombok.Data;
+import org.thingsboard.server.common.data.housekeeper.HousekeeperTask;
+import org.thingsboard.server.common.data.id.EntityId;
+import org.thingsboard.server.common.data.id.TenantId;
+import org.thingsboard.server.common.data.notification.rule.trigger.config.NotificationRuleTriggerType;
+
+@Data
+@Builder
+public class TaskProcessingFailureTrigger implements NotificationRuleTrigger {
+
+    private final HousekeeperTask task;
+    private final int attempt;
+    private final Throwable error;
+
+    @Override
+    public NotificationRuleTriggerType getType() {
+        return NotificationRuleTriggerType.TASK_PROCESSING_FAILURE;
+    }
+
+    @Override
+    public TenantId getTenantId() {
+        return task.getTenantId();
+    }
+
+    @Override
+    public EntityId getOriginatorEntityId() {
+        return task.getEntityId();
+    }
+
+    @Override
+    public boolean deduplicate() {
+        return false;
+    }
+
+}

common/data/src/main/java/org/thingsboard/server/common/data/notification/rule/trigger/config/NotificationRuleTriggerConfig.java

@@ -38,6 +38,7 @@ import java.io.Serializable;
         @Type(value = RateLimitsNotificationRuleTriggerConfig.class, name = "RATE_LIMITS"),
         @Type(value = EdgeConnectionNotificationRuleTriggerConfig.class, name = "EDGE_CONNECTION"),
         @Type(value = EdgeCommunicationFailureNotificationRuleTriggerConfig.class, name = "EDGE_COMMUNICATION_FAILURE"),
+        @Type(value = TaskProcessingFailureNotificationRuleTriggerConfig.class, name = "TASK_PROCESSING_FAILURE")
 })
 public interface NotificationRuleTriggerConfig extends Serializable {
 

common/data/src/main/java/org/thingsboard/server/common/data/notification/rule/trigger/config/NotificationRuleTriggerType.java

@@ -31,7 +31,8 @@ public enum NotificationRuleTriggerType {
     NEW_PLATFORM_VERSION(false),
     ENTITIES_LIMIT(false),
     API_USAGE_LIMIT(false),
-    RATE_LIMITS(false);
+    RATE_LIMITS(false),
+    TASK_PROCESSING_FAILURE(false);
 
     private final boolean tenantLevel;
 

common/dao-api/src/main/java/org/thingsboard/server/dao/housekeeper/HouseKeeperService.java → common/data/src/main/java/org/thingsboard/server/common/data/notification/rule/trigger/config/TaskProcessingFailureNotificationRuleTriggerConfig.java

@@ -13,17 +13,18 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.thingsboard.server.dao.housekeeper;
+package org.thingsboard.server.common.data.notification.rule.trigger.config;
 
-import com.google.common.util.concurrent.ListenableFuture;
-import org.thingsboard.server.common.data.User;
-import org.thingsboard.server.common.data.id.AlarmId;
-import org.thingsboard.server.common.data.id.TenantId;
+import lombok.Builder;
+import lombok.Data;
 
-import java.util.List;
+@Data
+@Builder
+public class TaskProcessingFailureNotificationRuleTriggerConfig implements NotificationRuleTriggerConfig {
 
-public interface HouseKeeperService {
-
-    ListenableFuture<List<AlarmId>> unassignDeletedUserAlarms(TenantId tenantId, User user, long unassignTs);
+    @Override
+    public NotificationRuleTriggerType getTriggerType() {
+        return NotificationRuleTriggerType.TASK_PROCESSING_FAILURE;
+    }
 
 }

common/data/src/main/java/org/thingsboard/server/common/data/page/PageData.java

@@ -19,6 +19,7 @@ import com.fasterxml.jackson.annotation.JsonCreator;
 import com.fasterxml.jackson.annotation.JsonProperty;
 import io.swagger.v3.oas.annotations.media.Schema;
 import lombok.EqualsAndHashCode;
+import lombok.ToString;
 
 import java.io.Serializable;
 import java.util.Collections;
@@ -28,6 +29,7 @@ import java.util.stream.Collectors;
 
 @Schema
 @EqualsAndHashCode
+@ToString
 public class PageData<T> implements Serializable {
 
     public static final PageData EMPTY_PAGE_DATA = new PageData<>();

common/data/src/main/java/org/thingsboard/server/common/data/page/PageLink.java

@@ -63,10 +63,10 @@ public class PageLink {
 
     @JsonIgnore
     public PageLink nextPageLink() {
-        return new PageLink(this.pageSize, this.page+1, this.textSearch, this.sortOrder);
+        return new PageLink(this.pageSize, this.page + 1, this.textSearch, this.sortOrder);
     }
 
-    public Sort toSort(SortOrder sortOrder, Map<String,String> columnMap, boolean addDefaultSorting) {
+    public Sort toSort(SortOrder sortOrder, Map<String, String> columnMap, boolean addDefaultSorting) {
         if (sortOrder == null) {
             return DEFAULT_SORT;
         } else {
@@ -74,7 +74,7 @@ public class PageLink {
         }
     }
 
-    public Sort toSort(List<SortOrder> sortOrders, Map<String,String> columnMap, boolean addDefaultSorting) {
+    public Sort toSort(List<SortOrder> sortOrders, Map<String, String> columnMap, boolean addDefaultSorting) {
         if (addDefaultSorting && !isDefaultSortOrderAvailable(sortOrders)) {
             sortOrders = new ArrayList<>(sortOrders);
             sortOrders.add(new SortOrder(DEFAULT_SORT_PROPERTY, SortOrder.Direction.ASC));
@@ -82,7 +82,7 @@ public class PageLink {
         return Sort.by(sortOrders.stream().map(s -> toSortOrder(s, columnMap)).collect(Collectors.toList()));
     }
 
-    private Sort.Order toSortOrder(SortOrder sortOrder, Map<String,String> columnMap) {
+    private Sort.Order toSortOrder(SortOrder sortOrder, Map<String, String> columnMap) {
         String property = sortOrder.getProperty();
         if (columnMap.containsKey(property)) {
             property = columnMap.get(property);

common/data/src/main/java/org/thingsboard/server/common/data/page/SortOrder.java

@@ -32,6 +32,10 @@ public class SortOrder {
         this.direction = direction;
     }
 
+    public static SortOrder of(String property, Direction direction) {
+        return new SortOrder(property, direction);
+    }
+
     public static enum Direction {
         ASC, DESC
     }

common/message/src/main/java/org/thingsboard/server/common/msg/housekeeper/HousekeeperClient.java

@@ -0,0 +1,24 @@
+/**
+ * Copyright © 2016-2024 The Thingsboard Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.thingsboard.server.common.msg.housekeeper;
+
+import org.thingsboard.server.common.data.housekeeper.HousekeeperTask;
+
+public interface HousekeeperClient {
+
+    void submitTask(HousekeeperTask task);
+
+}

common/proto/src/main/proto/queue.proto

@@ -1562,3 +1562,16 @@ message LifecycleEventProto {
   bool success = 7;
   string error = 8;
 }
+
+message ToHousekeeperServiceMsg {
+  HousekeeperTaskProto task = 1;
+}
+
+message HousekeeperTaskProto {
+  string value = 1;
+  int64 ts = 2;
+
+  // reprocessing
+  int32 attempt = 50;
+  repeated string errors = 51;
+}

common/queue/src/main/java/org/thingsboard/server/queue/housekeeper/DefaultHousekeeperClient.java

@@ -0,0 +1,86 @@
+/**
+ * Copyright © 2016-2024 The Thingsboard Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.thingsboard.server.queue.housekeeper;
+
+import lombok.extern.slf4j.Slf4j;
+import org.springframework.stereotype.Service;
+import org.thingsboard.common.util.JacksonUtil;
+import org.thingsboard.server.common.data.housekeeper.HousekeeperTask;
+import org.thingsboard.server.common.data.housekeeper.HousekeeperTaskType;
+import org.thingsboard.server.common.msg.housekeeper.HousekeeperClient;
+import org.thingsboard.server.common.msg.queue.TopicPartitionInfo;
+import org.thingsboard.server.gen.transport.TransportProtos;
+import org.thingsboard.server.gen.transport.TransportProtos.ToHousekeeperServiceMsg;
+import org.thingsboard.server.queue.TbQueueCallback;
+import org.thingsboard.server.queue.TbQueueMsgMetadata;
+import org.thingsboard.server.queue.TbQueueProducer;
+import org.thingsboard.server.queue.common.TbProtoQueueMsg;
+import org.thingsboard.server.queue.provider.TbQueueProducerProvider;
+
+@Service
+@Slf4j
+public class DefaultHousekeeperClient implements HousekeeperClient {
+
+    private final HousekeeperConfig config;
+    private final TbQueueProducer<TbProtoQueueMsg<ToHousekeeperServiceMsg>> producer;
+    private final TopicPartitionInfo submitTpi;
+    private final TbQueueCallback submitCallback;
+
+    public DefaultHousekeeperClient(HousekeeperConfig config,
+                                    TbQueueProducerProvider producerProvider) {
+        this.config = config;
+        this.producer = producerProvider.getHousekeeperMsgProducer();
+        this.submitTpi = TopicPartitionInfo.builder().topic(producer.getDefaultTopic()).build();
+        this.submitCallback = new TbQueueCallback() {
+            @Override
+            public void onSuccess(TbQueueMsgMetadata metadata) {
+                log.trace("Submitted Housekeeper task");
+            }
+
+            @Override
+            public void onFailure(Throwable t) {
+                log.error("Failed to submit Housekeeper task", t);
+            }
+        };
+    }
+
+    @Override
+    public void submitTask(HousekeeperTask task) {
+        HousekeeperTaskType taskType = task.getTaskType();
+        if (config.getDisabledTaskTypes().contains(taskType)) {
+            log.trace("Task type {} is disabled, ignoring {}", taskType, task);
+            return;
+        }
+
+        log.debug("[{}][{}][{}] Submitting task: {}", task.getTenantId(), task.getEntityId().getEntityType(), task.getEntityId(), task);
+        /*
+         * using msg key as entity id so that msgs related to certain entity are pushed to same partition,
+         * e.g. on tenant deletion (entity id is tenant id), we need to clean up tenant entities in certain order
+         * */
+        try {
+            producer.send(submitTpi, new TbProtoQueueMsg<>(task.getEntityId().getId(), ToHousekeeperServiceMsg.newBuilder()
+                    .setTask(TransportProtos.HousekeeperTaskProto.newBuilder()
+                            .setValue(JacksonUtil.toString(task))
+                            .setTs(task.getTs())
+                            .setAttempt(0)
+                            .build())
+                    .build()), submitCallback);
+        } catch (Throwable t) {
+            log.error("Failed to submit Housekeeper task {}", task, t);
+        }
+    }
+
+}

common/queue/src/main/java/org/thingsboard/server/queue/housekeeper/HousekeeperConfig.java

@@ -0,0 +1,40 @@
+/**
+ * Copyright © 2016-2024 The Thingsboard Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.thingsboard.server.queue.housekeeper;
+
+import lombok.Getter;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.stereotype.Component;
+import org.thingsboard.server.common.data.housekeeper.HousekeeperTaskType;
+
+import java.util.Set;
+
+@Component
+@Getter
+public class HousekeeperConfig {
+
+    @Value("${queue.core.housekeeper.disabled-task-types:}")
+    private Set<HousekeeperTaskType> disabledTaskTypes;
+    @Value("${queue.core.housekeeper.task-processing-timeout-ms:120000}")
+    private int taskProcessingTimeout;
+    @Value("${queue.core.housekeeper.poll-interval-ms:500}")
+    private int pollInterval;
+    @Value("${queue.core.housekeeper.task-reprocessing-delay-ms:3000}")
+    private int taskReprocessingDelay;
+    @Value("${queue.core.housekeeper.max-reprocessing-attempts:10}")
+    private int maxReprocessingAttempts;
+
+}

common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaTopicConfigs.java

@@ -43,6 +43,10 @@ public class TbKafkaTopicConfigs {
     private String fwUpdatesProperties;
     @Value("${queue.kafka.topic-properties.version-control:}")
     private String vcProperties;
+    @Value("${queue.kafka.topic-properties.housekeeper:}")
+    private String housekeeperProperties;
+    @Value("${queue.kafka.topic-properties.housekeeper-reprocessing:}")
+    private String housekeeperReprocessingProperties;
 
     @Getter
     private Map<String, String> coreConfigs;
@@ -62,6 +66,10 @@ public class TbKafkaTopicConfigs {
     private Map<String, String> fwUpdatesConfigs;
     @Getter
     private Map<String, String> vcConfigs;
+    @Getter
+    private Map<String, String> housekeeperConfigs;
+    @Getter
+    private Map<String, String> housekeeperReprocessingConfigs;
 
     @PostConstruct
     private void init() {
@@ -76,6 +84,8 @@ public class TbKafkaTopicConfigs {
         jsExecutorResponseConfigs.put(NUM_PARTITIONS_SETTING, "1");
         fwUpdatesConfigs = PropertyUtils.getProps(fwUpdatesProperties);
         vcConfigs = PropertyUtils.getProps(vcProperties);
+        housekeeperConfigs = PropertyUtils.getProps(housekeeperProperties);
+        housekeeperReprocessingConfigs = PropertyUtils.getProps(housekeeperReprocessingProperties);
     }
 
 }

common/queue/src/main/java/org/thingsboard/server/queue/memory/DefaultInMemoryStorage.java

@@ -20,8 +20,10 @@ import org.springframework.stereotype.Component;
 import org.thingsboard.server.queue.TbQueueMsg;
 
 import java.util.ArrayList;
+import java.util.Collection;
 import java.util.Collections;
 import java.util.List;
+import java.util.Optional;
 import java.util.concurrent.BlockingQueue;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.LinkedBlockingQueue;
@@ -47,6 +49,11 @@ public final class DefaultInMemoryStorage implements InMemoryStorage {
         return storage.values().stream().map(BlockingQueue::size).reduce(0, Integer::sum);
     }
 
+    @Override
+    public int getLag(String topic) {
+        return Optional.ofNullable(storage.get(topic)).map(Collection::size).orElse(0);
+    }
+
     @Override
     public boolean put(String topic, TbQueueMsg msg) {
         return storage.computeIfAbsent(topic, (t) -> new LinkedBlockingQueue<>()).add(msg);

common/queue/src/main/java/org/thingsboard/server/queue/memory/InMemoryStorage.java

@@ -25,6 +25,8 @@ public interface InMemoryStorage {
 
     int getLagTotal();
 
+    int getLag(String topic);
+
     boolean put(String topic, TbQueueMsg msg);
 
     <T extends TbQueueMsg> List<T> get(String topic) throws InterruptedException;

common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsMonolithQueueFactory.java

@@ -26,6 +26,7 @@ import org.thingsboard.server.gen.js.JsInvokeProtos.RemoteJsResponse;
 import org.thingsboard.server.gen.transport.TransportProtos;
 import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg;
 import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg;
+import org.thingsboard.server.gen.transport.TransportProtos.ToHousekeeperServiceMsg;
 import org.thingsboard.server.gen.transport.TransportProtos.ToOtaPackageStateServiceMsg;
 import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg;
 import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineNotificationMsg;
@@ -226,6 +227,28 @@ public class AwsSqsMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEng
         return new TbAwsSqsProducerTemplate<>(vcAdmin, sqsSettings, topicService.buildTopicName(vcSettings.getTopic()));
     }
 
+    @Override
+    public TbQueueProducer<TbProtoQueueMsg<ToHousekeeperServiceMsg>> createHousekeeperMsgProducer() {
+        return new TbAwsSqsProducerTemplate<>(coreAdmin, sqsSettings, topicService.buildTopicName(coreSettings.getHousekeeperTopic()));
+    }
+
+    @Override
+    public TbQueueConsumer<TbProtoQueueMsg<ToHousekeeperServiceMsg>> createHousekeeperMsgConsumer() {
+        return new TbAwsSqsConsumerTemplate<>(coreAdmin, sqsSettings, topicService.buildTopicName(coreSettings.getHousekeeperTopic()),
+                msg -> new TbProtoQueueMsg<>(msg.getKey(), ToHousekeeperServiceMsg.parseFrom(msg.getData()), msg.getHeaders()));
+    }
+
+    @Override
+    public TbQueueProducer<TbProtoQueueMsg<ToHousekeeperServiceMsg>> createHousekeeperReprocessingMsgProducer() {
+        return new TbAwsSqsProducerTemplate<>(coreAdmin, sqsSettings, topicService.buildTopicName(coreSettings.getHousekeeperReprocessingTopic()));
+    }
+
+    @Override
+    public TbQueueConsumer<TbProtoQueueMsg<ToHousekeeperServiceMsg>> createHousekeeperReprocessingMsgConsumer() {
+        return new TbAwsSqsConsumerTemplate<>(coreAdmin, sqsSettings, topicService.buildTopicName(coreSettings.getHousekeeperReprocessingTopic()),
+                msg -> new TbProtoQueueMsg<>(msg.getKey(), ToHousekeeperServiceMsg.parseFrom(msg.getData()), msg.getHeaders()));
+    }
+
     @PreDestroy
     private void destroy() {
         if (coreAdmin != null) {

common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTbCoreQueueFactory.java

@@ -24,6 +24,7 @@ import org.thingsboard.server.gen.js.JsInvokeProtos;
 import org.thingsboard.server.gen.transport.TransportProtos;
 import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg;
 import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg;
+import org.thingsboard.server.gen.transport.TransportProtos.ToHousekeeperServiceMsg;
 import org.thingsboard.server.gen.transport.TransportProtos.ToOtaPackageStateServiceMsg;
 import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg;
 import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineNotificationMsg;
@@ -205,6 +206,28 @@ public class AwsSqsTbCoreQueueFactory implements TbCoreQueueFactory {
         return new TbAwsSqsProducerTemplate<>(vcAdmin, sqsSettings, topicService.buildTopicName(vcSettings.getTopic()));
     }
 
+    @Override
+    public TbQueueProducer<TbProtoQueueMsg<ToHousekeeperServiceMsg>> createHousekeeperMsgProducer() {
+        return new TbAwsSqsProducerTemplate<>(coreAdmin, sqsSettings, topicService.buildTopicName(coreSettings.getHousekeeperTopic()));
+    }
+
+    @Override
+    public TbQueueConsumer<TbProtoQueueMsg<ToHousekeeperServiceMsg>> createHousekeeperMsgConsumer() {
+        return new TbAwsSqsConsumerTemplate<>(coreAdmin, sqsSettings, topicService.buildTopicName(coreSettings.getHousekeeperTopic()),
+                msg -> new TbProtoQueueMsg<>(msg.getKey(), ToHousekeeperServiceMsg.parseFrom(msg.getData()), msg.getHeaders()));
+    }
+
+    @Override
+    public TbQueueProducer<TbProtoQueueMsg<ToHousekeeperServiceMsg>> createHousekeeperReprocessingMsgProducer() {
+        return new TbAwsSqsProducerTemplate<>(coreAdmin, sqsSettings, topicService.buildTopicName(coreSettings.getHousekeeperReprocessingTopic()));
+    }
+
+    @Override
+    public TbQueueConsumer<TbProtoQueueMsg<ToHousekeeperServiceMsg>> createHousekeeperReprocessingMsgConsumer() {
+        return new TbAwsSqsConsumerTemplate<>(coreAdmin, sqsSettings, topicService.buildTopicName(coreSettings.getHousekeeperReprocessingTopic()),
+                msg -> new TbProtoQueueMsg<>(msg.getKey(), ToHousekeeperServiceMsg.parseFrom(msg.getData()), msg.getHeaders()));
+    }
+
     @PreDestroy
     private void destroy() {
         if (coreAdmin != null) {

common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTbRuleEngineQueueFactory.java

@@ -33,8 +33,8 @@ import org.thingsboard.server.queue.TbQueueRequestTemplate;
 import org.thingsboard.server.queue.common.DefaultTbQueueRequestTemplate;
 import org.thingsboard.server.queue.common.TbProtoJsQueueMsg;
 import org.thingsboard.server.queue.common.TbProtoQueueMsg;
-import org.thingsboard.server.queue.discovery.TopicService;
 import org.thingsboard.server.queue.discovery.TbServiceInfoProvider;
+import org.thingsboard.server.queue.discovery.TopicService;
 import org.thingsboard.server.queue.settings.TbQueueCoreSettings;
 import org.thingsboard.server.queue.settings.TbQueueRemoteJsInvokeSettings;
 import org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings;
@@ -159,6 +159,10 @@ public class AwsSqsTbRuleEngineQueueFactory implements TbRuleEngineQueueFactory
         return new TbAwsSqsProducerTemplate<>(otaAdmin, sqsSettings, topicService.buildTopicName(coreSettings.getOtaPackageTopic()));
     }
 
+    @Override
+    public TbQueueProducer<TbProtoQueueMsg<TransportProtos.ToHousekeeperServiceMsg>> createHousekeeperMsgProducer() {
+        return new TbAwsSqsProducerTemplate<>(coreAdmin, sqsSettings, topicService.buildTopicName(coreSettings.getHousekeeperTopic()));
+    }
 
     @PreDestroy
     private void destroy() {