diff --git a/application/src/main/java/org/thingsboard/server/install/ThingsboardInstallService.java b/application/src/main/java/org/thingsboard/server/install/ThingsboardInstallService.java index eb9d908bab..2eecd61798 100644 --- a/application/src/main/java/org/thingsboard/server/install/ThingsboardInstallService.java +++ b/application/src/main/java/org/thingsboard/server/install/ThingsboardInstallService.java @@ -110,7 +110,7 @@ public class ThingsboardInstallService { log.info("Upgrading ThingsBoard from version 3.5.1 to 3.6.0 ..."); databaseEntitiesUpgradeService.upgradeDatabase("3.5.1"); dataUpdateService.updateData("3.5.1"); - systemDataLoaderService.updateDefaultNotificationConfigs(); + systemDataLoaderService.updateDefaultNotificationConfigs(true); case "3.6.0": log.info("Upgrading ThingsBoard from version 3.6.0 to 3.6.1 ..."); databaseEntitiesUpgradeService.upgradeDatabase("3.6.0"); @@ -126,13 +126,14 @@ public class ThingsboardInstallService { case "3.6.2": log.info("Upgrading ThingsBoard from version 3.6.2 to 3.6.3 ..."); databaseEntitiesUpgradeService.upgradeDatabase("3.6.2"); - systemDataLoaderService.updateDefaultNotificationConfigs(); + systemDataLoaderService.updateDefaultNotificationConfigs(true); case "3.6.3": log.info("Upgrading ThingsBoard from version 3.6.3 to 3.6.4 ..."); databaseEntitiesUpgradeService.upgradeDatabase("3.6.3"); case "3.6.4": log.info("Upgrading ThingsBoard from version 3.6.4 to 3.7.0 ..."); databaseEntitiesUpgradeService.upgradeDatabase("3.6.4"); + systemDataLoaderService.updateDefaultNotificationConfigs(false); //TODO DON'T FORGET to update switch statement in the CacheCleanupService if you need to clear the cache break; default: diff --git a/application/src/main/java/org/thingsboard/server/service/edge/EdgeEventSourcingListener.java b/application/src/main/java/org/thingsboard/server/service/edge/EdgeEventSourcingListener.java index d975217e1f..651339e3a4 100644 --- a/application/src/main/java/org/thingsboard/server/service/edge/EdgeEventSourcingListener.java +++ b/application/src/main/java/org/thingsboard/server/service/edge/EdgeEventSourcingListener.java @@ -33,6 +33,7 @@ import org.thingsboard.server.common.data.alarm.AlarmComment; import org.thingsboard.server.common.data.audit.ActionType; import org.thingsboard.server.common.data.edge.EdgeEventActionType; import org.thingsboard.server.common.data.edge.EdgeEventType; +import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.oauth2.OAuth2Info; import org.thingsboard.server.common.data.relation.EntityRelation; import org.thingsboard.server.common.data.relation.RelationTypeGroup; @@ -44,6 +45,7 @@ import org.thingsboard.server.dao.eventsourcing.ActionEntityEvent; import org.thingsboard.server.dao.eventsourcing.DeleteEntityEvent; import org.thingsboard.server.dao.eventsourcing.RelationActionEvent; import org.thingsboard.server.dao.eventsourcing.SaveEntityEvent; +import org.thingsboard.server.dao.tenant.TenantService; import org.thingsboard.server.dao.user.UserServiceImpl; import javax.annotation.PostConstruct; @@ -69,6 +71,7 @@ public class EdgeEventSourcingListener { private final TbClusterService tbClusterService; private final EdgeSynchronizationManager edgeSynchronizationManager; + private final TenantService tenantService; @PostConstruct public void init() { @@ -95,19 +98,24 @@ public class EdgeEventSourcingListener { @TransactionalEventListener(fallbackExecution = true) public void handleEvent(DeleteEntityEvent event) { + TenantId tenantId = event.getTenantId(); EntityType entityType = event.getEntityId().getEntityType(); + if (!tenantId.isSysTenantId() && !tenantService.tenantExists(tenantId)) { + log.debug("[{}] Ignoring DeleteEntityEvent because tenant does not exist: {}", tenantId, event); + return; + } try { if (EntityType.EDGE.equals(entityType) || EntityType.TENANT.equals(entityType)) { return; } - log.trace("[{}] DeleteEntityEvent called: {}", event.getTenantId(), event); + log.trace("[{}] DeleteEntityEvent called: {}", tenantId, event); EdgeEventType type = getEdgeEventTypeForEntityEvent(event.getEntity()); EdgeEventActionType actionType = getEdgeEventActionTypeForEntityEvent(event.getEntity()); - tbClusterService.sendNotificationMsgToEdge(event.getTenantId(), null, event.getEntityId(), + tbClusterService.sendNotificationMsgToEdge(tenantId, null, event.getEntityId(), JacksonUtil.toString(event.getEntity()), type, actionType, edgeSynchronizationManager.getEdgeId().get()); } catch (Exception e) { - log.error("[{}] failed to process DeleteEntityEvent: {}", event.getTenantId(), event, e); + log.error("[{}] failed to process DeleteEntityEvent: {}", tenantId, event, e); } } diff --git a/application/src/main/java/org/thingsboard/server/service/entitiy/AbstractTbEntityService.java b/application/src/main/java/org/thingsboard/server/service/entitiy/AbstractTbEntityService.java index 4d4d0170bf..0c91d72ea5 100644 --- a/application/src/main/java/org/thingsboard/server/service/entitiy/AbstractTbEntityService.java +++ b/application/src/main/java/org/thingsboard/server/service/entitiy/AbstractTbEntityService.java @@ -27,12 +27,8 @@ import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.User; import org.thingsboard.server.common.data.exception.ThingsboardErrorCode; import org.thingsboard.server.common.data.exception.ThingsboardException; -import org.thingsboard.server.common.data.id.AlarmId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.EntityIdFactory; -import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.common.data.page.PageData; -import org.thingsboard.server.common.data.page.TimePageLink; import org.thingsboard.server.dao.alarm.AlarmService; import org.thingsboard.server.dao.customer.CustomerService; import org.thingsboard.server.dao.edge.EdgeService; @@ -71,13 +67,6 @@ public abstract class AbstractTbEntityService { @Lazy private EntitiesVersionControlService vcService; - protected void removeAlarmsByOriginatorId(TenantId tenantId, EntityId entityId) { - PageData alarms = - alarmService.findAlarmIdsByOriginatorId(tenantId, entityId, new TimePageLink(Integer.MAX_VALUE)); - - alarms.getData().forEach(alarmId -> alarmService.delAlarm(tenantId, alarmId)); - } - protected T checkNotNull(T reference) throws ThingsboardException { return checkNotNull(reference, "Requested item wasn't found!"); } diff --git a/application/src/main/java/org/thingsboard/server/service/entitiy/EntityStateSourcingListener.java b/application/src/main/java/org/thingsboard/server/service/entitiy/EntityStateSourcingListener.java index bc7dc62e78..c013650fa7 100644 --- a/application/src/main/java/org/thingsboard/server/service/entitiy/EntityStateSourcingListener.java +++ b/application/src/main/java/org/thingsboard/server/service/entitiy/EntityStateSourcingListener.java @@ -51,6 +51,7 @@ import org.thingsboard.server.common.msg.rule.engine.DeviceCredentialsUpdateNoti import org.thingsboard.server.dao.eventsourcing.ActionEntityEvent; import org.thingsboard.server.dao.eventsourcing.DeleteEntityEvent; import org.thingsboard.server.dao.eventsourcing.SaveEntityEvent; +import org.thingsboard.server.dao.tenant.TenantService; import javax.annotation.PostConstruct; import java.util.Set; @@ -61,6 +62,7 @@ import java.util.Set; public class EntityStateSourcingListener { private final TbClusterService tbClusterService; + private final TenantService tenantService; @PostConstruct public void init() { @@ -121,6 +123,10 @@ public class EntityStateSourcingListener { TenantId tenantId = event.getTenantId(); EntityId entityId = event.getEntityId(); EntityType entityType = entityId.getEntityType(); + if (!tenantId.isSysTenantId() && entityType != EntityType.TENANT && !tenantService.tenantExists(tenantId)) { + log.debug("[{}] Ignoring DeleteEntityEvent because tenant does not exist: {}", tenantId, event); + return; + } log.debug("[{}][{}][{}] Handling entity deletion event: {}", tenantId, entityType, entityId, event); switch (entityType) { diff --git a/application/src/main/java/org/thingsboard/server/service/entitiy/alarm/DefaultTbAlarmService.java b/application/src/main/java/org/thingsboard/server/service/entitiy/alarm/DefaultTbAlarmService.java index 2fa108d182..89e0bc944f 100644 --- a/application/src/main/java/org/thingsboard/server/service/entitiy/alarm/DefaultTbAlarmService.java +++ b/application/src/main/java/org/thingsboard/server/service/entitiy/alarm/DefaultTbAlarmService.java @@ -176,16 +176,16 @@ public class DefaultTbAlarmService extends AbstractTbEntityService implements Tb } @Override - public List unassignDeletedUserAlarms(TenantId tenantId, User user, long unassignTs) { + public List unassignDeletedUserAlarms(TenantId tenantId, UserId userId, String userTitle, long unassignTs) { List totalAlarmIds = new ArrayList<>(); PageLink pageLink = new PageLink(100, 0, null, new SortOrder("id", SortOrder.Direction.ASC)); while (true) { - PageData pageData = alarmService.findAlarmIdsByAssigneeId(user.getTenantId(), user.getId(), pageLink); + PageData pageData = alarmService.findAlarmIdsByAssigneeId(tenantId, userId, pageLink); List alarmIds = pageData.getData(); if (alarmIds.isEmpty()) { break; } - processAlarmsUnassignment(tenantId, user, alarmIds, unassignTs); + processAlarmsUnassignment(tenantId, userId, userTitle, alarmIds, unassignTs); totalAlarmIds.addAll(alarmIds); pageLink = pageLink.nextPageLink(); } @@ -204,16 +204,16 @@ public class DefaultTbAlarmService extends AbstractTbEntityService implements Tb return ts > 0 ? ts : System.currentTimeMillis(); } - private void processAlarmsUnassignment(TenantId tenantId, User user, List alarmIds, long unassignTs) { + private void processAlarmsUnassignment(TenantId tenantId, UserId userId, String userTitle, List alarmIds, long unassignTs) { for (AlarmId alarmId : alarmIds) { - log.trace("[{}] Unassigning alarm {} userId {}", tenantId, alarmId, user.getId()); - AlarmApiCallResult result = alarmSubscriptionService.unassignAlarm(user.getTenantId(), alarmId, unassignTs); + log.trace("[{}] Unassigning alarm {} userId {}", tenantId, alarmId, userId); + AlarmApiCallResult result = alarmSubscriptionService.unassignAlarm(tenantId, alarmId, unassignTs); if (!result.isSuccessful()) { - log.error("[{}] Cannot unassign alarm {} userId {}", tenantId, alarmId, user.getId()); + log.error("[{}] Cannot unassign alarm {} userId {}", tenantId, alarmId, userId); continue; } if (result.isModified()) { - String comment = String.format("Alarm was unassigned because user %s - was deleted", user.getTitle()); + String comment = String.format("Alarm was unassigned because user %s - was deleted", userTitle); addSystemAlarmComment(result.getAlarm(), null, "ASSIGN", comment); logEntityActionService.logEntityAction(result.getAlarm().getTenantId(), result.getAlarm().getOriginator(), result.getAlarm(), result.getAlarm().getCustomerId(), ActionType.ALARM_UNASSIGNED, null); } diff --git a/application/src/main/java/org/thingsboard/server/service/entitiy/alarm/TbAlarmService.java b/application/src/main/java/org/thingsboard/server/service/entitiy/alarm/TbAlarmService.java index 34dc3d11f2..c975291b58 100644 --- a/application/src/main/java/org/thingsboard/server/service/entitiy/alarm/TbAlarmService.java +++ b/application/src/main/java/org/thingsboard/server/service/entitiy/alarm/TbAlarmService.java @@ -41,7 +41,7 @@ public interface TbAlarmService { AlarmInfo unassign(Alarm alarm, long unassignTs, User user) throws ThingsboardException; - List unassignDeletedUserAlarms(TenantId tenantId, User user, long unassignTs); + List unassignDeletedUserAlarms(TenantId tenantId, UserId userId, String userTitle, long unassignTs); Boolean delete(Alarm alarm, User user); } diff --git a/application/src/main/java/org/thingsboard/server/service/entitiy/asset/DefaultTbAssetService.java b/application/src/main/java/org/thingsboard/server/service/entitiy/asset/DefaultTbAssetService.java index ddc9164575..671d58a789 100644 --- a/application/src/main/java/org/thingsboard/server/service/entitiy/asset/DefaultTbAssetService.java +++ b/application/src/main/java/org/thingsboard/server/service/entitiy/asset/DefaultTbAssetService.java @@ -61,7 +61,6 @@ public class DefaultTbAssetService extends AbstractTbEntityService implements Tb TenantId tenantId = asset.getTenantId(); AssetId assetId = asset.getId(); try { - removeAlarmsByOriginatorId(tenantId, assetId); assetService.deleteAsset(tenantId, assetId); logEntityActionService.logEntityAction(tenantId, assetId, asset, asset.getCustomerId(), actionType, user, assetId.toString()); } catch (Exception e) { diff --git a/application/src/main/java/org/thingsboard/server/service/entitiy/device/DefaultTbDeviceService.java b/application/src/main/java/org/thingsboard/server/service/entitiy/device/DefaultTbDeviceService.java index de9e757166..b05ece9c61 100644 --- a/application/src/main/java/org/thingsboard/server/service/entitiy/device/DefaultTbDeviceService.java +++ b/application/src/main/java/org/thingsboard/server/service/entitiy/device/DefaultTbDeviceService.java @@ -94,7 +94,6 @@ public class DefaultTbDeviceService extends AbstractTbEntityService implements T TenantId tenantId = device.getTenantId(); DeviceId deviceId = device.getId(); try { - removeAlarmsByOriginatorId(tenantId, deviceId); deviceService.deleteDevice(tenantId, deviceId); logEntityActionService.logEntityAction(tenantId, deviceId, device, device.getCustomerId(), actionType, user, deviceId.toString()); diff --git a/application/src/main/java/org/thingsboard/server/service/housekeeper/HousekeeperReprocessingService.java b/application/src/main/java/org/thingsboard/server/service/housekeeper/HousekeeperReprocessingService.java new file mode 100644 index 0000000000..ae16f925a0 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/housekeeper/HousekeeperReprocessingService.java @@ -0,0 +1,118 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.housekeeper; + +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.exception.ExceptionUtils; +import org.springframework.context.annotation.Lazy; +import org.springframework.stereotype.Service; +import org.thingsboard.common.util.ThingsBoardThreadFactory; +import org.thingsboard.server.common.data.StringUtils; +import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; +import org.thingsboard.server.gen.transport.TransportProtos.HousekeeperTaskProto; +import org.thingsboard.server.gen.transport.TransportProtos.ToHousekeeperServiceMsg; +import org.thingsboard.server.queue.TbQueueConsumer; +import org.thingsboard.server.queue.TbQueueProducer; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; +import org.thingsboard.server.queue.housekeeper.HousekeeperConfig; +import org.thingsboard.server.queue.provider.TbCoreQueueFactory; +import org.thingsboard.server.queue.util.AfterStartUp; +import org.thingsboard.server.queue.util.TbCoreComponent; +import org.thingsboard.server.service.queue.consumer.QueueConsumerManager; + +import javax.annotation.PreDestroy; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Set; +import java.util.UUID; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; + +@TbCoreComponent +@Service +@Slf4j +public class HousekeeperReprocessingService { + + private final HousekeeperConfig config; + private final HousekeeperService housekeeperService; + private final QueueConsumerManager> consumer; + private final TbQueueProducer> producer; + private final TopicPartitionInfo submitTpi; + + private final ExecutorService consumerExecutor = Executors.newSingleThreadExecutor(ThingsBoardThreadFactory.forName("housekeeper-reprocessing-consumer")); + + public HousekeeperReprocessingService(HousekeeperConfig config, + @Lazy HousekeeperService housekeeperService, + TbCoreQueueFactory queueFactory) { + this.config = config; + this.housekeeperService = housekeeperService; + this.consumer = QueueConsumerManager.>builder() + .name("Housekeeper reprocessing") + .msgPackProcessor(this::processMsgs) + .pollInterval(config.getPollInterval()) + .consumerCreator(queueFactory::createHousekeeperReprocessingMsgConsumer) + .consumerExecutor(consumerExecutor) + .build(); + this.producer = queueFactory.createHousekeeperReprocessingMsgProducer(); + this.submitTpi = TopicPartitionInfo.builder().topic(producer.getDefaultTopic()).build(); + } + + @AfterStartUp(order = AfterStartUp.REGULAR_SERVICE) + public void afterStartUp() { + consumer.subscribe(); // Kafka topic for tasks reprocessing has only 1 partition, so only one TB Core will reprocess tasks + consumer.launch(); + } + + private void processMsgs(List> msgs, TbQueueConsumer> consumer) throws Exception { + Thread.sleep(config.getTaskReprocessingDelay()); + + for (TbProtoQueueMsg msg : msgs) { + log.trace("Reprocessing task: {}", msg); + try { + housekeeperService.processTask(msg.getValue()); + } catch (InterruptedException e) { + return; + } catch (Throwable e) { + log.error("Unexpected error during message reprocessing [{}]", msg, e); + submitForReprocessing(msg.getValue(), e); + } + } + consumer.commit(); + } + + public void submitForReprocessing(ToHousekeeperServiceMsg msg, Throwable error) { + HousekeeperTaskProto task = msg.getTask(); + Set errors = new LinkedHashSet<>(task.getErrorsList()); + errors.add(StringUtils.truncate(ExceptionUtils.getStackTrace(error), 1024)); + msg = msg.toBuilder() + .setTask(task.toBuilder() + .setAttempt(task.getAttempt() + 1) + .clearErrors().addAllErrors(errors) + .build()) + .build(); + + log.trace("Submitting for reprocessing: {}", msg); + producer.send(submitTpi, new TbProtoQueueMsg<>(UUID.randomUUID(), msg), null); + } + + @PreDestroy + private void stop() { + consumer.stop(); + consumerExecutor.shutdownNow(); + log.info("Stopped Housekeeper reprocessing service"); + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/housekeeper/HousekeeperService.java b/application/src/main/java/org/thingsboard/server/service/housekeeper/HousekeeperService.java new file mode 100644 index 0000000000..06e38d0160 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/housekeeper/HousekeeperService.java @@ -0,0 +1,166 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.housekeeper; + +import lombok.extern.slf4j.Slf4j; +import org.springframework.context.annotation.Lazy; +import org.springframework.stereotype.Service; +import org.thingsboard.common.util.JacksonUtil; +import org.thingsboard.common.util.ThingsBoardThreadFactory; +import org.thingsboard.server.common.data.housekeeper.HousekeeperTask; +import org.thingsboard.server.common.data.housekeeper.HousekeeperTaskType; +import org.thingsboard.server.common.data.notification.rule.trigger.TaskProcessingFailureTrigger; +import org.thingsboard.server.common.msg.notification.NotificationRuleProcessor; +import org.thingsboard.server.gen.transport.TransportProtos.ToHousekeeperServiceMsg; +import org.thingsboard.server.queue.TbQueueConsumer; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; +import org.thingsboard.server.queue.housekeeper.HousekeeperConfig; +import org.thingsboard.server.queue.provider.TbCoreQueueFactory; +import org.thingsboard.server.queue.util.AfterStartUp; +import org.thingsboard.server.queue.util.TbCoreComponent; +import org.thingsboard.server.service.housekeeper.processor.HousekeeperTaskProcessor; +import org.thingsboard.server.service.housekeeper.stats.HousekeeperStatsService; +import org.thingsboard.server.service.queue.consumer.QueueConsumerManager; + +import javax.annotation.PreDestroy; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; +import java.util.stream.Collectors; + +@TbCoreComponent +@Service +@Slf4j +public class HousekeeperService { + + private final Map> taskProcessors; + + private final HousekeeperConfig config; + private final HousekeeperReprocessingService reprocessingService; + private final Optional statsService; + private final NotificationRuleProcessor notificationRuleProcessor; + private final QueueConsumerManager> consumer; + + private final ExecutorService consumerExecutor = Executors.newSingleThreadExecutor(ThingsBoardThreadFactory.forName("housekeeper-consumer")); + private final ExecutorService taskExecutor = Executors.newSingleThreadExecutor(ThingsBoardThreadFactory.forName("housekeeper-task-processor")); + + public HousekeeperService(HousekeeperConfig config, + HousekeeperReprocessingService reprocessingService, + TbCoreQueueFactory queueFactory, + Optional statsService, + NotificationRuleProcessor notificationRuleProcessor, + @Lazy List> taskProcessors) { + this.config = config; + this.reprocessingService = reprocessingService; + this.statsService = statsService; + this.notificationRuleProcessor = notificationRuleProcessor; + this.consumer = QueueConsumerManager.>builder() + .name("Housekeeper") + .msgPackProcessor(this::processMsgs) + .pollInterval(config.getPollInterval()) + .consumerCreator(queueFactory::createHousekeeperMsgConsumer) + .consumerExecutor(consumerExecutor) + .build(); + this.taskProcessors = taskProcessors.stream().collect(Collectors.toMap(HousekeeperTaskProcessor::getTaskType, p -> p)); + } + + @AfterStartUp(order = AfterStartUp.REGULAR_SERVICE) + public void afterStartUp() { + consumer.subscribe(); + consumer.launch(); + } + + private void processMsgs(List> msgs, TbQueueConsumer> consumer) { + for (TbProtoQueueMsg msg : msgs) { + log.trace("Processing task: {}", msg); + try { + processTask(msg.getValue()); + } catch (InterruptedException e) { + return; + } catch (Throwable e) { + log.error("Unexpected error during message processing [{}]", msg, e); + reprocessingService.submitForReprocessing(msg.getValue(), e); + } + } + consumer.commit(); + } + + @SuppressWarnings("unchecked") + protected void processTask(ToHousekeeperServiceMsg msg) throws Exception { + HousekeeperTask task = JacksonUtil.fromString(msg.getTask().getValue(), HousekeeperTask.class); + HousekeeperTaskType taskType = task.getTaskType(); + if (config.getDisabledTaskTypes().contains(taskType)) { + log.debug("Task type {} is disabled, ignoring {}", taskType, task); + return; + } + HousekeeperTaskProcessor taskProcessor = (HousekeeperTaskProcessor) taskProcessors.get(taskType); + if (taskProcessor == null) { + throw new IllegalArgumentException("Unsupported task type " + taskType); + } + + try { + long startTs = System.currentTimeMillis(); + Future future = taskExecutor.submit(() -> { + taskProcessor.process((T) task); + return null; + }); + future.get(config.getTaskProcessingTimeout(), TimeUnit.MILLISECONDS); + + long timing = System.currentTimeMillis() - startTs; + if (log.isDebugEnabled()) { + log.debug("[{}] Processed {} in {} ms (attempt {})", task.getTenantId(), task.getDescription(), timing, msg.getTask().getAttempt()); + } + statsService.ifPresent(statsService -> statsService.reportProcessed(taskType, msg, timing)); + } catch (InterruptedException e) { + throw e; + } catch (Throwable e) { + Throwable error = e; + if (e instanceof ExecutionException) { + error = e.getCause(); + } else if (e instanceof TimeoutException) { + error = new TimeoutException("Timeout after " + config.getTaskProcessingTimeout() + " seconds"); + } + + if (msg.getTask().getAttempt() < config.getMaxReprocessingAttempts()) { + log.warn("[{}] Failed to process {} (attempt {}), submitting for reprocessing", + task.getTenantId(), task.getDescription(), msg.getTask().getAttempt(), error); + reprocessingService.submitForReprocessing(msg, error); + } else { + log.error("[{}] Failed to process task in {} attempts: {}", task.getTenantId(), msg.getTask().getAttempt(), msg, e); + notificationRuleProcessor.process(TaskProcessingFailureTrigger.builder() + .task(task) + .error(error) + .attempt(msg.getTask().getAttempt()) + .build()); + } + statsService.ifPresent(statsService -> statsService.reportFailure(taskType, msg)); + } + } + + @PreDestroy + private void stop() { + consumer.stop(); + consumerExecutor.shutdownNow(); + log.info("Stopped Housekeeper service"); + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/housekeeper/InMemoryHouseKeeperServiceService.java b/application/src/main/java/org/thingsboard/server/service/housekeeper/InMemoryHouseKeeperServiceService.java deleted file mode 100644 index 4870990606..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/housekeeper/InMemoryHouseKeeperServiceService.java +++ /dev/null @@ -1,103 +0,0 @@ -/** - * Copyright © 2016-2024 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.housekeeper; - -import com.google.common.util.concurrent.FutureCallback; -import com.google.common.util.concurrent.Futures; -import com.google.common.util.concurrent.ListenableFuture; -import com.google.common.util.concurrent.ListeningExecutorService; -import com.google.common.util.concurrent.MoreExecutors; -import lombok.RequiredArgsConstructor; -import lombok.extern.slf4j.Slf4j; -import org.springframework.stereotype.Component; -import org.springframework.transaction.event.TransactionalEventListener; -import org.thingsboard.common.util.ThingsBoardThreadFactory; -import org.thingsboard.server.common.data.EntityType; -import org.thingsboard.server.common.data.User; -import org.thingsboard.server.common.data.id.AlarmId; -import org.thingsboard.server.common.data.id.EntityId; -import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.dao.eventsourcing.DeleteEntityEvent; -import org.thingsboard.server.dao.housekeeper.HouseKeeperService; -import org.thingsboard.server.service.entitiy.alarm.TbAlarmService; - -import javax.annotation.PostConstruct; -import javax.annotation.PreDestroy; -import java.util.List; -import java.util.concurrent.Executors; -import java.util.concurrent.atomic.AtomicInteger; - -@Component -@RequiredArgsConstructor -@Slf4j -public class InMemoryHouseKeeperServiceService implements HouseKeeperService { - - final TbAlarmService alarmService; - - ListeningExecutorService executor; - - AtomicInteger queueSize = new AtomicInteger(); - AtomicInteger totalProcessedCounter = new AtomicInteger(); - - @PostConstruct - public void init() { - log.debug("Starting HouseKeeper service"); - executor = MoreExecutors.listeningDecorator(Executors.newSingleThreadExecutor(ThingsBoardThreadFactory.forName("housekeeper"))); - } - - @PreDestroy - public void destroy() { - if (executor != null) { - log.debug("Stopping HouseKeeper service"); - executor.shutdown(); - } - } - - @TransactionalEventListener(fallbackExecution = true) - public void handleEvent(DeleteEntityEvent event) { - log.trace("[{}] DeleteEntityEvent handler: {}", event.getTenantId(), event); - EntityId entityId = event.getEntityId(); - if (EntityType.USER.equals(entityId.getEntityType())) { - unassignDeletedUserAlarms(event.getTenantId(), (User) event.getEntity(), event.getTs()); - } - } - - @Override - public ListenableFuture> unassignDeletedUserAlarms(TenantId tenantId, User user, long unassignTs) { - log.debug("[{}][{}] unassignDeletedUserAlarms submitting, pending queue size: {} ", tenantId, user.getId().getId(), queueSize.get()); - queueSize.incrementAndGet(); - ListenableFuture> future = executor.submit(() -> alarmService.unassignDeletedUserAlarms(tenantId, user, unassignTs)); - Futures.addCallback(future, new FutureCallback<>() { - @Override - public void onSuccess(List alarmIds) { - queueSize.decrementAndGet(); - totalProcessedCounter.incrementAndGet(); - log.debug("[{}][{}] unassignDeletedUserAlarms finished, pending queue size: {}, total processed count: {} ", - tenantId, user.getId().getId(), queueSize.get(), totalProcessedCounter.get()); - } - - @Override - public void onFailure(Throwable throwable) { - queueSize.decrementAndGet(); - totalProcessedCounter.incrementAndGet(); - log.error("[{}][{}] unassignDeletedUserAlarms failed, pending queue size: {}, total processed count: {}", - tenantId, user.getId().getId(), queueSize.get(), totalProcessedCounter.get(), throwable); - } - }, MoreExecutors.directExecutor()); - return future; - } - -} diff --git a/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/AlarmsDeletionTaskProcessor.java b/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/AlarmsDeletionTaskProcessor.java new file mode 100644 index 0000000000..cc6cb9bbab --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/AlarmsDeletionTaskProcessor.java @@ -0,0 +1,80 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.housekeeper.processor; + +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.stereotype.Component; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.housekeeper.AlarmsDeletionHousekeeperTask; +import org.thingsboard.server.common.data.housekeeper.HousekeeperTaskType; +import org.thingsboard.server.common.data.id.AlarmId; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.util.TbPair; +import org.thingsboard.server.dao.alarm.AlarmService; + +import java.util.List; +import java.util.UUID; + +@Component +@RequiredArgsConstructor +@Slf4j +public class AlarmsDeletionTaskProcessor extends HousekeeperTaskProcessor { + + private final AlarmService alarmService; + + @Override + public void process(AlarmsDeletionHousekeeperTask task) throws Exception { + EntityId entityId = task.getEntityId(); + EntityType entityType = entityId.getEntityType(); + TenantId tenantId = task.getTenantId(); + + if (entityType == EntityType.DEVICE || entityType == EntityType.ASSET) { + if (task.getAlarms() == null) { + AlarmId lastId = null; + long lastCreatedTime = 0; + while (true) { + List> alarms = alarmService.findAlarmIdsByOriginatorId(tenantId, entityId, lastCreatedTime, lastId, 128); + if (alarms.isEmpty()) { + break; + } + + housekeeperClient.submitTask(new AlarmsDeletionHousekeeperTask(tenantId, entityId, alarms.stream().map(TbPair::getFirst).toList())); + + TbPair last = alarms.get(alarms.size() - 1); + lastId = new AlarmId(last.getFirst()); + lastCreatedTime = last.getSecond(); + log.debug("[{}][{}][{}] Submitted task for deleting {} alarms", tenantId, entityType, entityId, alarms.size()); + } + } else { + for (UUID alarmId : task.getAlarms()) { + alarmService.delAlarm(tenantId, new AlarmId(alarmId)); + } + log.debug("[{}][{}][{}] Deleted {} alarms", tenantId, entityType, entityId, task.getAlarms().size()); + } + } + + int count = alarmService.deleteEntityAlarmRecords(tenantId, entityId); + log.debug("[{}][{}][{}] Deleted {} entity alarms", tenantId, entityType, entityId, count); + } + + @Override + public HousekeeperTaskType getTaskType() { + return HousekeeperTaskType.DELETE_ALARMS; + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/AlarmsUnassignTaskProcessor.java b/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/AlarmsUnassignTaskProcessor.java new file mode 100644 index 0000000000..3ba6ecde52 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/AlarmsUnassignTaskProcessor.java @@ -0,0 +1,47 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.housekeeper.processor; + +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.stereotype.Component; +import org.thingsboard.server.common.data.id.AlarmId; +import org.thingsboard.server.common.data.id.UserId; +import org.thingsboard.server.common.data.housekeeper.HousekeeperTaskType; +import org.thingsboard.server.common.data.housekeeper.AlarmsUnassignHousekeeperTask; +import org.thingsboard.server.service.entitiy.alarm.TbAlarmService; + +import java.util.List; + +@Component +@RequiredArgsConstructor +@Slf4j +public class AlarmsUnassignTaskProcessor extends HousekeeperTaskProcessor { + + private final TbAlarmService alarmService; + + @Override + public void process(AlarmsUnassignHousekeeperTask task) throws Exception { + List alarms = alarmService.unassignDeletedUserAlarms(task.getTenantId(), (UserId) task.getEntityId(), task.getUserTitle(), task.getTs()); + log.debug("[{}][{}] Unassigned {} alarms", task.getTenantId(), task.getEntityId(), alarms.size()); + } + + @Override + public HousekeeperTaskType getTaskType() { + return HousekeeperTaskType.UNASSIGN_ALARMS; + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/AttributesDeletionTaskProcessor.java b/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/AttributesDeletionTaskProcessor.java new file mode 100644 index 0000000000..5514921e2e --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/AttributesDeletionTaskProcessor.java @@ -0,0 +1,43 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.housekeeper.processor; + +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.stereotype.Component; +import org.thingsboard.server.dao.attributes.AttributesService; +import org.thingsboard.server.common.data.housekeeper.HousekeeperTask; +import org.thingsboard.server.common.data.housekeeper.HousekeeperTaskType; + +@Component +@RequiredArgsConstructor +@Slf4j +public class AttributesDeletionTaskProcessor extends HousekeeperTaskProcessor { + + private final AttributesService attributesService; + + @Override + public void process(HousekeeperTask task) throws Exception { + int deletedCount = attributesService.removeAllByEntityId(task.getTenantId(), task.getEntityId()); + log.debug("[{}][{}][{}] Deleted {} attributes", task.getTenantId(), task.getEntityId().getEntityType(), task.getEntityId(), deletedCount); + } + + @Override + public HousekeeperTaskType getTaskType() { + return HousekeeperTaskType.DELETE_ATTRIBUTES; + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/EntitiesDeletionTaskProcessor.java b/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/EntitiesDeletionTaskProcessor.java new file mode 100644 index 0000000000..186a290528 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/EntitiesDeletionTaskProcessor.java @@ -0,0 +1,57 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.housekeeper.processor; + +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.stereotype.Component; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.housekeeper.EntitiesDeletionHousekeeperTask; +import org.thingsboard.server.common.data.housekeeper.HousekeeperTaskType; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.EntityIdFactory; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.dao.entity.EntityDaoService; +import org.thingsboard.server.dao.entity.EntityServiceRegistry; + +import java.util.UUID; + +@Component +@RequiredArgsConstructor +@Slf4j +public class EntitiesDeletionTaskProcessor extends HousekeeperTaskProcessor { + + private final EntityServiceRegistry entityServiceRegistry; + + @Override + public void process(EntitiesDeletionHousekeeperTask task) throws Exception { + EntityType entityType = task.getEntityType(); + TenantId tenantId = task.getTenantId(); + EntityDaoService entityService = entityServiceRegistry.getServiceByEntityType(entityType); + + for (UUID entityUuid : task.getEntities()) { + EntityId entityId = EntityIdFactory.getByTypeAndUuid(entityType, entityUuid); + entityService.deleteEntity(tenantId, entityId, true); + } + log.debug("[{}] Deleted {} {}s", tenantId, task.getEntities().size(), entityType.getNormalName().toLowerCase()); + } + + @Override + public HousekeeperTaskType getTaskType() { + return HousekeeperTaskType.DELETE_ENTITIES; + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/EventsDeletionTaskProcessor.java b/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/EventsDeletionTaskProcessor.java new file mode 100644 index 0000000000..bf9c528eb7 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/EventsDeletionTaskProcessor.java @@ -0,0 +1,40 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.housekeeper.processor; + +import lombok.RequiredArgsConstructor; +import org.springframework.stereotype.Component; +import org.thingsboard.server.dao.event.EventService; +import org.thingsboard.server.common.data.housekeeper.HousekeeperTask; +import org.thingsboard.server.common.data.housekeeper.HousekeeperTaskType; + +@Component +@RequiredArgsConstructor +public class EventsDeletionTaskProcessor extends HousekeeperTaskProcessor { + + private final EventService eventService; + + @Override + public void process(HousekeeperTask task) throws Exception { + eventService.removeEvents(task.getTenantId(), task.getEntityId(), null, 0L, System.currentTimeMillis()); + } + + @Override + public HousekeeperTaskType getTaskType() { + return HousekeeperTaskType.DELETE_EVENTS; + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/HousekeeperTaskProcessor.java b/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/HousekeeperTaskProcessor.java new file mode 100644 index 0000000000..f193b154db --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/HousekeeperTaskProcessor.java @@ -0,0 +1,32 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.housekeeper.processor; + +import org.springframework.beans.factory.annotation.Autowired; +import org.thingsboard.server.common.data.housekeeper.HousekeeperTask; +import org.thingsboard.server.common.data.housekeeper.HousekeeperTaskType; +import org.thingsboard.server.common.msg.housekeeper.HousekeeperClient; + +public abstract class HousekeeperTaskProcessor { + + @Autowired + protected HousekeeperClient housekeeperClient; + + public abstract void process(T task) throws Exception; + + public abstract HousekeeperTaskType getTaskType(); + +} diff --git a/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/LatestTsDeletionTaskProcessor.java b/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/LatestTsDeletionTaskProcessor.java new file mode 100644 index 0000000000..1f4b6b57ae --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/LatestTsDeletionTaskProcessor.java @@ -0,0 +1,45 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.housekeeper.processor; + +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.stereotype.Component; +import org.thingsboard.server.common.data.housekeeper.HousekeeperTaskType; +import org.thingsboard.server.common.data.housekeeper.LatestTsDeletionHousekeeperTask; +import org.thingsboard.server.dao.timeseries.TimeseriesService; + +import java.util.List; + +@Component +@RequiredArgsConstructor +@Slf4j +public class LatestTsDeletionTaskProcessor extends HousekeeperTaskProcessor { + + private final TimeseriesService timeseriesService; + + @Override + public void process(LatestTsDeletionHousekeeperTask task) throws Exception { + timeseriesService.removeLatest(task.getTenantId(), task.getEntityId(), List.of(task.getKey())).get(); + log.debug("[{}][{}][{}] Deleted latest telemetry for key '{}'", task.getTenantId(), task.getEntityId().getEntityType(), task.getEntityId(), task.getKey()); + } + + @Override + public HousekeeperTaskType getTaskType() { + return HousekeeperTaskType.DELETE_LATEST_TS; + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/TelemetryDeletionTaskProcessor.java b/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/TelemetryDeletionTaskProcessor.java new file mode 100644 index 0000000000..5271e77c87 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/TelemetryDeletionTaskProcessor.java @@ -0,0 +1,60 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.housekeeper.processor; + +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.stereotype.Component; +import org.thingsboard.server.common.data.housekeeper.HousekeeperTask; +import org.thingsboard.server.common.data.housekeeper.HousekeeperTaskType; +import org.thingsboard.server.common.data.housekeeper.LatestTsDeletionHousekeeperTask; +import org.thingsboard.server.common.data.housekeeper.TsHistoryDeletionHousekeeperTask; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.dao.timeseries.TimeseriesService; + +import java.util.List; + +@Component +@RequiredArgsConstructor +@Slf4j +public class TelemetryDeletionTaskProcessor extends HousekeeperTaskProcessor { + + private final TimeseriesService timeseriesService; + + @Override + public void process(HousekeeperTask task) throws Exception { + TenantId tenantId = task.getTenantId(); + EntityId entityId = task.getEntityId(); + List keys = timeseriesService.findAllKeysByEntityIds(tenantId, List.of(entityId)); + + for (String key : keys) { + var latestTsDeletionTask = new LatestTsDeletionHousekeeperTask(tenantId, entityId, key); + housekeeperClient.submitTask(latestTsDeletionTask); + + var tsHistoryDeletionTask = new TsHistoryDeletionHousekeeperTask(tenantId, entityId, key); + housekeeperClient.submitTask(tsHistoryDeletionTask); + } + + log.trace("[{}][{}][{}] Submitted latest and ts history deletion tasks for {} keys", tenantId, entityId.getEntityType(), entityId, keys.size()); + } + + @Override + public HousekeeperTaskType getTaskType() { + return HousekeeperTaskType.DELETE_TELEMETRY; + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/TenantEntitiesDeletionTaskProcessor.java b/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/TenantEntitiesDeletionTaskProcessor.java new file mode 100644 index 0000000000..386afb1517 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/TenantEntitiesDeletionTaskProcessor.java @@ -0,0 +1,63 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.housekeeper.processor; + +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.stereotype.Component; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.housekeeper.EntitiesDeletionHousekeeperTask; +import org.thingsboard.server.common.data.housekeeper.HousekeeperTaskType; +import org.thingsboard.server.common.data.housekeeper.TenantEntitiesDeletionHousekeeperTask; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.dao.Dao; +import org.thingsboard.server.dao.entity.EntityDaoRegistry; + +import java.util.List; +import java.util.UUID; + +@Component +@RequiredArgsConstructor +@Slf4j +public class TenantEntitiesDeletionTaskProcessor extends HousekeeperTaskProcessor { + + private final EntityDaoRegistry entityDaoRegistry; + + @Override + public void process(TenantEntitiesDeletionHousekeeperTask task) throws Exception { + EntityType entityType = task.getEntityType(); + TenantId tenantId = task.getTenantId(); + Dao entityDao = entityDaoRegistry.getDao(entityType); + + UUID last = null; + while (true) { + List entities = entityDao.findIdsByTenantIdAndIdOffset(tenantId, last, 128); + if (entities.isEmpty()) { + break; + } + + housekeeperClient.submitTask(new EntitiesDeletionHousekeeperTask(tenantId, entityType, entities)); + last = entities.get(entities.size() - 1); + log.debug("[{}] Submitted task for deleting {} {}s", tenantId, entities.size(), entityType.getNormalName().toLowerCase()); + } + } + + @Override + public HousekeeperTaskType getTaskType() { + return HousekeeperTaskType.DELETE_TENANT_ENTITIES; + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/TsHistoryDeletionTaskProcessor.java b/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/TsHistoryDeletionTaskProcessor.java new file mode 100644 index 0000000000..cee99ae93e --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/housekeeper/processor/TsHistoryDeletionTaskProcessor.java @@ -0,0 +1,48 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.housekeeper.processor; + +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.stereotype.Component; +import org.thingsboard.server.common.data.housekeeper.HousekeeperTaskType; +import org.thingsboard.server.common.data.housekeeper.TsHistoryDeletionHousekeeperTask; +import org.thingsboard.server.common.data.kv.BaseDeleteTsKvQuery; +import org.thingsboard.server.common.data.kv.DeleteTsKvQuery; +import org.thingsboard.server.dao.timeseries.TimeseriesService; + +import java.util.List; + +@Component +@RequiredArgsConstructor +@Slf4j +public class TsHistoryDeletionTaskProcessor extends HousekeeperTaskProcessor { + + private final TimeseriesService timeseriesService; + + @Override + public void process(TsHistoryDeletionHousekeeperTask task) throws Exception { + DeleteTsKvQuery deleteQuery = new BaseDeleteTsKvQuery(task.getKey(), 0, System.currentTimeMillis(), false, false); + timeseriesService.remove(task.getTenantId(), task.getEntityId(), List.of(deleteQuery)).get(); + log.debug("[{}][{}][{}] Deleted timeseries history for key '{}'", task.getTenantId(), task.getEntityId().getEntityType(), task.getEntityId(), task.getKey()); + } + + @Override + public HousekeeperTaskType getTaskType() { + return HousekeeperTaskType.DELETE_TS_HISTORY; + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/housekeeper/stats/HousekeeperStatsService.java b/application/src/main/java/org/thingsboard/server/service/housekeeper/stats/HousekeeperStatsService.java new file mode 100644 index 0000000000..6723d51625 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/housekeeper/stats/HousekeeperStatsService.java @@ -0,0 +1,125 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.housekeeper.stats; + +import lombok.Getter; +import lombok.extern.slf4j.Slf4j; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.scheduling.annotation.Scheduled; +import org.springframework.stereotype.Service; +import org.thingsboard.server.common.data.housekeeper.HousekeeperTaskType; +import org.thingsboard.server.common.stats.DefaultCounter; +import org.thingsboard.server.common.stats.StatsCounter; +import org.thingsboard.server.common.stats.StatsFactory; +import org.thingsboard.server.common.stats.StatsTimer; +import org.thingsboard.server.common.stats.StatsType; +import org.thingsboard.server.gen.transport.TransportProtos.ToHousekeeperServiceMsg; + +import java.util.ArrayList; +import java.util.EnumMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.stream.Collectors; + +@Service +@Slf4j +@ConditionalOnProperty(name = "queue.core.housekeeper.stats.enabled", havingValue = "true", matchIfMissing = true) +public class HousekeeperStatsService { + + private final Map stats = new EnumMap<>(HousekeeperTaskType.class); + + public HousekeeperStatsService(StatsFactory statsFactory) { + for (HousekeeperTaskType taskType : HousekeeperTaskType.values()) { + stats.put(taskType, new HousekeeperStats(taskType, statsFactory)); + } + } + + @Scheduled(initialDelayString = "${queue.core.housekeeper.stats.print-interval-ms:60000}", + fixedDelayString = "${queue.core.housekeeper.stats.print-interval-ms:60000}") + private void reportStats() { + String statsStr = stats.values().stream().map(stats -> { + String countersStr = stats.getCounters().stream() + .filter(counter -> counter.get() > 0) + .map(counter -> counter.getName() + " = [" + counter.get() + "]") + .collect(Collectors.joining(" ")); + if (countersStr.isEmpty()) { + return null; + } else { + return stats.getTaskType() + " " + countersStr + " avgProcessingTime [" + stats.getProcessingTimer().getAvg() + " ms]"; + } + }).filter(Objects::nonNull).collect(Collectors.joining(", ")); + + if (!statsStr.isEmpty()) { + stats.values().forEach(HousekeeperStats::reset); + log.info("Housekeeper stats: {}", statsStr); + } + } + + public void reportProcessed(HousekeeperTaskType taskType, ToHousekeeperServiceMsg msg, long timing) { + HousekeeperStats stats = this.stats.get(taskType); + if (msg.getTask().getErrorsCount() == 0) { + stats.getProcessedCounter().increment(); + } else { + stats.getReprocessedCounter().increment(); + } + stats.getProcessingTimer().record(timing); + } + + public void reportFailure(HousekeeperTaskType taskType, ToHousekeeperServiceMsg msg) { + HousekeeperStats stats = this.stats.get(taskType); + if (msg.getTask().getErrorsCount() == 0) { + stats.getFailedProcessingCounter().increment(); + } else { + stats.getFailedReprocessingCounter().increment(); + } + } + + @Getter + static class HousekeeperStats { + private final HousekeeperTaskType taskType; + private final List counters = new ArrayList<>(); + + private final StatsCounter processedCounter; + private final StatsCounter failedProcessingCounter; + private final StatsCounter reprocessedCounter; + private final StatsCounter failedReprocessingCounter; + + private final StatsTimer processingTimer; + + public HousekeeperStats(HousekeeperTaskType taskType, StatsFactory statsFactory) { + this.taskType = taskType; + this.processedCounter = register("processed", statsFactory); + this.failedProcessingCounter = register("failedProcessing", statsFactory); + this.reprocessedCounter = register("reprocessed", statsFactory); + this.failedReprocessingCounter = register("failedReprocessing", statsFactory); + this.processingTimer = statsFactory.createTimer(StatsType.HOUSEKEEPER, "processingTime", "taskType", taskType.name()); + } + + private StatsCounter register(String statsName, StatsFactory statsFactory) { + StatsCounter counter = statsFactory.createStatsCounter(StatsType.HOUSEKEEPER.getName(), statsName, "taskType", taskType.name()); + counters.add(counter); + return counter; + } + + public void reset() { + counters.forEach(DefaultCounter::clear); + processingTimer.reset(); + } + + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/install/DefaultSystemDataLoaderService.java b/application/src/main/java/org/thingsboard/server/service/install/DefaultSystemDataLoaderService.java index 4da6a383e8..8c86c16edb 100644 --- a/application/src/main/java/org/thingsboard/server/service/install/DefaultSystemDataLoaderService.java +++ b/application/src/main/java/org/thingsboard/server/service/install/DefaultSystemDataLoaderService.java @@ -549,7 +549,7 @@ public class DefaultSystemDataLoaderService implements SystemDataLoaderService { } else { ListenableFuture> saveFuture = attributesService.save(TenantId.SYS_TENANT_ID, deviceId, AttributeScope.SERVER_SCOPE, Collections.singletonList(new BaseAttributeKvEntry(new BooleanDataEntry(key, value) - , System.currentTimeMillis()))); + , System.currentTimeMillis()))); addTsCallback(saveFuture, new TelemetrySaveCallback<>(deviceId, key, value)); } } @@ -693,23 +693,26 @@ public class DefaultSystemDataLoaderService implements SystemDataLoaderService { @Override @SneakyThrows - public void updateDefaultNotificationConfigs() { - PageDataIterable tenants = new PageDataIterable<>(tenantService::findTenantsIds, 500); - ExecutorService executor = Executors.newFixedThreadPool(Math.max(Runtime.getRuntime().availableProcessors(), 4)); - log.info("Updating default edge failure notification configs for all tenants"); - AtomicInteger count = new AtomicInteger(); - for (TenantId tenantId : tenants) { - executor.submit(() -> { - notificationSettingsService.updateDefaultNotificationConfigs(tenantId); - int n = count.incrementAndGet(); - if (n % 500 == 0) { - log.info("{} tenants processed", n); - } - }); - } - executor.shutdown(); - executor.awaitTermination(Integer.MAX_VALUE, TimeUnit.SECONDS); + public void updateDefaultNotificationConfigs(boolean updateTenants) { + log.info("Updating notification configs..."); notificationSettingsService.updateDefaultNotificationConfigs(TenantId.SYS_TENANT_ID); + + if (updateTenants) { + PageDataIterable tenants = new PageDataIterable<>(tenantService::findTenantsIds, 500); + ExecutorService executor = Executors.newFixedThreadPool(Math.max(Runtime.getRuntime().availableProcessors(), 4)); + AtomicInteger count = new AtomicInteger(); + for (TenantId tenantId : tenants) { + executor.submit(() -> { + notificationSettingsService.updateDefaultNotificationConfigs(tenantId); + int n = count.incrementAndGet(); + if (n % 500 == 0) { + log.info("{} tenants processed", n); + } + }); + } + executor.shutdown(); + executor.awaitTermination(Integer.MAX_VALUE, TimeUnit.SECONDS); + } } } diff --git a/application/src/main/java/org/thingsboard/server/service/install/SystemDataLoaderService.java b/application/src/main/java/org/thingsboard/server/service/install/SystemDataLoaderService.java index eeac1b6aa4..7c05ff620f 100644 --- a/application/src/main/java/org/thingsboard/server/service/install/SystemDataLoaderService.java +++ b/application/src/main/java/org/thingsboard/server/service/install/SystemDataLoaderService.java @@ -35,6 +35,6 @@ public interface SystemDataLoaderService { void createDefaultNotificationConfigs(); - void updateDefaultNotificationConfigs(); + void updateDefaultNotificationConfigs(boolean updateTenants); } diff --git a/application/src/main/java/org/thingsboard/server/service/notification/rule/trigger/RuleEngineComponentLifecycleEventTriggerProcessor.java b/application/src/main/java/org/thingsboard/server/service/notification/rule/trigger/RuleEngineComponentLifecycleEventTriggerProcessor.java index c17805e1b6..a009841897 100644 --- a/application/src/main/java/org/thingsboard/server/service/notification/rule/trigger/RuleEngineComponentLifecycleEventTriggerProcessor.java +++ b/application/src/main/java/org/thingsboard/server/service/notification/rule/trigger/RuleEngineComponentLifecycleEventTriggerProcessor.java @@ -47,7 +47,7 @@ public class RuleEngineComponentLifecycleEventTriggerProcessor implements Notifi return false; } } - if (!partitionService.resolve(ServiceType.TB_RULE_ENGINE, trigger.getTenantId(), trigger.getComponentId()).isMyPartition()) { + if (!partitionService.isMyPartition(ServiceType.TB_RULE_ENGINE, trigger.getTenantId(), trigger.getComponentId())) { return false; } diff --git a/application/src/main/java/org/thingsboard/server/service/notification/rule/trigger/TaskProcessingFailureTriggerProcessor.java b/application/src/main/java/org/thingsboard/server/service/notification/rule/trigger/TaskProcessingFailureTriggerProcessor.java new file mode 100644 index 0000000000..7754690215 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/notification/rule/trigger/TaskProcessingFailureTriggerProcessor.java @@ -0,0 +1,53 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.notification.rule.trigger; + +import org.apache.commons.lang3.exception.ExceptionUtils; +import org.springframework.stereotype.Service; +import org.thingsboard.server.common.data.StringUtils; +import org.thingsboard.server.common.data.housekeeper.HousekeeperTask; +import org.thingsboard.server.common.data.notification.info.TaskProcessingFailureNotificationInfo; +import org.thingsboard.server.common.data.notification.rule.trigger.TaskProcessingFailureTrigger; +import org.thingsboard.server.common.data.notification.rule.trigger.config.NotificationRuleTriggerType; +import org.thingsboard.server.common.data.notification.rule.trigger.config.TaskProcessingFailureNotificationRuleTriggerConfig; + +@Service +public class TaskProcessingFailureTriggerProcessor implements NotificationRuleTriggerProcessor { + + @Override + public boolean matchesFilter(TaskProcessingFailureTrigger trigger, TaskProcessingFailureNotificationRuleTriggerConfig triggerConfig) { + return true; + } + + @Override + public TaskProcessingFailureNotificationInfo constructNotificationInfo(TaskProcessingFailureTrigger trigger) { + HousekeeperTask task = trigger.getTask(); + return TaskProcessingFailureNotificationInfo.builder() + .tenantId(task.getTenantId()) + .entityId(task.getEntityId()) + .taskType(task.getTaskType()) + .taskDescription(task.getDescription()) + .error(StringUtils.truncate(ExceptionUtils.getStackTrace(trigger.getError()), 1024)) + .attempt(trigger.getAttempt()) + .build(); + } + + @Override + public NotificationRuleTriggerType getTriggerType() { + return NotificationRuleTriggerType.TASK_PROCESSING_FAILURE; + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/queue/consumer/QueueConsumerManager.java b/application/src/main/java/org/thingsboard/server/service/queue/consumer/QueueConsumerManager.java new file mode 100644 index 0000000000..e7b3a2f2b7 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/queue/consumer/QueueConsumerManager.java @@ -0,0 +1,109 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.queue.consumer; + +import lombok.Builder; +import lombok.Getter; +import lombok.extern.slf4j.Slf4j; +import org.thingsboard.common.util.ThingsBoardThreadFactory; +import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; +import org.thingsboard.server.queue.TbQueueConsumer; +import org.thingsboard.server.queue.TbQueueMsg; + +import java.util.List; +import java.util.Set; +import java.util.concurrent.ExecutorService; +import java.util.function.Supplier; + +@Slf4j +public class QueueConsumerManager { + + private final String name; + private final MsgPackProcessor msgPackProcessor; + private final long pollInterval; + private final ExecutorService consumerExecutor; + private final String threadPrefix; + + @Getter + private final TbQueueConsumer consumer; + private volatile boolean stopped; + + @Builder + public QueueConsumerManager(String name, MsgPackProcessor msgPackProcessor, + long pollInterval, Supplier> consumerCreator, + ExecutorService consumerExecutor, String threadPrefix) { + this.name = name; + this.pollInterval = pollInterval; + this.msgPackProcessor = msgPackProcessor; + this.consumerExecutor = consumerExecutor; + this.threadPrefix = threadPrefix; + this.consumer = consumerCreator.get(); + } + + public void subscribe() { + consumer.subscribe(); + } + + public void subscribe(Set partitions) { + consumer.subscribe(partitions); + } + + public void launch() { + log.info("[{}] Launching consumer", name); + consumerExecutor.submit(() -> { + if (threadPrefix != null) { + ThingsBoardThreadFactory.addThreadNamePrefix(threadPrefix); + } + try { + consumerLoop(consumer); + } catch (Throwable e) { + log.error("Failure in consumer loop", e); + } + log.info("[{}] Consumer stopped", name); + }); + } + + private void consumerLoop(TbQueueConsumer consumer) { + while (!stopped && !consumer.isStopped()) { + try { + List msgs = consumer.poll(pollInterval); + if (msgs.isEmpty()) { + continue; + } + msgPackProcessor.process(msgs, consumer); + } catch (Exception e) { + if (!consumer.isStopped()) { + log.warn("Failed to process messages from queue", e); + try { + Thread.sleep(pollInterval); + } catch (InterruptedException interruptedException) { + log.trace("Failed to wait until the server has capacity to handle new requests", interruptedException); + } + } + } + } + } + + public void stop() { + log.debug("[{}] Stopping consumer", name); + stopped = true; + consumer.unsubscribe(); + } + + public interface MsgPackProcessor { + void process(List msgs, TbQueueConsumer consumer) throws Exception; + } +} \ No newline at end of file diff --git a/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/DefaultExportableEntitiesService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/DefaultExportableEntitiesService.java index 95cbc06b27..49314e62fd 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/DefaultExportableEntitiesService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/DefaultExportableEntitiesService.java @@ -147,10 +147,7 @@ public class DefaultExportableEntitiesService implements ExportableEntitiesServi public void removeById(TenantId tenantId, I id) { EntityType entityType = id.getEntityType(); EntityDaoService entityService = entityServiceRegistry.getServiceByEntityType(entityType); - if (entityService == null) { - throw new IllegalArgumentException("Unsupported entity type " + entityType); - } - entityService.deleteEntity(tenantId, id); + entityService.deleteEntity(tenantId, id, false); } private > ExportableEntityDao getExportableEntityDao(EntityType entityType) { diff --git a/application/src/main/java/org/thingsboard/server/service/ttl/AlarmsCleanUpService.java b/application/src/main/java/org/thingsboard/server/service/ttl/AlarmsCleanUpService.java index 76de56cf13..c0f3158c61 100644 --- a/application/src/main/java/org/thingsboard/server/service/ttl/AlarmsCleanUpService.java +++ b/application/src/main/java/org/thingsboard/server/service/ttl/AlarmsCleanUpService.java @@ -56,7 +56,6 @@ public class AlarmsCleanUpService { private final TenantService tenantService; private final AlarmDao alarmDao; private final AlarmService alarmService; - private final RelationService relationService; private final EntityActionService entityActionService; private final PartitionService partitionService; private final TbTenantProfileCache tenantProfileCache; diff --git a/application/src/main/resources/thingsboard.yml b/application/src/main/resources/thingsboard.yml index 608bdaf9ea..c7486c4cf3 100644 --- a/application/src/main/resources/thingsboard.yml +++ b/application/src/main/resources/thingsboard.yml @@ -1445,6 +1445,16 @@ queue: # tb_rule_engine.sq: # - key: max.poll.records # value: "${TB_QUEUE_KAFKA_SQ_MAX_POLL_RECORDS:1024}" + tb_housekeeper: + # Consumer properties for Housekeeper tasks topic + - key: max.poll.records + # Amount of records to be returned in a single poll. For Housekeeper tasks topic, we should consume messages (tasks) one by one + value: "${TB_QUEUE_KAFKA_HOUSEKEEPER_MAX_POLL_RECORDS:1}" + tb_housekeeper.reprocessing: + # Consumer properties for Housekeeper reprocessing topic + - key: max.poll.records + # Amount of records to be returned in a single poll. For Housekeeper reprocessing topic, we should consume messages (tasks) one by one + value: "${TB_QUEUE_KAFKA_HOUSEKEEPER_REPROCESSING_MAX_POLL_RECORDS:1}" other-inline: "${TB_QUEUE_KAFKA_OTHER_PROPERTIES:}" # In this section you can specify custom parameters (semicolon separated) for Kafka consumer/producer/admin # Example "metrics.recording.level:INFO;metrics.sample.window.ms:30000" other: # DEPRECATED. In this section, you can specify custom parameters for Kafka consumer/producer and expose the env variables to configure outside # - key: "request.timeout.ms" # refer to https://docs.confluent.io/platform/current/installation/configuration/producer-configs.html#producerconfigs_request.timeout.ms @@ -1466,6 +1476,10 @@ queue: ota-updates: "${TB_QUEUE_KAFKA_OTA_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:1048576000;partitions:10;min.insync.replicas:1}" # Kafka properties for Version Control topic version-control: "${TB_QUEUE_KAFKA_VC_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:1048576000;partitions:1;min.insync.replicas:1}" + # Kafka properties for Housekeeper tasks topic + housekeeper: "${TB_QUEUE_KAFKA_HOUSEKEEPER_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:1048576000;partitions:10;min.insync.replicas:1}" + # Kafka properties for Housekeeper reprocessing topic; retention.ms is set to 90 days; partitions is set to 1 since only one reprocessing service is running at a time + housekeeper-reprocessing: "${TB_QUEUE_KAFKA_HOUSEKEEPER_REPROCESSING_TOPIC_PROPERTIES:retention.ms:7776000000;segment.bytes:26214400;retention.bytes:1048576000;partitions:1;min.insync.replicas:1}" consumer-stats: # Prints lag between consumer group offset and last messages offset in Kafka topics enabled: "${TB_QUEUE_KAFKA_CONSUMER_STATS_ENABLED:true}" @@ -1621,6 +1635,29 @@ queue: enabled: "${TB_QUEUE_CORE_STATS_ENABLED:true}" # Statistics printing interval for Core microservices print-interval-ms: "${TB_QUEUE_CORE_STATS_PRINT_INTERVAL_MS:60000}" + housekeeper: + # Topic name for Housekeeper tasks + topic: "${TB_HOUSEKEEPER_TOPIC:tb_housekeeper}" + # Topic name for Housekeeper tasks to be reprocessed + reprocessing-topic: "${TB_HOUSEKEEPER_REPROCESSING_TOPIC:tb_housekeeper.reprocessing}" + # Poll interval for topics related to Housekeeper + poll-interval-ms: "${TB_HOUSEKEEPER_POLL_INTERVAL_MS:500}" + # Timeout in milliseconds for task processing. Tasks that fail to finish on time will be submitted for reprocessing + task-processing-timeout-ms: "${TB_HOUSEKEEPER_TASK_PROCESSING_TIMEOUT_MS:120000}" + # Comma-separated list of task types that shouldn't be processed. Available task types: + # DELETE_ATTRIBUTES, DELETE_TELEMETRY (both DELETE_LATEST_TS and DELETE_TS_HISTORY will be disabled), + # DELETE_LATEST_TS, DELETE_TS_HISTORY, DELETE_EVENTS, DELETE_ALARMS, UNASSIGN_ALARMS + disabled-task-types: "${TB_HOUSEKEEPER_DISABLED_TASK_TYPES:}" + # Delay in milliseconds between tasks reprocessing + task-reprocessing-delay-ms: "${TB_HOUSEKEEPER_TASK_REPROCESSING_DELAY_MS:3000}" + # Maximum amount of task reprocessing attempts. After exceeding, the task will be dropped + max-reprocessing-attempts: "${TB_HOUSEKEEPER_MAX_REPROCESSING_ATTEMPTS:10}" + stats: + # Enable/disable statistics for Housekeeper + enabled: "${TB_HOUSEKEEPER_STATS_ENABLED:true}" + # Statistics printing interval for Housekeeper + print-interval-ms: "${TB_HOUSEKEEPER_STATS_PRINT_INTERVAL_MS:60000}" + vc: # Default topic name for Kafka, RabbitMQ, etc. topic: "${TB_QUEUE_VC_TOPIC:tb_version_control}" diff --git a/application/src/test/java/org/thingsboard/server/controller/AbstractWebTest.java b/application/src/test/java/org/thingsboard/server/controller/AbstractWebTest.java index e14b041e62..96e52d6c12 100644 --- a/application/src/test/java/org/thingsboard/server/controller/AbstractWebTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/AbstractWebTest.java @@ -25,7 +25,6 @@ import io.jsonwebtoken.Header; import io.jsonwebtoken.Jwt; import io.jsonwebtoken.Jwts; import lombok.extern.slf4j.Slf4j; -import org.apache.http.HttpStatus; import org.awaitility.Awaitility; import org.hamcrest.Matcher; import org.hibernate.exception.ConstraintViolationException; @@ -114,9 +113,11 @@ import org.thingsboard.server.common.data.tenant.profile.TenantProfileData; import org.thingsboard.server.common.msg.session.FeatureType; import org.thingsboard.server.config.ThingsboardSecurityConfiguration; import org.thingsboard.server.dao.Dao; +import org.thingsboard.server.dao.attributes.AttributesService; import org.thingsboard.server.dao.device.ClaimDevicesService; import org.thingsboard.server.dao.tenant.TenantProfileService; import org.thingsboard.server.dao.timeseries.TimeseriesService; +import org.thingsboard.server.queue.memory.InMemoryStorage; import org.thingsboard.server.service.entitiy.tenant.profile.TbTenantProfileService; import org.thingsboard.server.service.security.auth.jwt.RefreshTokenRequest; import org.thingsboard.server.service.security.auth.rest.LoginRequest; @@ -229,7 +230,10 @@ public abstract class AbstractWebTest extends AbstractInMemoryStorageTest { private TbTenantProfileService tbTenantProfileService; @Autowired - public TimeseriesService tsService; + protected TimeseriesService tsService; + + @Autowired + protected AttributesService attributesService; @Autowired protected DefaultActorService actorService; @@ -240,6 +244,9 @@ public abstract class AbstractWebTest extends AbstractInMemoryStorageTest { @SpyBean protected MailService mailService; + @Autowired + protected InMemoryStorage storage; + @Rule public TestRule watcher = new TestWatcher() { protected void starting(Description description) { @@ -282,7 +289,7 @@ public abstract class AbstractWebTest extends AbstractInMemoryStorageTest { Tenant tenant = new Tenant(); tenant.setTitle(TEST_TENANT_NAME); - Tenant savedTenant = doPost("/api/tenant", tenant, Tenant.class); + Tenant savedTenant = saveTenant(tenant); Assert.assertNotNull(savedTenant); tenantId = savedTenant.getId(); tenantProfileId = savedTenant.getTenantProfileId(); @@ -360,22 +367,14 @@ public abstract class AbstractWebTest extends AbstractInMemoryStorageTest { assertThat(loadedTenants).as("All tenants expected to be deleted, but some tenants left in the database").isEmpty(); } - private void deleteTenant(TenantId tenantId) { - int status = 0; - int retries = 0; - while (status != HttpStatus.SC_OK && retries < CLEANUP_TENANT_RETRIES_COUNT) { - retries++; - try { - status = doDelete("/api/tenant/" + tenantId.getId().toString()) - .andReturn().getResponse().getStatus(); - if (status != HttpStatus.SC_OK) { - log.warn("Tenant deletion failed, tenantId: {}", tenantId.getId().toString()); - Thread.sleep(1000L); - } - } catch (Exception e) { - throw new RuntimeException(e); - } + protected void deleteTenant(TenantId tenantId) { + try { + doDelete("/api/tenant/" + tenantId.getId()).andExpect(status().isOk()); + } catch (Exception e) { + throw new RuntimeException(e); } + Awaitility.await("all tasks processed").atMost(60, TimeUnit.SECONDS).during(300, TimeUnit.MILLISECONDS) + .until(() -> storage.getLag("tb_housekeeper") == 0); } private List getAllTenants() throws Exception { @@ -428,7 +427,7 @@ public abstract class AbstractWebTest extends AbstractInMemoryStorageTest { loginSysAdmin(); Tenant tenant = new Tenant(); tenant.setTitle(TEST_DIFFERENT_TENANT_NAME); - savedDifferentTenant = doPost("/api/tenant", tenant, Tenant.class); + savedDifferentTenant = saveTenant(tenant); differentTenantId = savedDifferentTenant.getId(); Assert.assertNotNull(savedDifferentTenant); User differentTenantAdmin = new User(); @@ -438,6 +437,10 @@ public abstract class AbstractWebTest extends AbstractInMemoryStorageTest { savedDifferentTenantUser = createUserAndLogin(differentTenantAdmin, DIFFERENT_TENANT_ADMIN_PASSWORD); } + protected Tenant saveTenant(Tenant tenant) throws Exception { + return doPost("/api/tenant", tenant, Tenant.class); + } + protected void loginDifferentCustomer() throws Exception { if (savedDifferentCustomer != null) { login(savedDifferentCustomer.getEmail(), CUSTOMER_USER_PASSWORD); diff --git a/application/src/test/java/org/thingsboard/server/controller/AlarmControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/AlarmControllerTest.java index 5290b429c0..7ae283cd46 100644 --- a/application/src/test/java/org/thingsboard/server/controller/AlarmControllerTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/AlarmControllerTest.java @@ -53,9 +53,8 @@ import java.util.Collections; import java.util.Comparator; import java.util.LinkedList; import java.util.List; -import java.util.stream.Collectors; -import java.util.Objects; import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; import static org.hamcrest.Matchers.containsString; import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; @@ -584,7 +583,7 @@ public class AlarmControllerTest extends AbstractControllerTest { Assert.assertEquals(savedUser.getId(), foundAlarm.getAssigneeId()); Assert.assertTrue(foundAlarm.getAssignTs() >= beforeAssignmentTs); - beforeAssignmentTs = System.currentTimeMillis(); + long afterAssignmentTs = System.currentTimeMillis(); loginSysAdmin(); @@ -592,12 +591,12 @@ public class AlarmControllerTest extends AbstractControllerTest { loginDifferentTenant(); - foundAlarm = Awaitility.await().atMost(TIMEOUT, TimeUnit.SECONDS) - .until(() -> doGet("/api/alarm/info/" + alarmId.getId(), AlarmInfo.class), Objects::nonNull); - - Assert.assertNotNull(foundAlarm); - Assert.assertNull(foundAlarm.getAssigneeId()); - Assert.assertTrue(foundAlarm.getAssignTs() >= beforeAssignmentTs); + Awaitility.await().atMost(TIMEOUT, TimeUnit.SECONDS).untilAsserted(() -> { + AlarmInfo alarmInfo = doGet("/api/alarm/info/" + alarmId.getId(), AlarmInfo.class); + Assert.assertNotNull(alarmInfo); + Assert.assertNull(alarmInfo.getAssigneeId()); + Assert.assertTrue(alarmInfo.getAssignTs() >= afterAssignmentTs); + }); } @Test @@ -631,71 +630,16 @@ public class AlarmControllerTest extends AbstractControllerTest { Assert.assertEquals(savedUser.getId(), foundAlarm.getAssigneeId()); Assert.assertTrue(foundAlarm.getAssignTs() >= beforeAssignmentTs); - beforeAssignmentTs = System.currentTimeMillis(); + long afterAssignmentTs = System.currentTimeMillis(); doDelete("/api/user/" + savedUser.getId().getId()).andExpect(status().isOk()); - foundAlarm = Awaitility.await().atMost(TIMEOUT, TimeUnit.SECONDS) - .until(() -> doGet("/api/alarm/info/" + alarmId.getId(), AlarmInfo.class), Objects::nonNull); - - Assert.assertNotNull(foundAlarm); - Assert.assertNull(foundAlarm.getAssigneeId()); - Assert.assertTrue(foundAlarm.getAssignTs() >= beforeAssignmentTs); - } - - @Test - public void testUnassignAlarmOnCustomerRemoving() throws Exception { - createDifferentTenantCustomer(); - loginDifferentTenant(); - - User user = new User(); - user.setAuthority(Authority.CUSTOMER_USER); - user.setTenantId(tenantId); - user.setCustomerId(differentTenantCustomerId); - user.setEmail("customerForAssign@thingsboard.org"); - User savedUser = createUser(user, "password"); - - Device device = createDevice("Different customer device", "default", "differentTenantTest"); - - Device assignedDevice = doPost("/api/customer/" + differentTenantCustomerId.getId() - + "/device/" + device.getId().getId(), Device.class); - Assert.assertEquals(differentTenantCustomerId, assignedDevice.getCustomerId()); - - Alarm alarm = Alarm.builder() - .type(TEST_ALARM_TYPE) - .tenantId(savedDifferentTenant.getId()) - .customerId(differentTenantCustomerId) - .originator(device.getId()) - .severity(AlarmSeverity.MAJOR) - .build(); - alarm = doPost("/api/alarm", alarm, Alarm.class); - Assert.assertNotNull(alarm); - AlarmId alarmId = alarm.getId(); - - alarm = doGet("/api/alarm/info/" + alarmId.getId(), AlarmInfo.class); - Assert.assertNotNull(alarm); - - Mockito.reset(tbClusterService, auditLogService); - long beforeAssignmentTs = System.currentTimeMillis(); - - doPost("/api/alarm/" + alarmId.getId() + "/assign/" + savedUser.getId().getId()).andExpect(status().isOk()); - AlarmInfo foundAlarm = doGet("/api/alarm/info/" + alarmId.getId(), AlarmInfo.class); - Assert.assertNotNull(foundAlarm); - Assert.assertEquals(savedUser.getId(), foundAlarm.getAssigneeId()); - Assert.assertTrue(foundAlarm.getAssignTs() >= beforeAssignmentTs); - - beforeAssignmentTs = System.currentTimeMillis(); - - Mockito.reset(tbClusterService, auditLogService); - - doDelete("/api/customer/" + differentTenantCustomerId.getId()).andExpect(status().isOk()); - - foundAlarm = Awaitility.await().atMost(TIMEOUT, TimeUnit.SECONDS) - .until(() -> doGet("/api/alarm/info/" + alarmId.getId(), AlarmInfo.class), Objects::nonNull); - - Assert.assertNotNull(foundAlarm); - Assert.assertNull(foundAlarm.getAssigneeId()); - Assert.assertTrue(foundAlarm.getAssignTs() >= beforeAssignmentTs); + Awaitility.await().atMost(TIMEOUT, TimeUnit.SECONDS).untilAsserted(() -> { + AlarmInfo alarmInfo = doGet("/api/alarm/info/" + alarmId.getId(), AlarmInfo.class); + Assert.assertNotNull(alarmInfo); + Assert.assertNull(alarmInfo.getAssigneeId()); + Assert.assertTrue(alarmInfo.getAssignTs() >= afterAssignmentTs); + }); } @Test @@ -1003,27 +947,20 @@ public class AlarmControllerTest extends AbstractControllerTest { types.remove(0); } - foundTypes = doGetTyped("/api/alarm/types?pageSize=1024&page=0", new TypeReference>() { - }) - .getData() - .stream() - .map(EntitySubtype::getType) - .collect(Collectors.toList()); - - Assert.assertEquals(types.size(), foundTypes.size()); - Assert.assertEquals(types, foundTypes); + Awaitility.await().atMost(TIMEOUT, TimeUnit.SECONDS).untilAsserted(() -> { + List actualTypes = doGetTyped("/api/alarm/types?pageSize=1024&page=0", new TypeReference>() {}) + .getData().stream().map(EntitySubtype::getType).toList(); + Assert.assertEquals(types.size(), actualTypes.size()); + Assert.assertEquals(types, actualTypes); + }); doDelete("/api/device/" + devices.get(0).getId()).andExpect(status().isOk()); - foundTypes = doGetTyped("/api/alarm/types?pageSize=1024&page=0", new TypeReference>() { - }) - .getData() - .stream() - .map(EntitySubtype::getType) - .sorted() - .collect(Collectors.toList()); - - Assert.assertTrue(foundTypes.isEmpty()); + Awaitility.await().atMost(TIMEOUT, TimeUnit.SECONDS).untilAsserted(() -> { + List actualTypes = doGetTyped("/api/alarm/types?pageSize=1024&page=0", new TypeReference>() {}) + .getData().stream().map(EntitySubtype::getType).sorted().toList(); + Assert.assertTrue(actualTypes.isEmpty()); + }); } } diff --git a/application/src/test/java/org/thingsboard/server/controller/AssetControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/AssetControllerTest.java index 06d15ddb44..123696ce3d 100644 --- a/application/src/test/java/org/thingsboard/server/controller/AssetControllerTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/AssetControllerTest.java @@ -42,6 +42,7 @@ import org.thingsboard.server.common.data.asset.Asset; import org.thingsboard.server.common.data.asset.AssetProfile; import org.thingsboard.server.common.data.audit.ActionType; import org.thingsboard.server.common.data.edge.Edge; +import org.thingsboard.server.common.data.id.AlarmId; import org.thingsboard.server.common.data.id.AssetId; import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.page.PageData; @@ -55,8 +56,10 @@ import org.thingsboard.server.dao.service.DaoSqlTest; import java.util.ArrayList; import java.util.List; +import java.util.concurrent.TimeUnit; import static org.assertj.core.api.Assertions.assertThat; +import static org.awaitility.Awaitility.await; import static org.hamcrest.Matchers.containsString; import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; import static org.thingsboard.server.dao.model.ModelConstants.NULL_UUID; @@ -87,7 +90,7 @@ public class AssetControllerTest extends AbstractControllerTest { Tenant tenant = new Tenant(); tenant.setTitle("My tenant"); - savedTenant = doPost("/api/tenant", tenant, Tenant.class); + savedTenant = saveTenant(tenant); Assert.assertNotNull(savedTenant); tenantAdmin = new User(); @@ -104,8 +107,7 @@ public class AssetControllerTest extends AbstractControllerTest { public void afterTest() throws Exception { loginSysAdmin(); - doDelete("/api/tenant/" + savedTenant.getId().getId().toString()) - .andExpect(status().isOk()); + deleteTenant(savedTenant.getId()); } @Test @@ -312,8 +314,9 @@ public class AssetControllerTest extends AbstractControllerTest { alarm = doPost("/api/alarm", alarm, Alarm.class); Assert.assertNotNull(alarm); + AlarmId alarmId = alarm.getId(); - AlarmInfo foundAlarm = doGet("/api/alarm/info/" + alarm.getId(), AlarmInfo.class); + AlarmInfo foundAlarm = doGet("/api/alarm/info/" + alarmId, AlarmInfo.class); Assert.assertNotNull(foundAlarm); doDelete("/api/asset/" + savedAsset.getId().getId().toString()) @@ -324,9 +327,11 @@ public class AssetControllerTest extends AbstractControllerTest { .andExpect(status().isNotFound()) .andExpect(statusReason(containsString(msgErrorNoFound("Asset", assetIdStr)))); - doGet("/api/alarm/info/" + alarm.getId()) - .andExpect(status().isNotFound()) - .andExpect(statusReason(containsString(msgErrorNoFound("Alarm", alarm.getId().getId().toString())))); + await().atMost(30, TimeUnit.SECONDS).untilAsserted(() -> { + doGet("/api/alarm/info/" + alarmId) + .andExpect(status().isNotFound()) + .andExpect(statusReason(containsString(msgErrorNoFound("Alarm", alarmId.getId().toString())))); + }); } @Test @@ -548,7 +553,7 @@ public class AssetControllerTest extends AbstractControllerTest { Tenant tenant2 = new Tenant(); tenant2.setTitle("Different tenant"); - Tenant savedTenant2 = doPost("/api/tenant", tenant2, Tenant.class); + Tenant savedTenant2 = saveTenant(tenant2); Assert.assertNotNull(savedTenant2); User tenantAdmin2 = new User(); @@ -582,8 +587,7 @@ public class AssetControllerTest extends AbstractControllerTest { loginSysAdmin(); - doDelete("/api/tenant/" + savedTenant2.getId().getId().toString()) - .andExpect(status().isOk()); + deleteTenant(savedTenant2.getId()); } @Test diff --git a/application/src/test/java/org/thingsboard/server/controller/AssetProfileControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/AssetProfileControllerTest.java index 432c309a4e..c408daaab3 100644 --- a/application/src/test/java/org/thingsboard/server/controller/AssetProfileControllerTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/AssetProfileControllerTest.java @@ -82,7 +82,7 @@ public class AssetProfileControllerTest extends AbstractControllerTest { Tenant tenant = new Tenant(); tenant.setTitle("My tenant"); - savedTenant = doPost("/api/tenant", tenant, Tenant.class); + savedTenant = saveTenant(tenant); Assert.assertNotNull(savedTenant); tenantAdmin = new User(); @@ -99,8 +99,7 @@ public class AssetProfileControllerTest extends AbstractControllerTest { public void afterTest() throws Exception { loginSysAdmin(); - doDelete("/api/tenant/" + savedTenant.getId().getId().toString()) - .andExpect(status().isOk()); + deleteTenant(savedTenant.getId()); } @Test diff --git a/application/src/test/java/org/thingsboard/server/controller/AuditLogControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/AuditLogControllerTest.java index bd2fe2e3f4..3150edeb37 100644 --- a/application/src/test/java/org/thingsboard/server/controller/AuditLogControllerTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/AuditLogControllerTest.java @@ -49,7 +49,6 @@ import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.reset; import static org.mockito.Mockito.verify; -import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; @DaoSqlTest public class AuditLogControllerTest extends AbstractControllerTest { @@ -75,7 +74,7 @@ public class AuditLogControllerTest extends AbstractControllerTest { Tenant tenant = new Tenant(); tenant.setTitle("My tenant"); - savedTenant = doPost("/api/tenant", tenant, Tenant.class); + savedTenant = saveTenant(tenant); Assert.assertNotNull(savedTenant); tenantAdmin = new User(); @@ -92,8 +91,7 @@ public class AuditLogControllerTest extends AbstractControllerTest { public void afterTest() throws Exception { loginSysAdmin(); - doDelete("/api/tenant/" + savedTenant.getId().getId().toString()) - .andExpect(status().isOk()); + deleteTenant(savedTenant.getId()); } @Test diff --git a/application/src/test/java/org/thingsboard/server/controller/ComponentDescriptorControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/ComponentDescriptorControllerTest.java index dd9eea7ea3..bfaecc803e 100644 --- a/application/src/test/java/org/thingsboard/server/controller/ComponentDescriptorControllerTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/ComponentDescriptorControllerTest.java @@ -47,7 +47,7 @@ public class ComponentDescriptorControllerTest extends AbstractControllerTest { Tenant tenant = new Tenant(); tenant.setTitle("My tenant"); - savedTenant = doPost("/api/tenant", tenant, Tenant.class); + savedTenant = saveTenant(tenant); Assert.assertNotNull(savedTenant); tenantAdmin = new User(); @@ -64,8 +64,7 @@ public class ComponentDescriptorControllerTest extends AbstractControllerTest { public void afterTest() throws Exception { loginSysAdmin(); - doDelete("/api/tenant/" + savedTenant.getId().getId().toString()) - .andExpect(status().isOk()); + deleteTenant(savedTenant.getId()); } @Test diff --git a/application/src/test/java/org/thingsboard/server/controller/CustomerControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/CustomerControllerTest.java index 370ba41422..f431f9ac57 100644 --- a/application/src/test/java/org/thingsboard/server/controller/CustomerControllerTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/CustomerControllerTest.java @@ -85,7 +85,7 @@ public class CustomerControllerTest extends AbstractControllerTest { Tenant tenant = new Tenant(); tenant.setTitle("My tenant"); - savedTenant = doPost("/api/tenant", tenant, Tenant.class); + savedTenant = saveTenant(tenant); Assert.assertNotNull(savedTenant); tenantAdmin = new User(); @@ -104,8 +104,7 @@ public class CustomerControllerTest extends AbstractControllerTest { loginSysAdmin(); - doDelete("/api/tenant/" + savedTenant.getId().getId().toString()) - .andExpect(status().isOk()); + deleteTenant(savedTenant.getId()); } @Test @@ -154,7 +153,7 @@ public class CustomerControllerTest extends AbstractControllerTest { .andExpect(statusReason(containsString(msgError))); customer.setTenantId(savedTenant.getId()); - testNotifyEntityEqualsOneTimeServiceNeverError(customer,savedTenant.getId(), + testNotifyEntityEqualsOneTimeServiceNeverError(customer, savedTenant.getId(), tenantAdmin.getId(), tenantAdmin.getEmail(), ActionType.ADDED, new DataValidationException(msgError)); Mockito.reset(tbClusterService, auditLogService); @@ -165,7 +164,7 @@ public class CustomerControllerTest extends AbstractControllerTest { .andExpect(status().isBadRequest()) .andExpect(statusReason(containsString(msgError))); - testNotifyEntityEqualsOneTimeServiceNeverError(customer,savedTenant.getId(), + testNotifyEntityEqualsOneTimeServiceNeverError(customer, savedTenant.getId(), tenantAdmin.getId(), tenantAdmin.getEmail(), ActionType.ADDED, new DataValidationException(msgError)); Mockito.reset(tbClusterService, auditLogService); @@ -176,7 +175,7 @@ public class CustomerControllerTest extends AbstractControllerTest { .andExpect(status().isBadRequest()) .andExpect(statusReason(containsString(msgError))); - testNotifyEntityEqualsOneTimeServiceNeverError(customer,savedTenant.getId(), + testNotifyEntityEqualsOneTimeServiceNeverError(customer, savedTenant.getId(), tenantAdmin.getId(), tenantAdmin.getEmail(), ActionType.ADDED, new DataValidationException(msgError)); Mockito.reset(tbClusterService, auditLogService); @@ -187,7 +186,7 @@ public class CustomerControllerTest extends AbstractControllerTest { .andExpect(status().isBadRequest()) .andExpect(statusReason(containsString(msgError))); - testNotifyEntityEqualsOneTimeServiceNeverError(customer,savedTenant.getId(), + testNotifyEntityEqualsOneTimeServiceNeverError(customer, savedTenant.getId(), tenantAdmin.getId(), tenantAdmin.getEmail(), ActionType.ADDED, new DataValidationException(msgError)); Mockito.reset(tbClusterService, auditLogService); @@ -198,7 +197,7 @@ public class CustomerControllerTest extends AbstractControllerTest { .andExpect(status().isBadRequest()) .andExpect(statusReason(containsString(msgError))); - testNotifyEntityEqualsOneTimeServiceNeverError(customer,savedTenant.getId(), + testNotifyEntityEqualsOneTimeServiceNeverError(customer, savedTenant.getId(), tenantAdmin.getId(), tenantAdmin.getEmail(), ActionType.ADDED, new DataValidationException(msgError)); Mockito.reset(tbClusterService, auditLogService); @@ -209,7 +208,7 @@ public class CustomerControllerTest extends AbstractControllerTest { .andExpect(status().isBadRequest()) .andExpect(statusReason(containsString(msgError))); - testNotifyEntityEqualsOneTimeServiceNeverError(customer,savedTenant.getId(), + testNotifyEntityEqualsOneTimeServiceNeverError(customer, savedTenant.getId(), tenantAdmin.getId(), tenantAdmin.getEmail(), ActionType.ADDED, new DataValidationException(msgError)); } @@ -293,7 +292,7 @@ public class CustomerControllerTest extends AbstractControllerTest { .andExpect(status().isBadRequest()) .andExpect(statusReason(containsString(msgError))); - testNotifyEntityEqualsOneTimeServiceNeverError(customer,savedTenant.getId(), + testNotifyEntityEqualsOneTimeServiceNeverError(customer, savedTenant.getId(), tenantAdmin.getId(), tenantAdmin.getEmail(), ActionType.ADDED, new DataValidationException(msgError)); } diff --git a/application/src/test/java/org/thingsboard/server/controller/DashboardControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/DashboardControllerTest.java index 71c758c1e1..7e1b9b30c9 100644 --- a/application/src/test/java/org/thingsboard/server/controller/DashboardControllerTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/DashboardControllerTest.java @@ -77,7 +77,7 @@ public class DashboardControllerTest extends AbstractControllerTest { Tenant tenant = new Tenant(); tenant.setTitle("My tenant"); - savedTenant = doPost("/api/tenant", tenant, Tenant.class); + savedTenant = saveTenant(tenant); Assert.assertNotNull(savedTenant); tenantAdmin = new User(); @@ -94,8 +94,7 @@ public class DashboardControllerTest extends AbstractControllerTest { public void afterTest() throws Exception { loginSysAdmin(); - doDelete("/api/tenant/" + savedTenant.getId().getId().toString()) - .andExpect(status().isOk()); + deleteTenant(savedTenant.getId()); } @Test @@ -166,7 +165,7 @@ public class DashboardControllerTest extends AbstractControllerTest { @Test public void testSaveDashboardWithEmptyTitle() throws Exception { Dashboard dashboard = new Dashboard(); - String msgError = "Dashboard title " + msgErrorShouldBeSpecified;; + String msgError = "Dashboard title " + msgErrorShouldBeSpecified; Mockito.reset(tbClusterService, auditLogService); @@ -196,7 +195,7 @@ public class DashboardControllerTest extends AbstractControllerTest { Assert.assertTrue(assignedDashboard.getAssignedCustomers().contains(savedCustomer.toShortCustomerInfo())); testNotifyEntityAllOneTimeLogEntityActionEntityEqClass(assignedDashboard, assignedDashboard.getId(), assignedDashboard.getId(), - savedTenant.getId(), savedCustomer.getId(), tenantAdmin.getId(), tenantAdmin.getEmail(), ActionType.ASSIGNED_TO_CUSTOMER, + savedTenant.getId(), savedCustomer.getId(), tenantAdmin.getId(), tenantAdmin.getEmail(), ActionType.ASSIGNED_TO_CUSTOMER, ActionType.UPDATED, assignedDashboard.getId().getId().toString(), savedCustomer.getId().getId().toString(), savedCustomer.getTitle()); Dashboard foundDashboard = doGet("/api/dashboard/" + savedDashboard.getId().getId().toString(), Dashboard.class); @@ -239,8 +238,8 @@ public class DashboardControllerTest extends AbstractControllerTest { Assert.assertTrue(publicCustomer.isPublic()); testNotifyEntityAllOneTimeLogEntityActionEntityEqClass(assignedDashboard, assignedDashboard.getId(), assignedDashboard.getId(), - savedTenant.getId(), publicCustomer.getId(), tenantAdmin.getId(), tenantAdmin.getEmail(), ActionType.ASSIGNED_TO_CUSTOMER, - ActionType.UPDATED, assignedDashboard .getId().getId().toString(), publicCustomer.getId().getId().toString(), publicCustomer.getTitle()); + savedTenant.getId(), publicCustomer.getId(), tenantAdmin.getId(), tenantAdmin.getEmail(), ActionType.ASSIGNED_TO_CUSTOMER, + ActionType.UPDATED, assignedDashboard.getId().getId().toString(), publicCustomer.getId().getId().toString(), publicCustomer.getTitle()); Dashboard foundDashboard = doGet("/api/dashboard/" + savedDashboard.getId().getId().toString(), Dashboard.class); Assert.assertTrue(foundDashboard.getAssignedCustomers().contains(publicCustomer.toShortCustomerInfo())); @@ -283,7 +282,7 @@ public class DashboardControllerTest extends AbstractControllerTest { Tenant tenant2 = new Tenant(); tenant2.setTitle("Different tenant"); - Tenant savedTenant2 = doPost("/api/tenant", tenant2, Tenant.class); + Tenant savedTenant2 = saveTenant(tenant2); Assert.assertNotNull(savedTenant2); User tenantAdmin2 = new User(); @@ -321,8 +320,7 @@ public class DashboardControllerTest extends AbstractControllerTest { loginSysAdmin(); - doDelete("/api/tenant/" + savedTenant2.getId().getId().toString()) - .andExpect(status().isOk()); + deleteTenant(savedTenant2.getId()); } @Test @@ -482,7 +480,7 @@ public class DashboardControllerTest extends AbstractControllerTest { testNotifyManyEntityManyTimeMsgToEdgeServiceEntityEqAny(new Dashboard(), new Dashboard(), savedTenant.getId(), tenantAdmin.getCustomerId(), tenantAdmin.getId(), tenantAdmin.getEmail(), - ActionType.ADDED, cntEntity, cntEntity, cntEntity*2); + ActionType.ADDED, cntEntity, cntEntity, cntEntity * 2); List loadedDashboards = new ArrayList<>(); PageLink pageLink = new PageLink(21); diff --git a/application/src/test/java/org/thingsboard/server/controller/DeviceConnectivityControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/DeviceConnectivityControllerTest.java index 8a67fcf396..38e537340f 100644 --- a/application/src/test/java/org/thingsboard/server/controller/DeviceConnectivityControllerTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/DeviceConnectivityControllerTest.java @@ -142,7 +142,7 @@ public class DeviceConnectivityControllerTest extends AbstractControllerTest { Tenant tenant = new Tenant(); tenant.setTitle("My tenant"); - savedTenant = doPost("/api/tenant", tenant, Tenant.class); + savedTenant = saveTenant(tenant); Assert.assertNotNull(savedTenant); tenantAdmin = new User(); @@ -187,8 +187,7 @@ public class DeviceConnectivityControllerTest extends AbstractControllerTest { public void afterTest() throws Exception { loginSysAdmin(); - doDelete("/api/tenant/" + savedTenant.getId().getId()) - .andExpect(status().isOk()); + deleteTenant(savedTenant.getId()); } @Test diff --git a/application/src/test/java/org/thingsboard/server/controller/DeviceControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/DeviceControllerTest.java index e59595e9c7..7e62d9d326 100644 --- a/application/src/test/java/org/thingsboard/server/controller/DeviceControllerTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/DeviceControllerTest.java @@ -132,7 +132,7 @@ public class DeviceControllerTest extends AbstractControllerTest { Tenant tenant = new Tenant(); tenant.setTitle("My tenant"); - savedTenant = doPost("/api/tenant", tenant, Tenant.class); + savedTenant = saveTenant(tenant); Assert.assertNotNull(savedTenant); tenantAdmin = new User(); @@ -151,8 +151,7 @@ public class DeviceControllerTest extends AbstractControllerTest { loginSysAdmin(); - doDelete("/api/tenant/" + savedTenant.getId().getId()) - .andExpect(status().isOk()); + deleteTenant(savedTenant.getId()); } @Test @@ -687,7 +686,7 @@ public class DeviceControllerTest extends AbstractControllerTest { Tenant tenant2 = new Tenant(); tenant2.setTitle("Different tenant"); - Tenant savedTenant2 = doPost("/api/tenant", tenant2, Tenant.class); + Tenant savedTenant2 = saveTenant(tenant2); Assert.assertNotNull(savedTenant2); User tenantAdmin2 = new User(); @@ -721,9 +720,7 @@ public class DeviceControllerTest extends AbstractControllerTest { testNotificationUpdateGatewayNever(); loginSysAdmin(); - - doDelete("/api/tenant/" + savedTenant2.getId().getId()) - .andExpect(status().isOk()); + deleteTenant(savedTenant2.getId()); } @Test @@ -914,7 +911,7 @@ public class DeviceControllerTest extends AbstractControllerTest { testNotifyManyEntityManyTimeMsgToEdgeServiceEntityEqAnyAdditionalInfoAny(new Device(), new Device(), savedTenant.getId(), tenantAdmin.getCustomerId(), tenantAdmin.getId(), tenantAdmin.getEmail(), - ActionType.DELETED, ActionType.DELETED, cntEntity, cntEntity,1); + ActionType.DELETED, ActionType.DELETED, cntEntity, cntEntity, 1); testNotificationUpdateGatewayNever(); } @@ -1319,7 +1316,7 @@ public class DeviceControllerTest extends AbstractControllerTest { loginSysAdmin(); Tenant tenant = new Tenant(); tenant.setTitle("Different tenant"); - Tenant savedDifferentTenant = doPost("/api/tenant", tenant, Tenant.class); + Tenant savedDifferentTenant = saveTenant(tenant); Assert.assertNotNull(savedDifferentTenant); User user = new User(); @@ -1380,8 +1377,7 @@ public class DeviceControllerTest extends AbstractControllerTest { .andExpect(statusReason(containsString(msgErrorNoFound("Device", savedAnotherDevice.getId().getId().toString())))); loginSysAdmin(); - doDelete("/api/tenant/" + savedDifferentTenant.getId().getId()) - .andExpect(status().isOk()); + deleteTenant(savedDifferentTenant.getId()); } @Test diff --git a/application/src/test/java/org/thingsboard/server/controller/DeviceProfileControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/DeviceProfileControllerTest.java index 4a93e9c0ac..b549d2af2a 100644 --- a/application/src/test/java/org/thingsboard/server/controller/DeviceProfileControllerTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/DeviceProfileControllerTest.java @@ -93,7 +93,7 @@ public class DeviceProfileControllerTest extends AbstractControllerTest { Tenant tenant = new Tenant(); tenant.setTitle("My tenant"); - savedTenant = doPost("/api/tenant", tenant, Tenant.class); + savedTenant = saveTenant(tenant); Assert.assertNotNull(savedTenant); tenantAdmin = new User(); @@ -110,8 +110,7 @@ public class DeviceProfileControllerTest extends AbstractControllerTest { public void afterTest() throws Exception { loginSysAdmin(); - doDelete("/api/tenant/" + savedTenant.getId().getId().toString()) - .andExpect(status().isOk()); + deleteTenant(savedTenant.getId()); } @Test diff --git a/application/src/test/java/org/thingsboard/server/controller/EdgeControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/EdgeControllerTest.java index 73a3a2e25e..49dfa71223 100644 --- a/application/src/test/java/org/thingsboard/server/controller/EdgeControllerTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/EdgeControllerTest.java @@ -361,7 +361,7 @@ public class EdgeControllerTest extends AbstractControllerTest { Tenant tenant2 = new Tenant(); tenant2.setTitle("Different tenant"); - Tenant savedTenant2 = doPost("/api/tenant", tenant2, Tenant.class); + Tenant savedTenant2 = saveTenant(tenant2); Assert.assertNotNull(savedTenant2); User tenantAdmin2 = new User(); @@ -394,8 +394,7 @@ public class EdgeControllerTest extends AbstractControllerTest { loginSysAdmin(); - doDelete("/api/tenant/" + savedTenant2.getId().getId().toString()) - .andExpect(status().isOk()); + deleteTenant(savedTenant2.getId()); } @Test diff --git a/application/src/test/java/org/thingsboard/server/controller/EntityQueryControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/EntityQueryControllerTest.java index 8de8514e5b..e67707b0c8 100644 --- a/application/src/test/java/org/thingsboard/server/controller/EntityQueryControllerTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/EntityQueryControllerTest.java @@ -83,7 +83,7 @@ public class EntityQueryControllerTest extends AbstractControllerTest { Tenant tenant = new Tenant(); tenant.setTitle("My tenant"); - savedTenant = doPost("/api/tenant", tenant, Tenant.class); + savedTenant = saveTenant(tenant); Assert.assertNotNull(savedTenant); tenantAdmin = new User(); @@ -99,9 +99,7 @@ public class EntityQueryControllerTest extends AbstractControllerTest { @After public void afterTest() throws Exception { loginSysAdmin(); - - doDelete("/api/tenant/" + savedTenant.getId().getId().toString()) - .andExpect(status().isOk()); + deleteTenant(savedTenant.getId()); } @Test diff --git a/application/src/test/java/org/thingsboard/server/controller/EntityRelationControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/EntityRelationControllerTest.java index 6c613ada60..48e0fb6196 100644 --- a/application/src/test/java/org/thingsboard/server/controller/EntityRelationControllerTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/EntityRelationControllerTest.java @@ -72,7 +72,7 @@ public class EntityRelationControllerTest extends AbstractControllerTest { Tenant tenant = new Tenant(); tenant.setTitle("Test tenant"); - savedTenant = doPost("/api/tenant", tenant, Tenant.class); + savedTenant = saveTenant(tenant); Assert.assertNotNull(savedTenant); tenantAdmin = new User(); @@ -93,8 +93,7 @@ public class EntityRelationControllerTest extends AbstractControllerTest { public void afterTest() throws Exception { loginSysAdmin(); - doDelete("/api/tenant/" + savedTenant.getId().getId().toString()) - .andExpect(status().isOk()); + deleteTenant(savedTenant.getId()); } @Test diff --git a/application/src/test/java/org/thingsboard/server/controller/EntityViewControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/EntityViewControllerTest.java index 3287af19b6..e525f0efd5 100644 --- a/application/src/test/java/org/thingsboard/server/controller/EntityViewControllerTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/EntityViewControllerTest.java @@ -370,7 +370,7 @@ public class EntityViewControllerTest extends AbstractControllerTest { loginSysAdmin(); Tenant tenant2 = getNewTenant("Different tenant"); - Tenant savedTenant2 = doPost("/api/tenant", tenant2, Tenant.class); + Tenant savedTenant2 = saveTenant(tenant2); Assert.assertNotNull(savedTenant2); User tenantAdmin2 = new User(); @@ -398,8 +398,7 @@ public class EntityViewControllerTest extends AbstractControllerTest { loginSysAdmin(); - doDelete("/api/tenant/" + savedTenant2.getId().getId().toString()) - .andExpect(status().isOk()); + deleteTenant(savedTenant2.getId()); } @Test @@ -431,7 +430,7 @@ public class EntityViewControllerTest extends AbstractControllerTest { testNotifyEntityBroadcastEntityStateChangeEventMany(new EntityView(), new EntityView(), tenantId, customerId, tenantAdminUserId, TENANT_ADMIN_EMAIL, ActionType.ASSIGNED_TO_CUSTOMER, ActionType.UPDATED, cntEntity, cntEntity, - cntEntity*2, 3); + cntEntity * 2, 3); } @Test diff --git a/application/src/test/java/org/thingsboard/server/controller/HomePageApiTest.java b/application/src/test/java/org/thingsboard/server/controller/HomePageApiTest.java index 717c17d437..31c82408c1 100644 --- a/application/src/test/java/org/thingsboard/server/controller/HomePageApiTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/HomePageApiTest.java @@ -103,7 +103,7 @@ public class HomePageApiTest extends AbstractControllerTest { for (int i = 0; i < 100; i++) { Tenant tenant = new Tenant(); tenant.setTitle("tenant" + i); - tenants.add(doPost("/api/tenant", tenant, Tenant.class)); + tenants.add(saveTenant(tenant)); } EntityTypeFilter ef = new EntityTypeFilter(); @@ -115,7 +115,7 @@ public class HomePageApiTest extends AbstractControllerTest { Assert.assertEquals(initialCount + 100, update.getCount()); for (Tenant tenant : tenants) { - doDelete("/api/tenant/" + tenant.getId().toString()); + deleteTenant(tenant.getId()); } } diff --git a/application/src/test/java/org/thingsboard/server/controller/OtaPackageControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/OtaPackageControllerTest.java index 9b5e0a65f8..4031045095 100644 --- a/application/src/test/java/org/thingsboard/server/controller/OtaPackageControllerTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/OtaPackageControllerTest.java @@ -72,7 +72,7 @@ public class OtaPackageControllerTest extends AbstractControllerTest { Tenant tenant = new Tenant(); tenant.setTitle("My tenant"); - savedTenant = doPost("/api/tenant", tenant, Tenant.class); + savedTenant = saveTenant(tenant); Assert.assertNotNull(savedTenant); tenantAdmin = new User(); @@ -94,8 +94,7 @@ public class OtaPackageControllerTest extends AbstractControllerTest { public void afterTest() throws Exception { loginSysAdmin(); - doDelete("/api/tenant/" + savedTenant.getId().getId().toString()) - .andExpect(status().isOk()); + deleteTenant(savedTenant.getId()); } @Test @@ -339,7 +338,7 @@ public class OtaPackageControllerTest extends AbstractControllerTest { testNotifyManyEntityManyTimeMsgToEdgeServiceEntityEqAny(new OtaPackageInfo(), new OtaPackageInfo(), savedTenant.getId(), tenantAdmin.getCustomerId(), tenantAdmin.getId(), tenantAdmin.getEmail(), - ActionType.ADDED, cntEntity, 0, (cntEntity*2 - startIndexSaveData)); + ActionType.ADDED, cntEntity, 0, (cntEntity * 2 - startIndexSaveData)); List loadedFirmwares = new ArrayList<>(); PageLink pageLink = new PageLink(24); diff --git a/application/src/test/java/org/thingsboard/server/controller/RpcControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/RpcControllerTest.java index 4d1f3d6407..d0f6426ab1 100644 --- a/application/src/test/java/org/thingsboard/server/controller/RpcControllerTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/RpcControllerTest.java @@ -49,7 +49,7 @@ public class RpcControllerTest extends AbstractControllerTest { Tenant tenant = new Tenant(); tenant.setTitle("My tenant"); - savedTenant = doPost("/api/tenant", tenant, Tenant.class); + savedTenant = saveTenant(tenant); Assert.assertNotNull(savedTenant); tenantAdmin = new User(); @@ -66,8 +66,7 @@ public class RpcControllerTest extends AbstractControllerTest { public void afterTest() throws Exception { loginSysAdmin(); - doDelete("/api/tenant/" + savedTenant.getId().getId().toString()) - .andExpect(status().isOk()); + deleteTenant(savedTenant.getId()); } private Device createDefaultDevice() { diff --git a/application/src/test/java/org/thingsboard/server/controller/RuleChainControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/RuleChainControllerTest.java index 7595a69931..a990f55224 100644 --- a/application/src/test/java/org/thingsboard/server/controller/RuleChainControllerTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/RuleChainControllerTest.java @@ -82,7 +82,7 @@ public class RuleChainControllerTest extends AbstractControllerTest { Tenant tenant = new Tenant(); tenant.setTitle("My tenant"); - savedTenant = doPost("/api/tenant", tenant, Tenant.class); + savedTenant = saveTenant(tenant); Assert.assertNotNull(savedTenant); tenantAdmin = new User(); @@ -99,8 +99,7 @@ public class RuleChainControllerTest extends AbstractControllerTest { public void afterTest() throws Exception { loginSysAdmin(); - doDelete("/api/tenant/" + savedTenant.getId().getId().toString()) - .andExpect(status().isOk()); + deleteTenant(savedTenant.getId()); } @Test @@ -147,9 +146,9 @@ public class RuleChainControllerTest extends AbstractControllerTest { int currentVersion = annotation.version(); String oldConfig = "{\"attrMapping\":{\"serialNumber\":\"sn\"}," + - "\"relationsQuery\":{\"direction\":\"FROM\",\"maxLevel\":1," + - "\"filters\":[{\"relationType\":\"Contains\",\"entityTypes\":[]}]," + - "\"fetchLastLevelOnly\":false},\"telemetry\":false}"; + "\"relationsQuery\":{\"direction\":\"FROM\",\"maxLevel\":1," + + "\"filters\":[{\"relationType\":\"Contains\",\"entityTypes\":[]}]," + + "\"fetchLastLevelOnly\":false},\"telemetry\":false}"; TbGetRelatedDataNodeConfiguration defaultConfiguration = new TbGetRelatedDataNodeConfiguration().defaultConfiguration(); String newConfig = JacksonUtil.toString(defaultConfiguration); diff --git a/application/src/test/java/org/thingsboard/server/controller/TbResourceControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/TbResourceControllerTest.java index 1d875d2ccc..0745761c25 100644 --- a/application/src/test/java/org/thingsboard/server/controller/TbResourceControllerTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/TbResourceControllerTest.java @@ -73,7 +73,7 @@ public class TbResourceControllerTest extends AbstractControllerTest { Tenant tenant = new Tenant(); tenant.setTitle("My tenant"); - savedTenant = doPost("/api/tenant", tenant, Tenant.class); + savedTenant = saveTenant(tenant); Assert.assertNotNull(savedTenant); tenantAdmin = new User(); @@ -90,8 +90,7 @@ public class TbResourceControllerTest extends AbstractControllerTest { public void afterTest() throws Exception { loginSysAdmin(); - doDelete("/api/tenant/" + savedTenant.getId().getId().toString()) - .andExpect(status().isOk()); + deleteTenant(savedTenant.getId()); } @Test diff --git a/application/src/test/java/org/thingsboard/server/controller/TenantControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/TenantControllerTest.java index 63747d2c19..3160d542f1 100644 --- a/application/src/test/java/org/thingsboard/server/controller/TenantControllerTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/TenantControllerTest.java @@ -140,7 +140,7 @@ public class TenantControllerTest extends AbstractControllerTest { Mockito.reset(tbClusterService); - Tenant savedTenant = doPost("/api/tenant", tenant, Tenant.class); + Tenant savedTenant = saveTenant(tenant); Assert.assertNotNull(savedTenant); Assert.assertNotNull(savedTenant.getId()); Assert.assertTrue(savedTenant.getCreatedTime() > 0); @@ -149,14 +149,13 @@ public class TenantControllerTest extends AbstractControllerTest { testBroadcastEntityStateChangeEventTimeManyTimeTenant(savedTenant, ComponentLifecycleEvent.CREATED, 1); savedTenant.setTitle("My new tenant"); - doPost("/api/tenant", savedTenant, Tenant.class); + saveTenant(savedTenant); Tenant foundTenant = doGet("/api/tenant/" + savedTenant.getId().getId().toString(), Tenant.class); Assert.assertEquals(foundTenant.getTitle(), savedTenant.getTitle()); testBroadcastEntityStateChangeEventTimeManyTimeTenant(savedTenant, ComponentLifecycleEvent.UPDATED, 1); - doDelete("/api/tenant/" + savedTenant.getId().getId().toString()) - .andExpect(status().isOk()); + deleteTenant(savedTenant.getId()); testBroadcastEntityStateChangeEventTimeManyTimeTenant(savedTenant, ComponentLifecycleEvent.DELETED, 1); } @@ -181,12 +180,11 @@ public class TenantControllerTest extends AbstractControllerTest { loginSysAdmin(); Tenant tenant = new Tenant(); tenant.setTitle("My tenant"); - Tenant savedTenant = doPost("/api/tenant", tenant, Tenant.class); + Tenant savedTenant = saveTenant(tenant); Tenant foundTenant = doGet("/api/tenant/" + savedTenant.getId().getId().toString(), Tenant.class); Assert.assertNotNull(foundTenant); Assert.assertEquals(savedTenant, foundTenant); - doDelete("/api/tenant/" + savedTenant.getId().getId().toString()) - .andExpect(status().isOk()); + deleteTenant(savedTenant.getId()); } @Test @@ -194,12 +192,11 @@ public class TenantControllerTest extends AbstractControllerTest { loginSysAdmin(); Tenant tenant = new Tenant(); tenant.setTitle("My tenant"); - Tenant savedTenant = doPost("/api/tenant", tenant, Tenant.class); + Tenant savedTenant = saveTenant(tenant); TenantInfo foundTenant = doGet("/api/tenant/info/" + savedTenant.getId().getId().toString(), TenantInfo.class); Assert.assertNotNull(foundTenant); Assert.assertEquals(new TenantInfo(savedTenant, "Default"), foundTenant); - doDelete("/api/tenant/" + savedTenant.getId().getId().toString()) - .andExpect(status().isOk()); + deleteTenant(savedTenant.getId()); } @Test @@ -237,11 +234,10 @@ public class TenantControllerTest extends AbstractControllerTest { loginSysAdmin(); Tenant tenant = new Tenant(); tenant.setTitle("My tenant"); - Tenant savedTenant = doPost("/api/tenant", tenant, Tenant.class); + Tenant savedTenant = saveTenant(tenant); String tenantIdStr = savedTenant.getId().getId().toString(); - doDelete("/api/tenant/" + tenantIdStr) - .andExpect(status().isOk()); + deleteTenant(savedTenant.getId()); doGet("/api/tenant/" + tenantIdStr) .andExpect(status().isNotFound()) .andExpect(statusReason(containsString(msgErrorNoFound("Tenant", tenantIdStr)))); @@ -264,8 +260,7 @@ public class TenantControllerTest extends AbstractControllerTest { for (int i = 0; i < cntEntity; i++) { Tenant tenant = new Tenant(); tenant.setTitle("Tenant" + i); - createFutures.add(executor.submit(() -> - doPost("/api/tenant", tenant, Tenant.class))); + createFutures.add(executor.submit(() -> saveTenant(tenant))); } tenants.addAll(Futures.allAsList(createFutures).get(TIMEOUT, TimeUnit.SECONDS)); @@ -308,8 +303,7 @@ public class TenantControllerTest extends AbstractControllerTest { String title = title1 + suffix; title = i % 2 == 0 ? title.toLowerCase() : title.toUpperCase(); tenant.setTitle(title); - createFutures.add(executor.submit(() -> - doPost("/api/tenant", tenant, Tenant.class))); + createFutures.add(executor.submit(() -> saveTenant(tenant))); } List tenantsTitle1 = Futures.allAsList(createFutures).get(TIMEOUT, TimeUnit.SECONDS); @@ -323,8 +317,7 @@ public class TenantControllerTest extends AbstractControllerTest { String title = title2 + suffix; title = i % 2 == 0 ? title.toLowerCase() : title.toUpperCase(); tenant.setTitle(title); - createFutures.add(executor.submit(() -> - doPost("/api/tenant", tenant, Tenant.class))); + createFutures.add(executor.submit(() -> saveTenant(tenant))); } List tenantsTitle2 = Futures.allAsList(createFutures).get(TIMEOUT, TimeUnit.SECONDS); @@ -396,7 +389,7 @@ public class TenantControllerTest extends AbstractControllerTest { Tenant tenant = new Tenant(); tenant.setTitle("Tenant" + i); createFutures.add(executor.submit(() -> - new TenantInfo(doPost("/api/tenant", tenant, Tenant.class), "Default"))); + new TenantInfo(saveTenant(tenant), "Default"))); } tenants.addAll(Futures.allAsList(createFutures).get(TIMEOUT, TimeUnit.SECONDS)); @@ -448,7 +441,7 @@ public class TenantControllerTest extends AbstractControllerTest { Tenant tenant = new Tenant(); tenant.setTitle("Isolated tenant"); - tenant = doPost("/api/tenant", tenant, Tenant.class); + tenant = saveTenant(tenant); User tenantUser = new User(); tenantUser.setAuthority(Authority.TENANT_ADMIN); @@ -477,7 +470,7 @@ public class TenantControllerTest extends AbstractControllerTest { tenantProfile = doPost("/api/tenantProfile", tenantProfile, TenantProfile.class); tenant.setTenantProfileId(tenantProfile.getId()); - doPost("/api/tenant", tenant, Tenant.class); + saveTenant(tenant); login(username, password); @@ -507,7 +500,7 @@ public class TenantControllerTest extends AbstractControllerTest { tenantProfile2 = doPost("/api/tenantProfile", tenantProfile2, TenantProfile.class); tenant.setTenantProfileId(tenantProfile2.getId()); - doPost("/api/tenant", tenant, Tenant.class); + saveTenant(tenant); login(username, password); @@ -549,7 +542,7 @@ public class TenantControllerTest extends AbstractControllerTest { loginSysAdmin(); tenant.setTenantProfileId(null); - doPost("/api/tenant", tenant, Tenant.class); + saveTenant(tenant); login(username, password); for (Queue queue : foundTenantQueues) { @@ -559,7 +552,7 @@ public class TenantControllerTest extends AbstractControllerTest { } loginSysAdmin(); - doDelete("/api/tenant/" + tenant.getId().getId().toString()).andExpect(status().isOk()); + deleteTenant(tenant.getId()); } @Test @@ -576,7 +569,7 @@ public class TenantControllerTest extends AbstractControllerTest { createDifferentTenant(); loginSysAdmin(); savedDifferentTenant.setTenantProfileId(tenantProfile.getId()); - savedDifferentTenant = doPost("/api/tenant", savedDifferentTenant, Tenant.class); + savedDifferentTenant = saveTenant(savedDifferentTenant); TenantId tenantId = differentTenantId; loginDifferentTenant(); @@ -661,7 +654,7 @@ public class TenantControllerTest extends AbstractControllerTest { createDifferentTenant(); loginSysAdmin(); savedDifferentTenant.setTenantProfileId(tenantProfile.getId()); - savedDifferentTenant = doPost("/api/tenant", savedDifferentTenant, Tenant.class); + savedDifferentTenant = saveTenant(savedDifferentTenant); TenantId tenantId = differentTenantId; await().atMost(30, TimeUnit.SECONDS) .until(() -> { @@ -716,7 +709,7 @@ public class TenantControllerTest extends AbstractControllerTest { createDifferentTenant(); loginSysAdmin(); savedDifferentTenant.setTenantProfileId(tenantProfile.getId()); - savedDifferentTenant = doPost("/api/tenant", savedDifferentTenant, Tenant.class); + savedDifferentTenant = saveTenant(savedDifferentTenant); TenantId tenantId = differentTenantId; await().atMost(30, TimeUnit.SECONDS).untilAsserted(() -> { assertThat(partitionService.getMyPartitions(new QueueKey(ServiceType.TB_RULE_ENGINE, tenantId))).isNotNull(); diff --git a/application/src/test/java/org/thingsboard/server/controller/TenantProfileControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/TenantProfileControllerTest.java index f25cfe2228..989e80e6e5 100644 --- a/application/src/test/java/org/thingsboard/server/controller/TenantProfileControllerTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/TenantProfileControllerTest.java @@ -75,7 +75,7 @@ public class TenantProfileControllerTest extends AbstractControllerTest { savedTenantProfile.setName("New tenant profile"); doPost("/api/tenantProfile", savedTenantProfile, TenantProfile.class); - TenantProfile foundTenantProfile = doGet("/api/tenantProfile/"+savedTenantProfile.getId().getId().toString(), TenantProfile.class); + TenantProfile foundTenantProfile = doGet("/api/tenantProfile/" + savedTenantProfile.getId().getId().toString(), TenantProfile.class); Assert.assertEquals(foundTenantProfile.getName(), savedTenantProfile.getName()); testBroadcastEntityStateChangeEventTimeManyTimeTenantProfile(savedTenantProfile, ComponentLifecycleEvent.UPDATED, 1); @@ -100,7 +100,7 @@ public class TenantProfileControllerTest extends AbstractControllerTest { loginSysAdmin(); TenantProfile tenantProfile = this.createTenantProfile("Tenant Profile"); TenantProfile savedTenantProfile = doPost("/api/tenantProfile", tenantProfile, TenantProfile.class); - TenantProfile foundTenantProfile = doGet("/api/tenantProfile/"+savedTenantProfile.getId().getId().toString(), TenantProfile.class); + TenantProfile foundTenantProfile = doGet("/api/tenantProfile/" + savedTenantProfile.getId().getId().toString(), TenantProfile.class); Assert.assertNotNull(foundTenantProfile); Assert.assertEquals(savedTenantProfile, foundTenantProfile); } @@ -110,7 +110,7 @@ public class TenantProfileControllerTest extends AbstractControllerTest { loginSysAdmin(); TenantProfile tenantProfile = this.createTenantProfile("Tenant Profile"); TenantProfile savedTenantProfile = doPost("/api/tenantProfile", tenantProfile, TenantProfile.class); - EntityInfo foundTenantProfileInfo = doGet("/api/tenantProfileInfo/"+savedTenantProfile.getId().getId().toString(), EntityInfo.class); + EntityInfo foundTenantProfileInfo = doGet("/api/tenantProfileInfo/" + savedTenantProfile.getId().getId().toString(), EntityInfo.class); Assert.assertNotNull(foundTenantProfileInfo); Assert.assertEquals(savedTenantProfile.getId(), foundTenantProfileInfo.getId()); Assert.assertEquals(savedTenantProfile.getName(), foundTenantProfileInfo.getName()); @@ -129,7 +129,7 @@ public class TenantProfileControllerTest extends AbstractControllerTest { loginSysAdmin(); TenantProfile tenantProfile = this.createTenantProfile("Tenant Profile 1"); TenantProfile savedTenantProfile = doPost("/api/tenantProfile", tenantProfile, TenantProfile.class); - TenantProfile defaultTenantProfile = doPost("/api/tenantProfile/"+savedTenantProfile.getId().getId().toString()+"/default", TenantProfile.class); + TenantProfile defaultTenantProfile = doPost("/api/tenantProfile/" + savedTenantProfile.getId().getId().toString() + "/default", TenantProfile.class); Assert.assertNotNull(defaultTenantProfile); EntityInfo foundDefaultTenantProfile = doGet("/api/tenantProfileInfo/default", EntityInfo.class); Assert.assertNotNull(foundDefaultTenantProfile); @@ -176,7 +176,7 @@ public class TenantProfileControllerTest extends AbstractControllerTest { Tenant tenant = new Tenant(); tenant.setTitle("My tenant with tenant profile"); tenant.setTenantProfileId(savedTenantProfile.getId()); - Tenant savedTenant = doPost("/api/tenant", tenant, Tenant.class); + Tenant savedTenant = saveTenant(tenant); Mockito.reset(tbClusterService); @@ -186,8 +186,7 @@ public class TenantProfileControllerTest extends AbstractControllerTest { testBroadcastEntityStateChangeEventNeverTenantProfile(); - doDelete("/api/tenant/"+savedTenant.getId().getId().toString()) - .andExpect(status().isOk()); + deleteTenant(savedTenant.getId()); } @Test @@ -214,7 +213,7 @@ public class TenantProfileControllerTest extends AbstractControllerTest { List tenantProfiles = new ArrayList<>(); PageLink pageLink = new PageLink(17); PageData pageData = doGetTypedWithPageLink("/api/tenantProfiles?", - new TypeReference<>(){}, pageLink); + new TypeReference<>() {}, pageLink); Assert.assertFalse(pageData.hasNext()); Assert.assertEquals(1, pageData.getTotalElements()); tenantProfiles.addAll(pageData.getData()); @@ -222,8 +221,8 @@ public class TenantProfileControllerTest extends AbstractControllerTest { Mockito.reset(tbClusterService); int cntEntity = 28; - for (int i=0;i<28;i++) { - TenantProfile tenantProfile = this.createTenantProfile("Tenant Profile"+i); + for (int i = 0; i < 28; i++) { + TenantProfile tenantProfile = this.createTenantProfile("Tenant Profile" + i); tenantProfiles.add(doPost("/api/tenantProfile", tenantProfile, TenantProfile.class)); } @@ -233,7 +232,7 @@ public class TenantProfileControllerTest extends AbstractControllerTest { pageLink = new PageLink(17); do { pageData = doGetTypedWithPageLink("/api/tenantProfiles?", - new TypeReference<>(){}, pageLink); + new TypeReference<>() {}, pageLink); loadedTenantProfiles.addAll(pageData.getData()); if (pageData.hasNext()) { pageLink = pageLink.nextPageLink(); @@ -256,7 +255,7 @@ public class TenantProfileControllerTest extends AbstractControllerTest { pageLink = new PageLink(17); pageData = doGetTypedWithPageLink("/api/tenantProfiles?", - new TypeReference>(){}, pageLink); + new TypeReference>() {}, pageLink); Assert.assertFalse(pageData.hasNext()); Assert.assertEquals(1, pageData.getTotalElements()); @@ -269,13 +268,13 @@ public class TenantProfileControllerTest extends AbstractControllerTest { List tenantProfiles = new ArrayList<>(); PageLink pageLink = new PageLink(17); PageData tenantProfilePageData = doGetTypedWithPageLink("/api/tenantProfiles?", - new TypeReference<>(){}, pageLink); + new TypeReference<>() {}, pageLink); Assert.assertFalse(tenantProfilePageData.hasNext()); Assert.assertEquals(1, tenantProfilePageData.getTotalElements()); tenantProfiles.addAll(tenantProfilePageData.getData()); - for (int i=0;i<28;i++) { - TenantProfile tenantProfile = this.createTenantProfile("Tenant Profile"+i); + for (int i = 0; i < 28; i++) { + TenantProfile tenantProfile = this.createTenantProfile("Tenant Profile" + i); tenantProfiles.add(doPost("/api/tenantProfile", tenantProfile, TenantProfile.class)); } @@ -284,7 +283,7 @@ public class TenantProfileControllerTest extends AbstractControllerTest { PageData pageData; do { pageData = doGetTypedWithPageLink("/api/tenantProfileInfos?", - new TypeReference>(){}, pageLink); + new TypeReference>() {}, pageLink); loadedTenantProfileInfos.addAll(pageData.getData()); if (pageData.hasNext()) { pageLink = pageLink.nextPageLink(); @@ -308,7 +307,7 @@ public class TenantProfileControllerTest extends AbstractControllerTest { pageLink = new PageLink(17); pageData = doGetTypedWithPageLink("/api/tenantProfileInfos?", - new TypeReference>(){}, pageLink); + new TypeReference>() {}, pageLink); Assert.assertFalse(pageData.hasNext()); Assert.assertEquals(1, pageData.getTotalElements()); } @@ -354,14 +353,14 @@ public class TenantProfileControllerTest extends AbstractControllerTest { ArgumentMatcher matcherTenantProfile = cntTime == 1 ? argument -> argument.equals(tenantProfile) : argument -> argument.getClass().equals(TenantProfile.class); if (ComponentLifecycleEvent.DELETED.equals(event)) { - Mockito.verify(tbClusterService, times( cntTime)).onTenantProfileDelete(Mockito.argThat( matcherTenantProfile), + Mockito.verify(tbClusterService, times(cntTime)).onTenantProfileDelete(Mockito.argThat(matcherTenantProfile), Mockito.isNull()); testBroadcastEntityStateChangeEventNever(createEntityId_NULL_UUID(new Tenant())); } else { - Mockito.verify(tbClusterService, times( cntTime)).onTenantProfileChange(Mockito.argThat(matcherTenantProfile), + Mockito.verify(tbClusterService, times(cntTime)).onTenantProfileChange(Mockito.argThat(matcherTenantProfile), Mockito.isNull()); TenantProfileId tenantProfileIdId = cntTime == 1 ? tenantProfile.getId() : (TenantProfileId) createEntityId_NULL_UUID(tenantProfile); - testBroadcastEntityStateChangeEventTime(tenantProfileIdId, null, cntTime); + testBroadcastEntityStateChangeEventTime(tenantProfileIdId, null, cntTime); } Mockito.reset(tbClusterService); } diff --git a/application/src/test/java/org/thingsboard/server/controller/UserControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/UserControllerTest.java index 72930e0327..c1f3a90529 100644 --- a/application/src/test/java/org/thingsboard/server/controller/UserControllerTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/UserControllerTest.java @@ -380,7 +380,7 @@ public class UserControllerTest extends AbstractControllerTest { //here created a new tenant despite already created on AbstractWebTest and then delete the tenant properly on the last line Tenant tenant = new Tenant(); tenant.setTitle("My tenant with many admins"); - Tenant savedTenant = doPost("/api/tenant", tenant, Tenant.class); + Tenant savedTenant = saveTenant(tenant); Assert.assertNotNull(savedTenant); TenantId tenantId = savedTenant.getId(); @@ -422,8 +422,7 @@ public class UserControllerTest extends AbstractControllerTest { assertThat(tenantAdmins).as("admins list size").hasSameSizeAs(loadedTenantAdmins); assertThat(tenantAdmins).as("admins list content").isEqualTo(loadedTenantAdmins); - doDelete("/api/tenant/" + tenantId.getId().toString()) - .andExpect(status().isOk()); + deleteTenant(tenantId); pageLink = new PageLink(33); pageData = doGetTypedWithPageLink("/api/tenant/" + tenantId.getId().toString() + "/users?", @@ -669,7 +668,7 @@ public class UserControllerTest extends AbstractControllerTest { List expectedCustomerUserIds = new ArrayList<>(); expectedCustomerUserIds.add(customerUserId); for (int i = 0; i < 45; i++) { - User customerUser = createCustomerUser( customerId); + User customerUser = createCustomerUser(customerId); customerUser.setEmail(email + StringUtils.randomAlphanumeric((int) (5 + Math.random() * 10)) + "@thingsboard.org"); User user = doPost("/api/user", customerUser, User.class); expectedCustomerUserIds.add(user.getId()); @@ -750,7 +749,7 @@ public class UserControllerTest extends AbstractControllerTest { String email = "testEmail1"; for (int i = 0; i < 45; i++) { - User customerUser = createCustomerUser( customerId); + User customerUser = createCustomerUser(customerId); customerUser.setEmail(email + StringUtils.randomAlphanumeric((int) (5 + Math.random() * 10)) + "@thingsboard.org"); doPost("/api/user", customerUser, User.class); } @@ -993,8 +992,8 @@ public class UserControllerTest extends AbstractControllerTest { List usersInfo = getUsersInfo(pageLink); List expectedUserInfos = customerUsersContainingWord.stream().map(customerUser -> new UserEmailInfo(customerUser.getId(), - customerUser.getEmail(), customerUser.getFirstName() == null ? "" : customerUser.getFirstName(), - customerUser.getLastName() == null ? "" : customerUser.getLastName())) + customerUser.getEmail(), customerUser.getFirstName() == null ? "" : customerUser.getFirstName(), + customerUser.getLastName() == null ? "" : customerUser.getLastName())) .sorted(userDataIdComparator).collect(Collectors.toList()); usersInfo.sort(userDataIdComparator); @@ -1046,8 +1045,8 @@ public class UserControllerTest extends AbstractControllerTest { List usersInfo = getUsersInfo(pageLink); List expectedUserInfos = usersContainingWord.stream().map(customerUser -> new UserEmailInfo(customerUser.getId(), - customerUser.getEmail(), customerUser.getFirstName() == null ? "" : customerUser.getFirstName(), - customerUser.getLastName() == null ? "" : customerUser.getLastName())) + customerUser.getEmail(), customerUser.getFirstName() == null ? "" : customerUser.getFirstName(), + customerUser.getLastName() == null ? "" : customerUser.getLastName())) .sorted(userDataIdComparator).collect(Collectors.toList()); usersInfo.sort(userDataIdComparator); diff --git a/application/src/test/java/org/thingsboard/server/controller/WidgetTypeControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/WidgetTypeControllerTest.java index c689b12b43..5c4e2afde5 100644 --- a/application/src/test/java/org/thingsboard/server/controller/WidgetTypeControllerTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/WidgetTypeControllerTest.java @@ -24,7 +24,6 @@ import org.junit.Test; import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.server.common.data.Tenant; import org.thingsboard.server.common.data.User; -import org.thingsboard.server.common.data.id.WidgetTypeId; import org.thingsboard.server.common.data.security.Authority; import org.thingsboard.server.common.data.widget.WidgetType; import org.thingsboard.server.common.data.widget.WidgetTypeDetails; @@ -53,7 +52,7 @@ public class WidgetTypeControllerTest extends AbstractControllerTest { Tenant tenant = new Tenant(); tenant.setTitle("My tenant"); - savedTenant = doPost("/api/tenant", tenant, Tenant.class); + savedTenant = saveTenant(tenant); Assert.assertNotNull(savedTenant); tenantAdmin = new User(); @@ -70,8 +69,7 @@ public class WidgetTypeControllerTest extends AbstractControllerTest { public void afterTest() throws Exception { loginSysAdmin(); - doDelete("/api/tenant/"+savedTenant.getId().getId().toString()) - .andExpect(status().isOk()); + deleteTenant(savedTenant.getId()); } @Test @@ -127,10 +125,10 @@ public class WidgetTypeControllerTest extends AbstractControllerTest { widgetType.setDescriptor(JacksonUtil.fromString("{ \"someKey\": \"someValue\" }", JsonNode.class)); WidgetTypeDetails savedWidgetType = doPost("/api/widgetType", widgetType, WidgetTypeDetails.class); - doDelete("/api/widgetType/"+savedWidgetType.getId().getId().toString()) + doDelete("/api/widgetType/" + savedWidgetType.getId().getId().toString()) .andExpect(status().isOk()); - doGet("/api/widgetType/"+savedWidgetType.getId().getId().toString()) + doGet("/api/widgetType/" + savedWidgetType.getId().getId().toString()) .andExpect(status().isNotFound()); } @@ -173,7 +171,7 @@ public class WidgetTypeControllerTest extends AbstractControllerTest { widgetsBundle = doPost("/api/widgetsBundle", widgetsBundle, WidgetsBundle.class); List widgetTypes = new ArrayList<>(); - for (int i=0;i<89;i++) { + for (int i = 0; i < 89; i++) { WidgetTypeDetails widgetType = new WidgetTypeDetails(); widgetType.setName("Widget Type " + i); widgetType.setDescriptor(JacksonUtil.fromString("{ \"someKey\": \"someValue\" }", JsonNode.class)); @@ -184,7 +182,7 @@ public class WidgetTypeControllerTest extends AbstractControllerTest { doPost("/api/widgetsBundle/" + widgetsBundle.getId().getId().toString() + "/widgetTypes", widgetTypeIds); List loadedWidgetTypes = doGetTyped("/api/widgetTypes?widgetsBundleId={widgetsBundleId}", - new TypeReference<>(){}, widgetsBundle.getId().getId().toString()); + new TypeReference<>() {}, widgetsBundle.getId().getId().toString()); Collections.sort(widgetTypes, idComparator); Collections.sort(loadedWidgetTypes, idComparator); @@ -194,12 +192,12 @@ public class WidgetTypeControllerTest extends AbstractControllerTest { loginCustomerUser(); List loadedWidgetTypesCustomer = doGetTyped("/api/widgetTypes?widgetsBundleId={widgetsBundleId}", - new TypeReference<>(){}, widgetsBundle.getId().getId().toString()); + new TypeReference<>() {}, widgetsBundle.getId().getId().toString()); Collections.sort(loadedWidgetTypesCustomer, idComparator); Assert.assertEquals(widgetTypes, loadedWidgetTypesCustomer); List customerLoadedWidgetTypesDetails = doGetTyped("/api/widgetTypesDetails?widgetsBundleId={widgetsBundleId}", - new TypeReference<>(){}, widgetsBundle.getId().getId().toString()); + new TypeReference<>() {}, widgetsBundle.getId().getId().toString()); List widgetTypesFromDetailsListCustomer = customerLoadedWidgetTypesDetails.stream().map(WidgetType::new).collect(Collectors.toList()); Collections.sort(widgetTypesFromDetailsListCustomer, idComparator); Assert.assertEquals(widgetTypesFromDetailsListCustomer, loadedWidgetTypes); @@ -207,12 +205,12 @@ public class WidgetTypeControllerTest extends AbstractControllerTest { loginSysAdmin(); List sysAdminLoadedWidgetTypes = doGetTyped("/api/widgetTypes?widgetsBundleId={widgetsBundleId}", - new TypeReference<>(){}, widgetsBundle.getId().getId().toString()); + new TypeReference<>() {}, widgetsBundle.getId().getId().toString()); Collections.sort(sysAdminLoadedWidgetTypes, idComparator); Assert.assertEquals(widgetTypes, sysAdminLoadedWidgetTypes); List sysAdminLoadedWidgetTypesDetails = doGetTyped("/api/widgetTypesDetails?widgetsBundleId={widgetsBundleId}", - new TypeReference<>(){}, widgetsBundle.getId().getId().toString()); + new TypeReference<>() {}, widgetsBundle.getId().getId().toString()); List widgetTypesFromDetailsListSysAdmin = sysAdminLoadedWidgetTypesDetails.stream().map(WidgetType::new).collect(Collectors.toList()); Collections.sort(widgetTypesFromDetailsListSysAdmin, idComparator); Assert.assertEquals(widgetTypesFromDetailsListSysAdmin, loadedWidgetTypes); @@ -225,7 +223,7 @@ public class WidgetTypeControllerTest extends AbstractControllerTest { widgetType.setDescriptor(JacksonUtil.fromString("{ \"someKey\": \"someValue\" }", JsonNode.class)); WidgetTypeDetails savedWidgetType = doPost("/api/widgetType", widgetType, WidgetTypeDetails.class); WidgetType foundWidgetType = doGet("/api/widgetType?fqn={fqn}", - WidgetType.class, "tenant."+savedWidgetType.getFqn()); + WidgetType.class, "tenant." + savedWidgetType.getFqn()); Assert.assertNotNull(foundWidgetType); Assert.assertEquals(new WidgetType(savedWidgetType), foundWidgetType); } diff --git a/application/src/test/java/org/thingsboard/server/controller/WidgetsBundleControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/WidgetsBundleControllerTest.java index bb7bab22f3..03506af95c 100644 --- a/application/src/test/java/org/thingsboard/server/controller/WidgetsBundleControllerTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/WidgetsBundleControllerTest.java @@ -53,7 +53,7 @@ public class WidgetsBundleControllerTest extends AbstractControllerTest { Tenant tenant = new Tenant(); tenant.setTitle("My tenant"); - savedTenant = doPost("/api/tenant", tenant, Tenant.class); + savedTenant = saveTenant(tenant); Assert.assertNotNull(savedTenant); tenantAdmin = new User(); @@ -70,8 +70,7 @@ public class WidgetsBundleControllerTest extends AbstractControllerTest { public void afterTest() throws Exception { loginSysAdmin(); - doDelete("/api/tenant/" + savedTenant.getId().getId().toString()) - .andExpect(status().isOk()); + deleteTenant(savedTenant.getId()); } @Test diff --git a/application/src/test/java/org/thingsboard/server/edge/TenantEdgeTest.java b/application/src/test/java/org/thingsboard/server/edge/TenantEdgeTest.java index 7d740ca472..2611aa433c 100644 --- a/application/src/test/java/org/thingsboard/server/edge/TenantEdgeTest.java +++ b/application/src/test/java/org/thingsboard/server/edge/TenantEdgeTest.java @@ -40,7 +40,7 @@ public class TenantEdgeTest extends AbstractEdgeTest { // updated edge tenant savedTenant.setTitle("Updated Title for Tenant Edge Test"); edgeImitator.expectMessageAmount(2); // expect tenant and tenant profile update msg - savedTenant = doPost("/api/tenant", savedTenant, Tenant.class); + savedTenant = saveTenant(savedTenant); Assert.assertTrue(edgeImitator.waitForMessages()); Optional tenantUpdateMsgOpt = edgeImitator.findMessageByType(TenantUpdateMsg.class); Assert.assertTrue(tenantUpdateMsgOpt.isPresent()); @@ -61,7 +61,7 @@ public class TenantEdgeTest extends AbstractEdgeTest { TenantProfile tenantProfile = createTenantProfile(); savedTenant.setTenantProfileId(tenantProfile.getId()); edgeImitator.expectMessageAmount(2); // expect tenant and tenant profile update msg - savedTenant = doPost("/api/tenant", savedTenant, Tenant.class); + savedTenant = saveTenant(savedTenant); Assert.assertTrue(edgeImitator.waitForMessages()); tenantUpdateMsgOpt = edgeImitator.findMessageByType(TenantUpdateMsg.class); Assert.assertTrue(tenantUpdateMsgOpt.isPresent()); diff --git a/application/src/test/java/org/thingsboard/server/rules/flow/AbstractRuleEngineFlowIntegrationTest.java b/application/src/test/java/org/thingsboard/server/rules/flow/AbstractRuleEngineFlowIntegrationTest.java index 1a172be8dd..9e36fed677 100644 --- a/application/src/test/java/org/thingsboard/server/rules/flow/AbstractRuleEngineFlowIntegrationTest.java +++ b/application/src/test/java/org/thingsboard/server/rules/flow/AbstractRuleEngineFlowIntegrationTest.java @@ -27,9 +27,9 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.test.util.ReflectionTestUtils; import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.rule.engine.flow.TbRuleChainInputNodeConfiguration; -import org.thingsboard.rule.engine.util.TbMsgSource; import org.thingsboard.rule.engine.metadata.TbGetAttributesNode; import org.thingsboard.rule.engine.metadata.TbGetAttributesNodeConfiguration; +import org.thingsboard.rule.engine.util.TbMsgSource; import org.thingsboard.server.actors.ActorSystemContext; import org.thingsboard.server.common.data.AttributeScope; import org.thingsboard.server.common.data.DataConstants; @@ -62,7 +62,6 @@ import java.util.List; import java.util.stream.Collectors; import static org.mockito.Mockito.spy; -import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; /** * @author Valerii Sosliuk @@ -103,7 +102,7 @@ public abstract class AbstractRuleEngineFlowIntegrationTest extends AbstractRule Tenant tenant = new Tenant(); tenant.setTitle("My tenant"); - savedTenant = doPost("/api/tenant", tenant, Tenant.class); + savedTenant = saveTenant(tenant); Assert.assertNotNull(savedTenant); ruleChainService.deleteRuleChainsByTenantId(savedTenant.getId()); @@ -121,7 +120,7 @@ public abstract class AbstractRuleEngineFlowIntegrationTest extends AbstractRule public void afterTest() throws Exception { loginSysAdmin(); if (savedTenant != null) { - doDelete("/api/tenant/" + savedTenant.getId().getId().toString()).andExpect(status().isOk()); + deleteTenant(savedTenant.getId()); } } diff --git a/application/src/test/java/org/thingsboard/server/service/entitiy/alarm/DefaultTbAlarmServiceTest.java b/application/src/test/java/org/thingsboard/server/service/entitiy/alarm/DefaultTbAlarmServiceTest.java index 39651b8b27..eb20210938 100644 --- a/application/src/test/java/org/thingsboard/server/service/entitiy/alarm/DefaultTbAlarmServiceTest.java +++ b/application/src/test/java/org/thingsboard/server/service/entitiy/alarm/DefaultTbAlarmServiceTest.java @@ -173,7 +173,7 @@ public class DefaultTbAlarmServiceTest { User user = new User(); user.setEmail("testEmail@gmail.com"); user.setId(new UserId(UUID.randomUUID())); - service.unassignDeletedUserAlarms(new TenantId(UUID.randomUUID()), user, System.currentTimeMillis()); + service.unassignDeletedUserAlarms(new TenantId(UUID.randomUUID()), user.getId(), user.getTitle(), System.currentTimeMillis()); ObjectNode commentNode = JacksonUtil.newObjectNode(); commentNode.put("subtype", "ASSIGN"); diff --git a/application/src/test/java/org/thingsboard/server/service/housekeeper/HousekeeperServiceTest.java b/application/src/test/java/org/thingsboard/server/service/housekeeper/HousekeeperServiceTest.java new file mode 100644 index 0000000000..e8f1a43ad1 --- /dev/null +++ b/application/src/test/java/org/thingsboard/server/service/housekeeper/HousekeeperServiceTest.java @@ -0,0 +1,478 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.housekeeper; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.node.TextNode; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.mockito.ArgumentMatcher; +import org.mockito.Mockito; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.mock.mockito.SpyBean; +import org.springframework.test.context.TestPropertySource; +import org.testcontainers.shaded.org.apache.commons.lang3.RandomStringUtils; +import org.thingsboard.common.util.JacksonUtil; +import org.thingsboard.rule.engine.metadata.TbGetAttributesNode; +import org.thingsboard.rule.engine.metadata.TbGetAttributesNodeConfiguration; +import org.thingsboard.server.common.data.ApiUsageState; +import org.thingsboard.server.common.data.AttributeScope; +import org.thingsboard.server.common.data.Device; +import org.thingsboard.server.common.data.EventInfo; +import org.thingsboard.server.common.data.alarm.Alarm; +import org.thingsboard.server.common.data.alarm.AlarmSeverity; +import org.thingsboard.server.common.data.alarm.EntityAlarm; +import org.thingsboard.server.common.data.asset.Asset; +import org.thingsboard.server.common.data.event.EventType; +import org.thingsboard.server.common.data.event.LifecycleEvent; +import org.thingsboard.server.common.data.housekeeper.HousekeeperTask; +import org.thingsboard.server.common.data.housekeeper.HousekeeperTaskType; +import org.thingsboard.server.common.data.id.AlarmId; +import org.thingsboard.server.common.data.id.AssetId; +import org.thingsboard.server.common.data.id.DeviceId; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.RuleChainId; +import org.thingsboard.server.common.data.id.RuleNodeId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.id.UserId; +import org.thingsboard.server.common.data.kv.BaseAttributeKvEntry; +import org.thingsboard.server.common.data.kv.BaseReadTsKvQuery; +import org.thingsboard.server.common.data.kv.BasicTsKvEntry; +import org.thingsboard.server.common.data.kv.StringDataEntry; +import org.thingsboard.server.common.data.kv.TsKvEntry; +import org.thingsboard.server.common.data.msg.TbNodeConnectionType; +import org.thingsboard.server.common.data.page.PageLink; +import org.thingsboard.server.common.data.page.TimePageLink; +import org.thingsboard.server.common.data.relation.EntityRelation; +import org.thingsboard.server.common.data.relation.RelationTypeGroup; +import org.thingsboard.server.common.data.rule.RuleChain; +import org.thingsboard.server.common.data.rule.RuleChainMetaData; +import org.thingsboard.server.common.data.rule.RuleChainType; +import org.thingsboard.server.common.data.rule.RuleNode; +import org.thingsboard.server.controller.AbstractControllerTest; +import org.thingsboard.server.dao.alarm.AlarmDao; +import org.thingsboard.server.dao.alarm.AlarmService; +import org.thingsboard.server.dao.attributes.AttributesService; +import org.thingsboard.server.dao.entity.EntityServiceRegistry; +import org.thingsboard.server.dao.event.EventService; +import org.thingsboard.server.dao.relation.RelationService; +import org.thingsboard.server.dao.rule.RuleChainService; +import org.thingsboard.server.dao.service.DaoSqlTest; +import org.thingsboard.server.dao.timeseries.TimeseriesService; +import org.thingsboard.server.dao.usagerecord.ApiUsageStateDao; +import org.thingsboard.server.gen.transport.TransportProtos.HousekeeperTaskProto; +import org.thingsboard.server.gen.transport.TransportProtos.ToHousekeeperServiceMsg; +import org.thingsboard.server.service.housekeeper.processor.TsHistoryDeletionTaskProcessor; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.Optional; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; +import java.util.function.Function; +import java.util.function.Predicate; +import java.util.stream.Collectors; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.awaitility.Awaitility.await; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.argThat; +import static org.mockito.Mockito.doCallRealMethod; +import static org.mockito.Mockito.doThrow; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; +import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; + +@DaoSqlTest +@TestPropertySource(properties = { + "transport.http.enabled=true", + "queue.core.housekeeper.task-reprocessing-delay-ms=2000", + "queue.core.housekeeper.poll-interval-ms=1000", + "queue.core.housekeeper.max-reprocessing-attempts=5" +}) +public class HousekeeperServiceTest extends AbstractControllerTest { + + @SpyBean + private HousekeeperService housekeeperService; + @SpyBean + private HousekeeperReprocessingService housekeeperReprocessingService; + @Autowired + private EventService eventService; + @Autowired + private TimeseriesService timeseriesService; + @Autowired + private AttributesService attributesService; + @Autowired + private RuleChainService ruleChainService; + @Autowired + private AlarmService alarmService; + @Autowired + private AlarmDao alarmDao; + @Autowired + private RelationService relationService; + @Autowired + private ApiUsageStateDao apiUsageStateDao; + @Autowired + private EntityServiceRegistry entityServiceRegistry; + @SpyBean + private TsHistoryDeletionTaskProcessor tsHistoryDeletionTaskProcessor; + + private TenantId tenantId; + + private static final String TELEMETRY_KEY = "telemetry1"; + private static final String ATTRIBUTE_KEY = "_attribute1"; + private static final String KV_VALUE = "ewfewfwef"; + + @Before + public void setUp() throws Exception { + loginTenantAdmin(); + this.tenantId = super.tenantId; + } + + @After + public void tearDown() throws Exception { + } + + @Test + public void whenDeviceIsDeleted_thenCleanUpRelatedData() throws Exception { + Device device = createDevice("test", "test"); + createRelatedData(device.getId()); + + doDelete("/api/device/" + device.getId()).andExpect(status().isOk()); + + await().atMost(10, TimeUnit.SECONDS).untilAsserted(() -> { + verifyNoRelatedData(device.getId()); + }); + } + + @Test + public void whenRuleChainIsDeleted_thenCleanUpRelatedData() throws Exception { + RuleChainMetaData ruleChainMetaData = createRuleChain(); + RuleChainId ruleChainId = ruleChainMetaData.getRuleChainId(); + RuleNodeId ruleNode1Id = ruleChainMetaData.getNodes().get(0).getId(); + RuleNodeId ruleNode2Id = ruleChainMetaData.getNodes().get(1).getId(); + createRelatedData(ruleChainId); + createRelatedData(ruleNode1Id); + createRelatedData(ruleNode2Id); + + doDelete("/api/ruleChain/" + ruleChainId).andExpect(status().isOk()); + + await().atMost(10, TimeUnit.SECONDS).untilAsserted(() -> { + verifyNoRelatedData(ruleNode1Id); + verifyNoRelatedData(ruleNode2Id); + verifyNoRelatedData(ruleChainId); + }); + } + + @Test + public void whenUserIsDeleted_thenCleanUpRelatedData() throws Exception { + Device device = createDevice("test", "test"); + UserId userId = customerUserId; + createRelatedData(userId); + Alarm alarm = Alarm.builder() + .type("test") + .tenantId(tenantId) + .originator(device.getId()) + .severity(AlarmSeverity.MAJOR) + .build(); + alarm = doPost("/api/alarm", alarm, Alarm.class); + AlarmId alarmId = alarm.getId(); + alarm = doPost("/api/alarm/" + alarmId + "/assign/" + userId, "", Alarm.class); + assertThat(alarm.getAssigneeId()).isEqualTo(userId); + assertThat(alarmService.findAlarmIdsByAssigneeId(tenantId, userId, new PageLink(100)).getData()).isNotEmpty(); + + doDelete("/api/user/" + userId).andExpect(status().isOk()); + + await().atMost(10, TimeUnit.SECONDS).untilAsserted(() -> { + verifyNoRelatedData(userId); + assertThat(alarmService.findAlarmById(tenantId, alarmId).getAssigneeId()).isNull(); + }); + } + + @Test + public void whenDeviceIsDeleted_thenDeleteAllAlarms() throws Exception { + Device device = createDevice("test", "test"); + for (int i = 1; i <= 1000; i++) { + createAlarm(device.getId()); + } + + doDelete("/api/device/" + device.getId()).andExpect(status().isOk()); + + await().atMost(30, TimeUnit.SECONDS).untilAsserted(() -> { + verifyNoAlarms(device.getId()); + }); + } + + @Test + public void whenTenantIsDeleted_thenDeleteAllEntitiesAndCleanUpRelatedData() throws Exception { + loginDifferentTenant(); + tenantId = differentTenantId; + + createRelatedData(tenantId); + createDifferentTenantCustomer(); + createRelatedData(differentTenantCustomerId); + loginDifferentTenant(); + + List devices = new ArrayList<>(); + for (int i = 1; i <= 300; i++) { + Device device = createDevice("test" + i, "test" + i); + devices.add(device.getId()); + } + DeviceId firstDevice = devices.get(0); + createRelatedData(firstDevice); + DeviceId lastDevice = devices.get(devices.size() - 1); + createRelatedData(lastDevice); + + Asset asset = createAsset(); + createRelatedData(asset.getId()); + createRelation(firstDevice, asset.getId()); + createAlarm(firstDevice, asset.getId()); + + RuleChainMetaData ruleChainMetaData = createRuleChain(); + RuleChainId ruleChainId = ruleChainMetaData.getRuleChainId(); + RuleNodeId ruleNode1Id = ruleChainMetaData.getNodes().get(0).getId(); + RuleNodeId ruleNode2Id = ruleChainMetaData.getNodes().get(1).getId(); + createRelatedData(ruleChainId); + createRelatedData(ruleNode1Id); + createRelatedData(ruleNode2Id); + + UserId userId = savedDifferentTenantUser.getId(); + createRelatedData(userId); + + ApiUsageState tenantApiUsageState = apiUsageStateDao.findApiUsageStateByEntityId(differentTenantId); + + loginSysAdmin(); + deleteDifferentTenant(); + + await().atMost(60, TimeUnit.SECONDS).pollInterval(1, TimeUnit.SECONDS).untilAsserted(() -> { + for (DeviceId deviceId : devices) { + verifyNoRelatedData(deviceId); + } + verifyNoRelatedData(asset.getId()); + verifyNoRelatedData(ruleNode1Id); + verifyNoRelatedData(ruleNode2Id); + verifyNoRelatedData(ruleChainId); + verifyNoRelatedData(userId); + verifyNoRelatedData(differentTenantCustomerId); + verifyNoRelatedData(tenantApiUsageState.getId()); + verifyNoRelatedData(tenantId); + }); + } + + @Test + public void whenTaskProcessingFails_thenReprocess() throws Exception { + TimeoutException error = new TimeoutException("Test timeout"); + doThrow(error).when(tsHistoryDeletionTaskProcessor).process(any()); + + Device device = createDevice("test", "test"); + createRelatedData(device.getId()); + + doDelete("/api/device/" + device.getId()).andExpect(status().isOk()); + + int attempts = 2; + await().atMost(30, TimeUnit.SECONDS).pollInterval(1, TimeUnit.SECONDS).untilAsserted(() -> { + for (int i = 0; i <= attempts; i++) { + int attempt = i; + verify(housekeeperReprocessingService).submitForReprocessing(argThat(getTaskMatcher(device.getId(), HousekeeperTaskType.DELETE_TS_HISTORY, + task -> task.getAttempt() == attempt)), argThat(e -> e.getMessage().equals(error.getMessage()))); + } + }); + + assertThat(getTimeseriesHistory(device.getId())).isNotEmpty(); + doCallRealMethod().when(tsHistoryDeletionTaskProcessor).process(any()); + await().atMost(30, TimeUnit.SECONDS).untilAsserted(() -> { + assertThat(getTimeseriesHistory(device.getId())).isEmpty(); + }); + } + + @Test + public void whenReprocessingAttemptsExceeded_thenDropTheTask() throws Exception { + TimeoutException error = new TimeoutException("Test timeout"); + doThrow(error).when(tsHistoryDeletionTaskProcessor).process(any()); + + Device device = createDevice("test", "test"); + createRelatedData(device.getId()); + + doDelete("/api/device/" + device.getId()).andExpect(status().isOk()); + + int maxAttempts = 5; + await().atMost(30, TimeUnit.SECONDS).untilAsserted(() -> { + for (int i = 1; i <= maxAttempts; i++) { + verifyTaskProcessing(device.getId(), HousekeeperTaskType.DELETE_TS_HISTORY, i); + } + }); + + Mockito.clearInvocations(housekeeperService); + doCallRealMethod().when(tsHistoryDeletionTaskProcessor).process(any()); + TimeUnit.SECONDS.sleep(2); + verify(housekeeperService, never()).processTask(argThat(getTaskMatcher(device.getId(), HousekeeperTaskType.DELETE_TS_HISTORY, null))); + } + + private void verifyTaskProcessing(EntityId entityId, HousekeeperTaskType taskType, int expectedAttempt) throws Exception { + verify(housekeeperService).processTask(argThat(getTaskMatcher(entityId, taskType, task -> task.getAttempt() == expectedAttempt))); + } + + private ArgumentMatcher getTaskMatcher(EntityId entityId, HousekeeperTaskType taskType, + Predicate additionalCheck) { + return msg -> { + HousekeeperTask task = JacksonUtil.fromString(msg.getTask().getValue(), HousekeeperTask.class); + return task.getEntityId().equals(entityId) && task.getTaskType() == taskType && (additionalCheck == null || additionalCheck.test(msg.getTask())); + }; + } + + private void createRelatedData(EntityId entityId) throws Exception { + createTelemetry(entityId); + for (AttributeScope scope : AttributeScope.values()) { + createAttribute(entityId, scope, scope + ATTRIBUTE_KEY); + } + createEvent(entityId); + } + + private void verifyNoRelatedData(EntityId entityId) throws Exception { + assertThat(entityServiceRegistry.getServiceByEntityType(entityId.getEntityType()).findEntity(tenantId, entityId)).isEmpty(); + + assertThat(getLatestTelemetry(entityId)).isNull(); + assertThat(getTimeseriesHistory(entityId)).isEmpty(); + for (AttributeScope scope : AttributeScope.values()) { + assertThat(attributesService.findAll(tenantId, entityId, scope).get()).isEmpty(); + } + assertThat(getEvents(entityId)).isEmpty(); + assertThat(alarmDao.findEntityAlarmRecordsByEntityId(tenantId, entityId)).isEmpty(); + verifyNoAlarms(entityId); + assertThat(relationService.findByTo(tenantId, entityId, RelationTypeGroup.COMMON)).isEmpty(); + assertThat(relationService.findByFrom(tenantId, entityId, RelationTypeGroup.COMMON)).isEmpty(); + } + + private void verifyNoAlarms(EntityId entityId) { + assertThat(alarmService.findAlarmIdsByOriginatorId(tenantId, entityId, 0, null, 10)).isEmpty(); + } + + private void createAttribute(EntityId entityId, AttributeScope scope, String key) throws Exception { + attributesService.save(tenantId, entityId, scope, new BaseAttributeKvEntry(System.currentTimeMillis(), new StringDataEntry(key, KV_VALUE))).get(); + } + + private void createTelemetry(EntityId entityId) throws Exception { + timeseriesService.save(tenantId, entityId, new BasicTsKvEntry(System.currentTimeMillis(), new StringDataEntry(TELEMETRY_KEY, KV_VALUE))).get(); + } + + private void createEvent(EntityId entityId) { + LifecycleEvent event = LifecycleEvent.builder() + .tenantId(tenantId) + .entityId(entityId.getId()) + .serviceId("test") + .lcEventType("test") + .success(true) + .build(); + eventService.saveAsync(event); + await().atMost(10, TimeUnit.SECONDS) + .until(() -> !getEvents(entityId).isEmpty()); + + } + + private void createRelation(DeviceId to, AssetId from) { + EntityRelation relation = new EntityRelation(from, to, EntityRelation.CONTAINS_TYPE, RelationTypeGroup.COMMON); + relationService.saveRelation(tenantId, relation); + } + + private void createAlarm(DeviceId deviceId, EntityId propagatedEntityId) { + Alarm alarm = doPost("/api/alarm", Alarm.builder() + .tenantId(tenantId) + .originator(deviceId) + .severity(AlarmSeverity.CRITICAL) + .type("test alarm for " + deviceId) + .propagate(true) + .build(), Alarm.class); + + List entityAlarms = alarmDao.findEntityAlarmRecords(tenantId, alarm.getId()); + assertThat(entityAlarms).anyMatch(entityAlarm -> entityAlarm.getEntityId().equals(deviceId) && entityAlarm.getAlarmType().equals(alarm.getType())); + assertThat(entityAlarms).anyMatch(entityAlarm -> entityAlarm.getEntityId().equals(propagatedEntityId) && entityAlarm.getAlarmType().equals(alarm.getType())); + assertThat(alarmService.findAlarmIdsByOriginatorId(tenantId, deviceId, 0, null, 10)).isNotEmpty(); + } + + private void createAlarm(DeviceId deviceId) { + Alarm alarm = doPost("/api/alarm", Alarm.builder() + .tenantId(tenantId) + .originator(deviceId) + .severity(AlarmSeverity.CRITICAL) + .type("test alarm for " + deviceId + " " + RandomStringUtils.randomAlphabetic(10)) + .build(), Alarm.class); + assertThat(alarmService.findAlarmIdsByOriginatorId(tenantId, deviceId, 0, null, 10)).isNotEmpty(); + } + + private TsKvEntry getLatestTelemetry(EntityId entityId) throws Exception { + return timeseriesService.findLatest(tenantId, entityId, HousekeeperServiceTest.TELEMETRY_KEY).get().orElse(null); + } + + private List getTimeseriesHistory(EntityId entityId) throws Exception { + return timeseriesService.findAll(tenantId, entityId, List.of(new BaseReadTsKvQuery(HousekeeperServiceTest.TELEMETRY_KEY, 0, System.currentTimeMillis(), 10, "DESC"))).get(); + } + + private List getEvents(EntityId entityId) { + return eventService.findEvents(tenantId, entityId, EventType.LC_EVENT, new TimePageLink(100)).getData() + .stream().filter(event -> Optional.ofNullable(event.getBody()).map(body -> body.get("event")) + .map(JsonNode::asText).orElse("").equals("test")) + .collect(Collectors.toList()); + } + + private Asset createAsset() { + Asset asset = new Asset(); + asset.setName("test"); + asset.setType("test"); + return doPost("/api/asset", asset, Asset.class); + } + + private RuleChainMetaData createRuleChain() { + RuleChain ruleChain = new RuleChain(); + ruleChain.setTenantId(tenantId); + ruleChain.setName("Test"); + ruleChain.setType(RuleChainType.CORE); + ruleChain.setDebugMode(true); + ruleChain.setConfiguration(JacksonUtil.newObjectNode().set("a", new TextNode("b"))); + ruleChain = ruleChainService.saveRuleChain(ruleChain); + RuleChainId ruleChainId = ruleChain.getId(); + + RuleChainMetaData metaData = new RuleChainMetaData(); + metaData.setRuleChainId(ruleChainId); + + RuleNode ruleNode1 = new RuleNode(); + ruleNode1.setName("Simple Rule Node 1"); + ruleNode1.setType(org.thingsboard.rule.engine.metadata.TbGetAttributesNode.class.getName()); + ruleNode1.setConfigurationVersion(TbGetAttributesNode.class.getAnnotation(org.thingsboard.rule.engine.api.RuleNode.class).version()); + ruleNode1.setDebugMode(true); + TbGetAttributesNodeConfiguration configuration1 = new TbGetAttributesNodeConfiguration(); + configuration1.setServerAttributeNames(Collections.singletonList("serverAttributeKey1")); + ruleNode1.setConfiguration(JacksonUtil.valueToTree(configuration1)); + + RuleNode ruleNode2 = new RuleNode(); + ruleNode2.setName("Simple Rule Node 2"); + ruleNode2.setType(org.thingsboard.rule.engine.metadata.TbGetAttributesNode.class.getName()); + ruleNode2.setConfigurationVersion(TbGetAttributesNode.class.getAnnotation(org.thingsboard.rule.engine.api.RuleNode.class).version()); + ruleNode2.setDebugMode(true); + TbGetAttributesNodeConfiguration configuration2 = new TbGetAttributesNodeConfiguration(); + configuration2.setServerAttributeNames(Collections.singletonList("serverAttributeKey2")); + ruleNode2.setConfiguration(JacksonUtil.valueToTree(configuration2)); + + metaData.setNodes(Arrays.asList(ruleNode1, ruleNode2)); + metaData.setFirstNodeIndex(0); + metaData.addConnectionInfo(0, 1, TbNodeConnectionType.SUCCESS); + ruleChainService.saveRuleChainMetaData(tenantId, metaData, Function.identity()); + return ruleChainService.loadRuleChainMetaData(tenantId, ruleChainId); + } + +} diff --git a/application/src/test/java/org/thingsboard/server/service/notification/NotificationApiTest.java b/application/src/test/java/org/thingsboard/server/service/notification/NotificationApiTest.java index 4bb8058e7f..1a788d437b 100644 --- a/application/src/test/java/org/thingsboard/server/service/notification/NotificationApiTest.java +++ b/application/src/test/java/org/thingsboard/server/service/notification/NotificationApiTest.java @@ -323,8 +323,10 @@ public class NotificationApiTest extends AbstractNotificationApiTest { deleteDifferentTenant(); - assertThat(notificationRequestService.findNotificationRequestsByTenantIdAndOriginatorType(tenantId, EntityType.USER, new PageLink(1)).getTotalElements()) - .isZero(); + await().atMost(TIMEOUT, TimeUnit.SECONDS).untilAsserted(() -> { + assertThat(notificationRequestService.findNotificationRequestsByTenantIdAndOriginatorType(tenantId, EntityType.USER, new PageLink(1)).getTotalElements()) + .isZero(); + }); } @Test diff --git a/application/src/test/java/org/thingsboard/server/service/notification/NotificationTargetApiTest.java b/application/src/test/java/org/thingsboard/server/service/notification/NotificationTargetApiTest.java index c7ef9a5836..a5bdc6c2e4 100644 --- a/application/src/test/java/org/thingsboard/server/service/notification/NotificationTargetApiTest.java +++ b/application/src/test/java/org/thingsboard/server/service/notification/NotificationTargetApiTest.java @@ -38,8 +38,10 @@ import org.thingsboard.server.dao.service.DaoSqlTest; import java.util.Collections; import java.util.List; +import java.util.concurrent.TimeUnit; import static org.assertj.core.api.Assertions.assertThat; +import static org.awaitility.Awaitility.await; import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; @DaoSqlTest @@ -148,7 +150,9 @@ public class NotificationTargetApiTest extends AbstractNotificationApiTest { assertThat(notificationTargetDao.findByTenantIdAndSupportedNotificationTypeAndPageLink(differentTenantId, NotificationType.GENERAL, new PageLink(10)).getData()).isNotEmpty(); deleteDifferentTenant(); - assertThat(notificationTargetDao.findByTenantIdAndPageLink(differentTenantId, new PageLink(10)).getData()).isEmpty(); + await().atMost(TIMEOUT, TimeUnit.SECONDS).untilAsserted(() -> { + assertThat(notificationTargetDao.findByTenantIdAndPageLink(differentTenantId, new PageLink(10)).getData()).isEmpty(); + }); } @Test diff --git a/application/src/test/java/org/thingsboard/server/service/resource/sql/BaseTbResourceServiceTest.java b/application/src/test/java/org/thingsboard/server/service/resource/sql/BaseTbResourceServiceTest.java index d9b33edba2..566d272a2f 100644 --- a/application/src/test/java/org/thingsboard/server/service/resource/sql/BaseTbResourceServiceTest.java +++ b/application/src/test/java/org/thingsboard/server/service/resource/sql/BaseTbResourceServiceTest.java @@ -51,7 +51,6 @@ import static org.assertj.core.api.Assertions.assertThatThrownBy; import static org.junit.Assert.assertArrayEquals; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertThrows; -import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; @DaoSqlTest public class BaseTbResourceServiceTest extends AbstractControllerTest { @@ -129,7 +128,7 @@ public class BaseTbResourceServiceTest extends AbstractControllerTest { Tenant tenant = new Tenant(); tenant.setTitle("My tenant"); - savedTenant = doPost("/api/tenant", tenant, Tenant.class); + savedTenant = saveTenant(tenant); tenantId = savedTenant.getId(); Assert.assertNotNull(savedTenant); @@ -147,8 +146,7 @@ public class BaseTbResourceServiceTest extends AbstractControllerTest { public void afterTest() throws Exception { loginSysAdmin(); - doDelete("/api/tenant/" + savedTenant.getId().getId().toString()) - .andExpect(status().isOk()); + deleteTenant(savedTenant.getId()); } @Test @@ -397,7 +395,7 @@ public class BaseTbResourceServiceTest extends AbstractControllerTest { loginSysAdmin(); Tenant tenant = new Tenant(); tenant.setTitle("Test tenant"); - tenant = doPost("/api/tenant", tenant, Tenant.class); + tenant = saveTenant(tenant); TenantId tenantId = tenant.getId(); @@ -441,8 +439,7 @@ public class BaseTbResourceServiceTest extends AbstractControllerTest { Assert.assertFalse(pageData.hasNext()); Assert.assertTrue(pageData.getData().isEmpty()); - doDelete("/api/tenant/" + tenantId.getId().toString()) - .andExpect(status().isOk()); + deleteTenant(tenantId); } @Test @@ -451,7 +448,7 @@ public class BaseTbResourceServiceTest extends AbstractControllerTest { Tenant tenant = new Tenant(); tenant.setTitle("Test tenant"); - tenant = doPost("/api/tenant", tenant, Tenant.class); + tenant = saveTenant(tenant); TenantId tenantId = tenant.getId(); @@ -518,8 +515,7 @@ public class BaseTbResourceServiceTest extends AbstractControllerTest { Assert.assertFalse(pageData.hasNext()); Assert.assertTrue(pageData.getData().isEmpty()); - doDelete("/api/tenant/" + tenantId.getId().toString()) - .andExpect(status().isOk()); + deleteTenant(tenantId); } } diff --git a/application/src/test/java/org/thingsboard/server/service/sql/SequentialTimeseriesPersistenceTest.java b/application/src/test/java/org/thingsboard/server/service/sql/SequentialTimeseriesPersistenceTest.java index 5cae75ef0d..02281abac3 100644 --- a/application/src/test/java/org/thingsboard/server/service/sql/SequentialTimeseriesPersistenceTest.java +++ b/application/src/test/java/org/thingsboard/server/service/sql/SequentialTimeseriesPersistenceTest.java @@ -50,8 +50,6 @@ import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; -import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; - @DaoSqlTest public class SequentialTimeseriesPersistenceTest extends AbstractControllerTest { @@ -77,7 +75,7 @@ public class SequentialTimeseriesPersistenceTest extends AbstractControllerTest Tenant tenant = new Tenant(); tenant.setTitle("My tenant"); - savedTenant = doPost("/api/tenant", tenant, Tenant.class); + savedTenant = saveTenant(tenant); Assert.assertNotNull(savedTenant); tenantAdmin = new User(); @@ -93,7 +91,7 @@ public class SequentialTimeseriesPersistenceTest extends AbstractControllerTest @After public void afterTest() throws Exception { loginSysAdmin(); - doDelete("/api/tenant/" + savedTenant.getId().getId().toString()).andExpect(status().isOk()); + deleteTenant(savedTenant.getId()); } @Test diff --git a/application/src/test/resources/logback-test.xml b/application/src/test/resources/logback-test.xml index 8b20a2817a..f43144a24e 100644 --- a/application/src/test/resources/logback-test.xml +++ b/application/src/test/resources/logback-test.xml @@ -31,6 +31,7 @@ + diff --git a/common/dao-api/src/main/java/org/thingsboard/server/dao/alarm/AlarmService.java b/common/dao-api/src/main/java/org/thingsboard/server/dao/alarm/AlarmService.java index a6e6e2390b..5bf6b95853 100644 --- a/common/dao-api/src/main/java/org/thingsboard/server/dao/alarm/AlarmService.java +++ b/common/dao-api/src/main/java/org/thingsboard/server/dao/alarm/AlarmService.java @@ -38,10 +38,13 @@ import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.common.data.query.AlarmCountQuery; import org.thingsboard.server.common.data.query.AlarmData; import org.thingsboard.server.common.data.query.AlarmDataQuery; +import org.thingsboard.server.common.data.util.TbPair; import org.thingsboard.server.dao.entity.EntityDaoService; import java.util.Collection; +import java.util.List; import java.util.Set; +import java.util.UUID; public interface AlarmService extends EntityDaoService { @@ -106,9 +109,9 @@ public interface AlarmService extends EntityDaoService { PageData findAlarmIdsByAssigneeId(TenantId tenantId, UserId userId, PageLink pageLink); - PageData findAlarmIdsByOriginatorId(TenantId tenantId, EntityId originatorId, PageLink pageLink); + List> findAlarmIdsByOriginatorId(TenantId tenantId, EntityId originatorId, long createdTimeOffset, AlarmId idOffset, int limit); - void deleteEntityAlarmRelations(TenantId tenantId, EntityId entityId); + int deleteEntityAlarmRecords(TenantId tenantId, EntityId entityId); void deleteEntityAlarmRecordsByTenantId(TenantId tenantId); diff --git a/common/dao-api/src/main/java/org/thingsboard/server/dao/attributes/AttributesService.java b/common/dao-api/src/main/java/org/thingsboard/server/dao/attributes/AttributesService.java index 57c15580f6..d869471fa3 100644 --- a/common/dao-api/src/main/java/org/thingsboard/server/dao/attributes/AttributesService.java +++ b/common/dao-api/src/main/java/org/thingsboard/server/dao/attributes/AttributesService.java @@ -71,4 +71,6 @@ public interface AttributesService { List findAllKeysByEntityIds(TenantId tenantId, List entityIds, String scope); + int removeAllByEntityId(TenantId tenantId, EntityId entityId); + } diff --git a/common/dao-api/src/main/java/org/thingsboard/server/dao/entity/EntityDaoService.java b/common/dao-api/src/main/java/org/thingsboard/server/dao/entity/EntityDaoService.java index b1f6ab9cdb..cf55cf1054 100644 --- a/common/dao-api/src/main/java/org/thingsboard/server/dao/entity/EntityDaoService.java +++ b/common/dao-api/src/main/java/org/thingsboard/server/dao/entity/EntityDaoService.java @@ -30,10 +30,14 @@ public interface EntityDaoService { throw new IllegalArgumentException("Not implemented for " + getEntityType()); } - default void deleteEntity(TenantId tenantId, EntityId id) { + default void deleteEntity(TenantId tenantId, EntityId id, boolean force) { throw new IllegalArgumentException(getEntityType().getNormalName() + " deletion not supported"); } + default void deleteByTenantId(TenantId tenantId) { + throw new IllegalArgumentException("Deletion by tenant id not supported for " + getEntityType().getNormalName()); + } + EntityType getEntityType(); } diff --git a/common/dao-api/src/main/java/org/thingsboard/server/dao/notification/NotificationTemplateService.java b/common/dao-api/src/main/java/org/thingsboard/server/dao/notification/NotificationTemplateService.java index 1f4019063b..664f976904 100644 --- a/common/dao-api/src/main/java/org/thingsboard/server/dao/notification/NotificationTemplateService.java +++ b/common/dao-api/src/main/java/org/thingsboard/server/dao/notification/NotificationTemplateService.java @@ -32,6 +32,8 @@ public interface NotificationTemplateService { PageData findNotificationTemplatesByTenantIdAndNotificationTypes(TenantId tenantId, List notificationTypes, PageLink pageLink); + int countNotificationTemplatesByTenantIdAndNotificationTypes(TenantId tenantId, List notificationTypes); + void deleteNotificationTemplateById(TenantId tenantId, NotificationTemplateId id); void deleteNotificationTemplatesByTenantId(TenantId tenantId); diff --git a/common/dao-api/src/main/java/org/thingsboard/server/dao/queue/QueueStatsService.java b/common/dao-api/src/main/java/org/thingsboard/server/dao/queue/QueueStatsService.java index 9ab67c73a7..c9e3f605c6 100644 --- a/common/dao-api/src/main/java/org/thingsboard/server/dao/queue/QueueStatsService.java +++ b/common/dao-api/src/main/java/org/thingsboard/server/dao/queue/QueueStatsService.java @@ -32,6 +32,4 @@ public interface QueueStatsService extends EntityDaoService { List findByTenantId(TenantId tenantId); - void deleteByTenantId(TenantId tenantId); - -} \ No newline at end of file +} diff --git a/common/dao-api/src/main/java/org/thingsboard/server/dao/usagerecord/ApiUsageStateService.java b/common/dao-api/src/main/java/org/thingsboard/server/dao/usagerecord/ApiUsageStateService.java index 2ed877dabd..3d09247c7e 100644 --- a/common/dao-api/src/main/java/org/thingsboard/server/dao/usagerecord/ApiUsageStateService.java +++ b/common/dao-api/src/main/java/org/thingsboard/server/dao/usagerecord/ApiUsageStateService.java @@ -31,8 +31,6 @@ public interface ApiUsageStateService extends EntityDaoService { ApiUsageState findApiUsageStateByEntityId(EntityId entityId); - void deleteApiUsageStateByTenantId(TenantId tenantId); - void deleteApiUsageStateByEntityId(EntityId entityId); ApiUsageState findApiUsageStateById(TenantId tenantId, ApiUsageStateId id); diff --git a/common/dao-api/src/main/java/org/thingsboard/server/dao/user/UserService.java b/common/dao-api/src/main/java/org/thingsboard/server/dao/user/UserService.java index f137bcc334..0cad776e78 100644 --- a/common/dao-api/src/main/java/org/thingsboard/server/dao/user/UserService.java +++ b/common/dao-api/src/main/java/org/thingsboard/server/dao/user/UserService.java @@ -77,6 +77,8 @@ public interface UserService extends EntityDaoService { void deleteTenantAdmins(TenantId tenantId); + void deleteAllByTenantId(TenantId tenantId); + PageData findCustomerUsers(TenantId tenantId, CustomerId customerId, PageLink pageLink); PageData findUsersByCustomerIds(TenantId tenantId, List customerIds, PageLink pageLink); diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/AttributeScope.java b/common/data/src/main/java/org/thingsboard/server/common/data/AttributeScope.java index 0e8335c7cb..3c58dd3b72 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/AttributeScope.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/AttributeScope.java @@ -17,6 +17,10 @@ package org.thingsboard.server.common.data; import lombok.Getter; +import java.util.Arrays; +import java.util.Map; +import java.util.stream.Collectors; + public enum AttributeScope { CLIENT_SCOPE(1), @@ -25,8 +29,15 @@ public enum AttributeScope { @Getter private final int id; + private static final Map values = Arrays.stream(values()) + .collect(Collectors.toMap(AttributeScope::getId, scope -> scope)); + AttributeScope(int id) { this.id = id; } + public static AttributeScope valueOf(int id) { + return values.get(id); + } + } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/EntityType.java b/common/data/src/main/java/org/thingsboard/server/common/data/EntityType.java index bb43bf4d11..6748aebb50 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/EntityType.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/EntityType.java @@ -20,7 +20,6 @@ import org.apache.commons.lang3.StringUtils; import java.util.EnumSet; import java.util.List; -import java.util.stream.Collectors; /** * @author Andrew Shvayka @@ -28,51 +27,57 @@ import java.util.stream.Collectors; public enum EntityType { TENANT(1), CUSTOMER(2), - USER(3), + USER(3, "tb_user"), DASHBOARD(4), ASSET(5), DEVICE(6), - ALARM (7), - RULE_CHAIN (11), - RULE_NODE (12), - - ENTITY_VIEW (15) { + ALARM(7), + RULE_CHAIN(11), + RULE_NODE(12), + ENTITY_VIEW(15) { // backward compatibility for TbOriginatorTypeSwitchNode to return correct rule node connection. @Override - public String getNormalName () { + public String getNormalName() { return "Entity View"; } }, - WIDGETS_BUNDLE (16), - WIDGET_TYPE (17), - TENANT_PROFILE (20), - DEVICE_PROFILE (21), - ASSET_PROFILE (22), - API_USAGE_STATE (23), - TB_RESOURCE (24), - OTA_PACKAGE (25), - EDGE (26), - RPC (27), - QUEUE (28), - NOTIFICATION_TARGET (29), - NOTIFICATION_TEMPLATE (30), - NOTIFICATION_REQUEST (31), - NOTIFICATION (32), - NOTIFICATION_RULE (33), + WIDGETS_BUNDLE(16), + WIDGET_TYPE(17), + TENANT_PROFILE(20), + DEVICE_PROFILE(21), + ASSET_PROFILE(22), + API_USAGE_STATE(23), + TB_RESOURCE(24, "resource"), + OTA_PACKAGE(25), + EDGE(26), + RPC(27), + QUEUE(28), + NOTIFICATION_TARGET(29), + NOTIFICATION_TEMPLATE(30), + NOTIFICATION_REQUEST(31), + NOTIFICATION(32), + NOTIFICATION_RULE(33), QUEUE_STATS(34); @Getter private final int protoNumber; // Corresponds to EntityTypeProto + @Getter + private final String tableName; + @Getter + private final String normalName = StringUtils.capitalize(StringUtils.removeStart(name(), "TB_") + .toLowerCase().replaceAll("_", " ")); + + public static final List NORMAL_NAMES = EnumSet.allOf(EntityType.class).stream() + .map(EntityType::getNormalName).toList(); - private EntityType(int protoNumber) { + EntityType(int protoNumber) { this.protoNumber = protoNumber; + this.tableName = name().toLowerCase(); } - public static final List NORMAL_NAMES = EnumSet.allOf(EntityType.class).stream() - .map(EntityType::getNormalName).collect(Collectors.toUnmodifiableList()); - - @Getter - private final String normalName = StringUtils.capitalize(StringUtils.removeStart(name(), "TB_") - .toLowerCase().replaceAll("_", " ")); + EntityType(int protoNumber, String tableName) { + this.protoNumber = protoNumber; + this.tableName = tableName; + } } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/housekeeper/AlarmsDeletionHousekeeperTask.java b/common/data/src/main/java/org/thingsboard/server/common/data/housekeeper/AlarmsDeletionHousekeeperTask.java new file mode 100644 index 0000000000..60a50916ef --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/housekeeper/AlarmsDeletionHousekeeperTask.java @@ -0,0 +1,51 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.housekeeper; + +import lombok.AccessLevel; +import lombok.Data; +import lombok.EqualsAndHashCode; +import lombok.NoArgsConstructor; +import lombok.ToString; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.TenantId; + +import java.util.List; +import java.util.UUID; + +@Data +@ToString(callSuper = true) +@EqualsAndHashCode(callSuper = true) +@NoArgsConstructor(access = AccessLevel.PROTECTED) +public class AlarmsDeletionHousekeeperTask extends HousekeeperTask { + + private List alarms; + + public AlarmsDeletionHousekeeperTask(TenantId tenantId, EntityId entityId) { + this(tenantId, entityId, null); + } + + public AlarmsDeletionHousekeeperTask(TenantId tenantId, EntityId entityId, List alarms) { + super(tenantId, entityId, HousekeeperTaskType.DELETE_ALARMS); + this.alarms = alarms; + } + + @Override + public String getDescription() { + return super.getDescription() + (alarms != null ? " (" + alarms + ")" : ""); + } + +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/housekeeper/AlarmsUnassignHousekeeperTask.java b/common/data/src/main/java/org/thingsboard/server/common/data/housekeeper/AlarmsUnassignHousekeeperTask.java new file mode 100644 index 0000000000..5911418dc3 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/housekeeper/AlarmsUnassignHousekeeperTask.java @@ -0,0 +1,38 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.housekeeper; + +import lombok.AccessLevel; +import lombok.Data; +import lombok.EqualsAndHashCode; +import lombok.NoArgsConstructor; +import lombok.ToString; +import org.thingsboard.server.common.data.User; + +@Data +@ToString(callSuper = true) +@EqualsAndHashCode(callSuper = true) +@NoArgsConstructor(access = AccessLevel.PROTECTED) +public class AlarmsUnassignHousekeeperTask extends HousekeeperTask { + + private String userTitle; + + protected AlarmsUnassignHousekeeperTask(User user) { + super(user.getTenantId(), user.getId(), HousekeeperTaskType.UNASSIGN_ALARMS); + this.userTitle = user.getTitle(); + } + +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/housekeeper/EntitiesDeletionHousekeeperTask.java b/common/data/src/main/java/org/thingsboard/server/common/data/housekeeper/EntitiesDeletionHousekeeperTask.java new file mode 100644 index 0000000000..b98af48d42 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/housekeeper/EntitiesDeletionHousekeeperTask.java @@ -0,0 +1,50 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.housekeeper; + +import com.fasterxml.jackson.annotation.JsonIgnore; +import lombok.Data; +import lombok.EqualsAndHashCode; +import lombok.NoArgsConstructor; +import lombok.ToString; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.id.TenantId; + +import java.util.List; +import java.util.UUID; + +@Data +@ToString(callSuper = true) +@EqualsAndHashCode(callSuper = true) +@NoArgsConstructor +public class EntitiesDeletionHousekeeperTask extends HousekeeperTask { + + private EntityType entityType; + private List entities; + + public EntitiesDeletionHousekeeperTask(TenantId tenantId, EntityType entityType, List entities) { + super(tenantId, tenantId, HousekeeperTaskType.DELETE_ENTITIES); + this.entityType = entityType; + this.entities = entities; + } + + @JsonIgnore + @Override + public String getDescription() { + return entityType.getNormalName().toLowerCase() + "s deletion (" + entities + ")"; + } + +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/housekeeper/HousekeeperTask.java b/common/data/src/main/java/org/thingsboard/server/common/data/housekeeper/HousekeeperTask.java new file mode 100644 index 0000000000..8757b1121e --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/housekeeper/HousekeeperTask.java @@ -0,0 +1,89 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.housekeeper; + +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonSubTypes; +import com.fasterxml.jackson.annotation.JsonSubTypes.Type; +import com.fasterxml.jackson.annotation.JsonTypeInfo; +import lombok.AccessLevel; +import lombok.Data; +import lombok.NoArgsConstructor; +import lombok.NonNull; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.User; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.TenantId; + +import java.io.Serializable; + +@JsonIgnoreProperties(ignoreUnknown = true) +@JsonTypeInfo(use = JsonTypeInfo.Id.NAME, property = "taskType", visible = true, include = JsonTypeInfo.As.EXISTING_PROPERTY, defaultImpl = HousekeeperTask.class) +@JsonSubTypes({ + @Type(name = "DELETE_TS_HISTORY", value = TsHistoryDeletionHousekeeperTask.class), + @Type(name = "DELETE_LATEST_TS", value = LatestTsDeletionHousekeeperTask.class), + @Type(name = "DELETE_TENANT_ENTITIES", value = TenantEntitiesDeletionHousekeeperTask.class), + @Type(name = "DELETE_ENTITIES", value = EntitiesDeletionHousekeeperTask.class), + @Type(name = "DELETE_ALARMS", value = AlarmsDeletionHousekeeperTask.class), + @Type(name = "UNASSIGN_ALARMS", value = AlarmsUnassignHousekeeperTask.class) +}) +@Data +@NoArgsConstructor(access = AccessLevel.PROTECTED) +public class HousekeeperTask implements Serializable { + + private TenantId tenantId; + private EntityId entityId; + private HousekeeperTaskType taskType; + private long ts; + + protected HousekeeperTask(@NonNull TenantId tenantId, @NonNull EntityId entityId, @NonNull HousekeeperTaskType taskType) { + this.tenantId = tenantId; + this.entityId = entityId; + this.taskType = taskType; + this.ts = System.currentTimeMillis(); + } + + public static HousekeeperTask deleteAttributes(TenantId tenantId, EntityId entityId) { + return new HousekeeperTask(tenantId, entityId, HousekeeperTaskType.DELETE_ATTRIBUTES); + } + + public static HousekeeperTask deleteTelemetry(TenantId tenantId, EntityId entityId) { + return new HousekeeperTask(tenantId, entityId, HousekeeperTaskType.DELETE_TELEMETRY); + } + + public static HousekeeperTask deleteEvents(TenantId tenantId, EntityId entityId) { + return new HousekeeperTask(tenantId, entityId, HousekeeperTaskType.DELETE_EVENTS); + } + + public static HousekeeperTask unassignAlarms(User user) { + return new AlarmsUnassignHousekeeperTask(user); + } + + public static HousekeeperTask deleteAlarms(TenantId tenantId, EntityId entityId) { + return new AlarmsDeletionHousekeeperTask(tenantId, entityId); + } + + public static HousekeeperTask deleteTenantEntities(TenantId tenantId, EntityType entityType) { + return new TenantEntitiesDeletionHousekeeperTask(tenantId, entityType); + } + + @JsonIgnore + public String getDescription() { + return taskType.getDescription() + " for " + entityId.getEntityType().getNormalName().toLowerCase() + " " + entityId.getId(); + } + +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/housekeeper/HousekeeperTaskType.java b/common/data/src/main/java/org/thingsboard/server/common/data/housekeeper/HousekeeperTaskType.java new file mode 100644 index 0000000000..edd13b3d5d --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/housekeeper/HousekeeperTaskType.java @@ -0,0 +1,37 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.housekeeper; + +import lombok.Getter; +import lombok.RequiredArgsConstructor; + +@RequiredArgsConstructor +@Getter +public enum HousekeeperTaskType { + + DELETE_ATTRIBUTES("attributes deletion"), + DELETE_TELEMETRY("telemetry deletion"), + DELETE_LATEST_TS("latest telemetry deletion"), + DELETE_TS_HISTORY("timeseries history deletion"), + DELETE_EVENTS("events deletion"), + DELETE_ALARMS("alarms deletion"), + UNASSIGN_ALARMS("alarms unassigning"), + DELETE_TENANT_ENTITIES("tenant entities deletion"), + DELETE_ENTITIES("entities deletion"); + + private final String description; + +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/housekeeper/LatestTsDeletionHousekeeperTask.java b/common/data/src/main/java/org/thingsboard/server/common/data/housekeeper/LatestTsDeletionHousekeeperTask.java new file mode 100644 index 0000000000..4f0f56479e --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/housekeeper/LatestTsDeletionHousekeeperTask.java @@ -0,0 +1,44 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.housekeeper; + +import lombok.AccessLevel; +import lombok.Data; +import lombok.EqualsAndHashCode; +import lombok.NoArgsConstructor; +import lombok.ToString; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.TenantId; + +@Data +@ToString(callSuper = true) +@EqualsAndHashCode(callSuper = true) +@NoArgsConstructor(access = AccessLevel.PROTECTED) +public class LatestTsDeletionHousekeeperTask extends HousekeeperTask { + + private String key; + + public LatestTsDeletionHousekeeperTask(TenantId tenantId, EntityId entityId, String key) { + super(tenantId, entityId, HousekeeperTaskType.DELETE_LATEST_TS); + this.key = key; + } + + @Override + public String getDescription() { + return super.getDescription() + (key != null ? " for key '" + key + "'" : ""); + } + +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/housekeeper/TenantEntitiesDeletionHousekeeperTask.java b/common/data/src/main/java/org/thingsboard/server/common/data/housekeeper/TenantEntitiesDeletionHousekeeperTask.java new file mode 100644 index 0000000000..f6c6d5030c --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/housekeeper/TenantEntitiesDeletionHousekeeperTask.java @@ -0,0 +1,45 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.housekeeper; + +import com.fasterxml.jackson.annotation.JsonIgnore; +import lombok.Data; +import lombok.EqualsAndHashCode; +import lombok.NoArgsConstructor; +import lombok.ToString; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.id.TenantId; + +@Data +@ToString(callSuper = true) +@EqualsAndHashCode(callSuper = true) +@NoArgsConstructor +public class TenantEntitiesDeletionHousekeeperTask extends HousekeeperTask { + + private EntityType entityType; + + public TenantEntitiesDeletionHousekeeperTask(TenantId tenantId, EntityType entityType) { + super(tenantId, tenantId, HousekeeperTaskType.DELETE_TENANT_ENTITIES); + this.entityType = entityType; + } + + @JsonIgnore + @Override + public String getDescription() { + return entityType.getNormalName().toLowerCase() + "s deletion"; + } + +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/housekeeper/TsHistoryDeletionHousekeeperTask.java b/common/data/src/main/java/org/thingsboard/server/common/data/housekeeper/TsHistoryDeletionHousekeeperTask.java new file mode 100644 index 0000000000..8f462f2dff --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/housekeeper/TsHistoryDeletionHousekeeperTask.java @@ -0,0 +1,44 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.housekeeper; + +import lombok.AccessLevel; +import lombok.Data; +import lombok.EqualsAndHashCode; +import lombok.NoArgsConstructor; +import lombok.ToString; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.TenantId; + +@Data +@ToString(callSuper = true) +@EqualsAndHashCode(callSuper = true) +@NoArgsConstructor(access = AccessLevel.PROTECTED) +public class TsHistoryDeletionHousekeeperTask extends HousekeeperTask { + + private String key; + + public TsHistoryDeletionHousekeeperTask(TenantId tenantId, EntityId entityId, String key) { + super(tenantId, entityId, HousekeeperTaskType.DELETE_TS_HISTORY); + this.key = key; + } + + @Override + public String getDescription() { + return super.getDescription() + (key != null ? " for key '" + key + "'" : ""); + } + +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/notification/NotificationType.java b/common/data/src/main/java/org/thingsboard/server/common/data/notification/NotificationType.java index 5a8fd87933..5f91725b40 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/notification/NotificationType.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/notification/NotificationType.java @@ -30,6 +30,6 @@ public enum NotificationType { RULE_NODE, RATE_LIMITS, EDGE_CONNECTION, - EDGE_COMMUNICATION_FAILURE - + EDGE_COMMUNICATION_FAILURE, + TASK_PROCESSING_FAILURE } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/notification/info/TaskProcessingFailureNotificationInfo.java b/common/data/src/main/java/org/thingsboard/server/common/data/notification/info/TaskProcessingFailureNotificationInfo.java new file mode 100644 index 0000000000..cbee861c04 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/notification/info/TaskProcessingFailureNotificationInfo.java @@ -0,0 +1,61 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.notification.info; + +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; +import org.thingsboard.server.common.data.housekeeper.HousekeeperTaskType; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.TenantId; + +import java.util.Map; + +import static org.thingsboard.server.common.data.util.CollectionsUtil.mapOf; + +@Data +@NoArgsConstructor +@AllArgsConstructor +@Builder +public class TaskProcessingFailureNotificationInfo implements RuleOriginatedNotificationInfo { + + private TenantId tenantId; + private EntityId entityId; + private HousekeeperTaskType taskType; + private String taskDescription; + private String error; + private int attempt; + + @Override + public Map getTemplateData() { + return mapOf( + "tenantId", tenantId.toString(), + "entityType", entityId.getEntityType().getNormalName(), + "entityId", entityId.getId().toString(), + "taskType", taskType.getDescription(), + "taskDescription", taskDescription, + "error", error, + "attempt", String.valueOf(attempt) + ); + } + + @Override + public TenantId getAffectedTenantId() { + return tenantId; + } + +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/notification/rule/trigger/TaskProcessingFailureTrigger.java b/common/data/src/main/java/org/thingsboard/server/common/data/notification/rule/trigger/TaskProcessingFailureTrigger.java new file mode 100644 index 0000000000..3626e09c0b --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/notification/rule/trigger/TaskProcessingFailureTrigger.java @@ -0,0 +1,53 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.notification.rule.trigger; + +import lombok.Builder; +import lombok.Data; +import org.thingsboard.server.common.data.housekeeper.HousekeeperTask; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.notification.rule.trigger.config.NotificationRuleTriggerType; + +@Data +@Builder +public class TaskProcessingFailureTrigger implements NotificationRuleTrigger { + + private final HousekeeperTask task; + private final int attempt; + private final Throwable error; + + @Override + public NotificationRuleTriggerType getType() { + return NotificationRuleTriggerType.TASK_PROCESSING_FAILURE; + } + + @Override + public TenantId getTenantId() { + return task.getTenantId(); + } + + @Override + public EntityId getOriginatorEntityId() { + return task.getEntityId(); + } + + @Override + public boolean deduplicate() { + return false; + } + +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/notification/rule/trigger/config/NotificationRuleTriggerConfig.java b/common/data/src/main/java/org/thingsboard/server/common/data/notification/rule/trigger/config/NotificationRuleTriggerConfig.java index 15a5e59255..1621ec8120 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/notification/rule/trigger/config/NotificationRuleTriggerConfig.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/notification/rule/trigger/config/NotificationRuleTriggerConfig.java @@ -38,6 +38,7 @@ import java.io.Serializable; @Type(value = RateLimitsNotificationRuleTriggerConfig.class, name = "RATE_LIMITS"), @Type(value = EdgeConnectionNotificationRuleTriggerConfig.class, name = "EDGE_CONNECTION"), @Type(value = EdgeCommunicationFailureNotificationRuleTriggerConfig.class, name = "EDGE_COMMUNICATION_FAILURE"), + @Type(value = TaskProcessingFailureNotificationRuleTriggerConfig.class, name = "TASK_PROCESSING_FAILURE") }) public interface NotificationRuleTriggerConfig extends Serializable { diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/notification/rule/trigger/config/NotificationRuleTriggerType.java b/common/data/src/main/java/org/thingsboard/server/common/data/notification/rule/trigger/config/NotificationRuleTriggerType.java index 8469fac752..8846ae284d 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/notification/rule/trigger/config/NotificationRuleTriggerType.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/notification/rule/trigger/config/NotificationRuleTriggerType.java @@ -31,7 +31,8 @@ public enum NotificationRuleTriggerType { NEW_PLATFORM_VERSION(false), ENTITIES_LIMIT(false), API_USAGE_LIMIT(false), - RATE_LIMITS(false); + RATE_LIMITS(false), + TASK_PROCESSING_FAILURE(false); private final boolean tenantLevel; diff --git a/common/dao-api/src/main/java/org/thingsboard/server/dao/housekeeper/HouseKeeperService.java b/common/data/src/main/java/org/thingsboard/server/common/data/notification/rule/trigger/config/TaskProcessingFailureNotificationRuleTriggerConfig.java similarity index 58% rename from common/dao-api/src/main/java/org/thingsboard/server/dao/housekeeper/HouseKeeperService.java rename to common/data/src/main/java/org/thingsboard/server/common/data/notification/rule/trigger/config/TaskProcessingFailureNotificationRuleTriggerConfig.java index 55b164df52..73f0517238 100644 --- a/common/dao-api/src/main/java/org/thingsboard/server/dao/housekeeper/HouseKeeperService.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/notification/rule/trigger/config/TaskProcessingFailureNotificationRuleTriggerConfig.java @@ -13,17 +13,18 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.dao.housekeeper; +package org.thingsboard.server.common.data.notification.rule.trigger.config; -import com.google.common.util.concurrent.ListenableFuture; -import org.thingsboard.server.common.data.User; -import org.thingsboard.server.common.data.id.AlarmId; -import org.thingsboard.server.common.data.id.TenantId; +import lombok.Builder; +import lombok.Data; -import java.util.List; +@Data +@Builder +public class TaskProcessingFailureNotificationRuleTriggerConfig implements NotificationRuleTriggerConfig { -public interface HouseKeeperService { - - ListenableFuture> unassignDeletedUserAlarms(TenantId tenantId, User user, long unassignTs); + @Override + public NotificationRuleTriggerType getTriggerType() { + return NotificationRuleTriggerType.TASK_PROCESSING_FAILURE; + } } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/page/PageData.java b/common/data/src/main/java/org/thingsboard/server/common/data/page/PageData.java index 49824c524d..3df123d870 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/page/PageData.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/page/PageData.java @@ -19,6 +19,7 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; import io.swagger.v3.oas.annotations.media.Schema; import lombok.EqualsAndHashCode; +import lombok.ToString; import java.io.Serializable; import java.util.Collections; @@ -28,6 +29,7 @@ import java.util.stream.Collectors; @Schema @EqualsAndHashCode +@ToString public class PageData implements Serializable { public static final PageData EMPTY_PAGE_DATA = new PageData<>(); diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/page/PageLink.java b/common/data/src/main/java/org/thingsboard/server/common/data/page/PageLink.java index 042a9b1896..d6a3b5ad7d 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/page/PageLink.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/page/PageLink.java @@ -63,10 +63,10 @@ public class PageLink { @JsonIgnore public PageLink nextPageLink() { - return new PageLink(this.pageSize, this.page+1, this.textSearch, this.sortOrder); + return new PageLink(this.pageSize, this.page + 1, this.textSearch, this.sortOrder); } - public Sort toSort(SortOrder sortOrder, Map columnMap, boolean addDefaultSorting) { + public Sort toSort(SortOrder sortOrder, Map columnMap, boolean addDefaultSorting) { if (sortOrder == null) { return DEFAULT_SORT; } else { @@ -74,7 +74,7 @@ public class PageLink { } } - public Sort toSort(List sortOrders, Map columnMap, boolean addDefaultSorting) { + public Sort toSort(List sortOrders, Map columnMap, boolean addDefaultSorting) { if (addDefaultSorting && !isDefaultSortOrderAvailable(sortOrders)) { sortOrders = new ArrayList<>(sortOrders); sortOrders.add(new SortOrder(DEFAULT_SORT_PROPERTY, SortOrder.Direction.ASC)); @@ -82,7 +82,7 @@ public class PageLink { return Sort.by(sortOrders.stream().map(s -> toSortOrder(s, columnMap)).collect(Collectors.toList())); } - private Sort.Order toSortOrder(SortOrder sortOrder, Map columnMap) { + private Sort.Order toSortOrder(SortOrder sortOrder, Map columnMap) { String property = sortOrder.getProperty(); if (columnMap.containsKey(property)) { property = columnMap.get(property); diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/page/SortOrder.java b/common/data/src/main/java/org/thingsboard/server/common/data/page/SortOrder.java index 5cafed6df2..539dd1cb40 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/page/SortOrder.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/page/SortOrder.java @@ -32,6 +32,10 @@ public class SortOrder { this.direction = direction; } + public static SortOrder of(String property, Direction direction) { + return new SortOrder(property, direction); + } + public static enum Direction { ASC, DESC } diff --git a/common/message/src/main/java/org/thingsboard/server/common/msg/housekeeper/HousekeeperClient.java b/common/message/src/main/java/org/thingsboard/server/common/msg/housekeeper/HousekeeperClient.java new file mode 100644 index 0000000000..e2a3af9242 --- /dev/null +++ b/common/message/src/main/java/org/thingsboard/server/common/msg/housekeeper/HousekeeperClient.java @@ -0,0 +1,24 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.msg.housekeeper; + +import org.thingsboard.server.common.data.housekeeper.HousekeeperTask; + +public interface HousekeeperClient { + + void submitTask(HousekeeperTask task); + +} diff --git a/common/proto/src/main/proto/queue.proto b/common/proto/src/main/proto/queue.proto index 44f0b0caf0..7b2aa2fe1b 100644 --- a/common/proto/src/main/proto/queue.proto +++ b/common/proto/src/main/proto/queue.proto @@ -1562,3 +1562,16 @@ message LifecycleEventProto { bool success = 7; string error = 8; } + +message ToHousekeeperServiceMsg { + HousekeeperTaskProto task = 1; +} + +message HousekeeperTaskProto { + string value = 1; + int64 ts = 2; + + // reprocessing + int32 attempt = 50; + repeated string errors = 51; +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/housekeeper/DefaultHousekeeperClient.java b/common/queue/src/main/java/org/thingsboard/server/queue/housekeeper/DefaultHousekeeperClient.java new file mode 100644 index 0000000000..ce5311b0e7 --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/housekeeper/DefaultHousekeeperClient.java @@ -0,0 +1,86 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.housekeeper; + +import lombok.extern.slf4j.Slf4j; +import org.springframework.stereotype.Service; +import org.thingsboard.common.util.JacksonUtil; +import org.thingsboard.server.common.data.housekeeper.HousekeeperTask; +import org.thingsboard.server.common.data.housekeeper.HousekeeperTaskType; +import org.thingsboard.server.common.msg.housekeeper.HousekeeperClient; +import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; +import org.thingsboard.server.gen.transport.TransportProtos; +import org.thingsboard.server.gen.transport.TransportProtos.ToHousekeeperServiceMsg; +import org.thingsboard.server.queue.TbQueueCallback; +import org.thingsboard.server.queue.TbQueueMsgMetadata; +import org.thingsboard.server.queue.TbQueueProducer; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; +import org.thingsboard.server.queue.provider.TbQueueProducerProvider; + +@Service +@Slf4j +public class DefaultHousekeeperClient implements HousekeeperClient { + + private final HousekeeperConfig config; + private final TbQueueProducer> producer; + private final TopicPartitionInfo submitTpi; + private final TbQueueCallback submitCallback; + + public DefaultHousekeeperClient(HousekeeperConfig config, + TbQueueProducerProvider producerProvider) { + this.config = config; + this.producer = producerProvider.getHousekeeperMsgProducer(); + this.submitTpi = TopicPartitionInfo.builder().topic(producer.getDefaultTopic()).build(); + this.submitCallback = new TbQueueCallback() { + @Override + public void onSuccess(TbQueueMsgMetadata metadata) { + log.trace("Submitted Housekeeper task"); + } + + @Override + public void onFailure(Throwable t) { + log.error("Failed to submit Housekeeper task", t); + } + }; + } + + @Override + public void submitTask(HousekeeperTask task) { + HousekeeperTaskType taskType = task.getTaskType(); + if (config.getDisabledTaskTypes().contains(taskType)) { + log.trace("Task type {} is disabled, ignoring {}", taskType, task); + return; + } + + log.debug("[{}][{}][{}] Submitting task: {}", task.getTenantId(), task.getEntityId().getEntityType(), task.getEntityId(), task); + /* + * using msg key as entity id so that msgs related to certain entity are pushed to same partition, + * e.g. on tenant deletion (entity id is tenant id), we need to clean up tenant entities in certain order + * */ + try { + producer.send(submitTpi, new TbProtoQueueMsg<>(task.getEntityId().getId(), ToHousekeeperServiceMsg.newBuilder() + .setTask(TransportProtos.HousekeeperTaskProto.newBuilder() + .setValue(JacksonUtil.toString(task)) + .setTs(task.getTs()) + .setAttempt(0) + .build()) + .build()), submitCallback); + } catch (Throwable t) { + log.error("Failed to submit Housekeeper task {}", task, t); + } + } + +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/housekeeper/HousekeeperConfig.java b/common/queue/src/main/java/org/thingsboard/server/queue/housekeeper/HousekeeperConfig.java new file mode 100644 index 0000000000..4ca3776d4d --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/housekeeper/HousekeeperConfig.java @@ -0,0 +1,40 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.housekeeper; + +import lombok.Getter; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Component; +import org.thingsboard.server.common.data.housekeeper.HousekeeperTaskType; + +import java.util.Set; + +@Component +@Getter +public class HousekeeperConfig { + + @Value("${queue.core.housekeeper.disabled-task-types:}") + private Set disabledTaskTypes; + @Value("${queue.core.housekeeper.task-processing-timeout-ms:120000}") + private int taskProcessingTimeout; + @Value("${queue.core.housekeeper.poll-interval-ms:500}") + private int pollInterval; + @Value("${queue.core.housekeeper.task-reprocessing-delay-ms:3000}") + private int taskReprocessingDelay; + @Value("${queue.core.housekeeper.max-reprocessing-attempts:10}") + private int maxReprocessingAttempts; + +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaTopicConfigs.java b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaTopicConfigs.java index dbc6d3dd82..c55d8a5b8a 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaTopicConfigs.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaTopicConfigs.java @@ -43,6 +43,10 @@ public class TbKafkaTopicConfigs { private String fwUpdatesProperties; @Value("${queue.kafka.topic-properties.version-control:}") private String vcProperties; + @Value("${queue.kafka.topic-properties.housekeeper:}") + private String housekeeperProperties; + @Value("${queue.kafka.topic-properties.housekeeper-reprocessing:}") + private String housekeeperReprocessingProperties; @Getter private Map coreConfigs; @@ -62,6 +66,10 @@ public class TbKafkaTopicConfigs { private Map fwUpdatesConfigs; @Getter private Map vcConfigs; + @Getter + private Map housekeeperConfigs; + @Getter + private Map housekeeperReprocessingConfigs; @PostConstruct private void init() { @@ -76,6 +84,8 @@ public class TbKafkaTopicConfigs { jsExecutorResponseConfigs.put(NUM_PARTITIONS_SETTING, "1"); fwUpdatesConfigs = PropertyUtils.getProps(fwUpdatesProperties); vcConfigs = PropertyUtils.getProps(vcProperties); + housekeeperConfigs = PropertyUtils.getProps(housekeeperProperties); + housekeeperReprocessingConfigs = PropertyUtils.getProps(housekeeperReprocessingProperties); } } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/memory/DefaultInMemoryStorage.java b/common/queue/src/main/java/org/thingsboard/server/queue/memory/DefaultInMemoryStorage.java index 9c7dc02b4f..f46a05ca26 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/memory/DefaultInMemoryStorage.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/memory/DefaultInMemoryStorage.java @@ -20,8 +20,10 @@ import org.springframework.stereotype.Component; import org.thingsboard.server.queue.TbQueueMsg; import java.util.ArrayList; +import java.util.Collection; import java.util.Collections; import java.util.List; +import java.util.Optional; import java.util.concurrent.BlockingQueue; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.LinkedBlockingQueue; @@ -47,6 +49,11 @@ public final class DefaultInMemoryStorage implements InMemoryStorage { return storage.values().stream().map(BlockingQueue::size).reduce(0, Integer::sum); } + @Override + public int getLag(String topic) { + return Optional.ofNullable(storage.get(topic)).map(Collection::size).orElse(0); + } + @Override public boolean put(String topic, TbQueueMsg msg) { return storage.computeIfAbsent(topic, (t) -> new LinkedBlockingQueue<>()).add(msg); diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/memory/InMemoryStorage.java b/common/queue/src/main/java/org/thingsboard/server/queue/memory/InMemoryStorage.java index 0e753d1a81..81b174a0d3 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/memory/InMemoryStorage.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/memory/InMemoryStorage.java @@ -25,6 +25,8 @@ public interface InMemoryStorage { int getLagTotal(); + int getLag(String topic); + boolean put(String topic, TbQueueMsg msg); List get(String topic) throws InterruptedException; diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsMonolithQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsMonolithQueueFactory.java index ed6f6c2192..26ec51097f 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsMonolithQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsMonolithQueueFactory.java @@ -26,6 +26,7 @@ import org.thingsboard.server.gen.js.JsInvokeProtos.RemoteJsResponse; import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToHousekeeperServiceMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToOtaPackageStateServiceMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineNotificationMsg; @@ -226,6 +227,28 @@ public class AwsSqsMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEng return new TbAwsSqsProducerTemplate<>(vcAdmin, sqsSettings, topicService.buildTopicName(vcSettings.getTopic())); } + @Override + public TbQueueProducer> createHousekeeperMsgProducer() { + return new TbAwsSqsProducerTemplate<>(coreAdmin, sqsSettings, topicService.buildTopicName(coreSettings.getHousekeeperTopic())); + } + + @Override + public TbQueueConsumer> createHousekeeperMsgConsumer() { + return new TbAwsSqsConsumerTemplate<>(coreAdmin, sqsSettings, topicService.buildTopicName(coreSettings.getHousekeeperTopic()), + msg -> new TbProtoQueueMsg<>(msg.getKey(), ToHousekeeperServiceMsg.parseFrom(msg.getData()), msg.getHeaders())); + } + + @Override + public TbQueueProducer> createHousekeeperReprocessingMsgProducer() { + return new TbAwsSqsProducerTemplate<>(coreAdmin, sqsSettings, topicService.buildTopicName(coreSettings.getHousekeeperReprocessingTopic())); + } + + @Override + public TbQueueConsumer> createHousekeeperReprocessingMsgConsumer() { + return new TbAwsSqsConsumerTemplate<>(coreAdmin, sqsSettings, topicService.buildTopicName(coreSettings.getHousekeeperReprocessingTopic()), + msg -> new TbProtoQueueMsg<>(msg.getKey(), ToHousekeeperServiceMsg.parseFrom(msg.getData()), msg.getHeaders())); + } + @PreDestroy private void destroy() { if (coreAdmin != null) { diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTbCoreQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTbCoreQueueFactory.java index 73d91c0272..0c4fe824fa 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTbCoreQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTbCoreQueueFactory.java @@ -24,6 +24,7 @@ import org.thingsboard.server.gen.js.JsInvokeProtos; import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToHousekeeperServiceMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToOtaPackageStateServiceMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineNotificationMsg; @@ -205,6 +206,28 @@ public class AwsSqsTbCoreQueueFactory implements TbCoreQueueFactory { return new TbAwsSqsProducerTemplate<>(vcAdmin, sqsSettings, topicService.buildTopicName(vcSettings.getTopic())); } + @Override + public TbQueueProducer> createHousekeeperMsgProducer() { + return new TbAwsSqsProducerTemplate<>(coreAdmin, sqsSettings, topicService.buildTopicName(coreSettings.getHousekeeperTopic())); + } + + @Override + public TbQueueConsumer> createHousekeeperMsgConsumer() { + return new TbAwsSqsConsumerTemplate<>(coreAdmin, sqsSettings, topicService.buildTopicName(coreSettings.getHousekeeperTopic()), + msg -> new TbProtoQueueMsg<>(msg.getKey(), ToHousekeeperServiceMsg.parseFrom(msg.getData()), msg.getHeaders())); + } + + @Override + public TbQueueProducer> createHousekeeperReprocessingMsgProducer() { + return new TbAwsSqsProducerTemplate<>(coreAdmin, sqsSettings, topicService.buildTopicName(coreSettings.getHousekeeperReprocessingTopic())); + } + + @Override + public TbQueueConsumer> createHousekeeperReprocessingMsgConsumer() { + return new TbAwsSqsConsumerTemplate<>(coreAdmin, sqsSettings, topicService.buildTopicName(coreSettings.getHousekeeperReprocessingTopic()), + msg -> new TbProtoQueueMsg<>(msg.getKey(), ToHousekeeperServiceMsg.parseFrom(msg.getData()), msg.getHeaders())); + } + @PreDestroy private void destroy() { if (coreAdmin != null) { diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTbRuleEngineQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTbRuleEngineQueueFactory.java index 5477f6981f..000578333a 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTbRuleEngineQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTbRuleEngineQueueFactory.java @@ -33,8 +33,8 @@ import org.thingsboard.server.queue.TbQueueRequestTemplate; import org.thingsboard.server.queue.common.DefaultTbQueueRequestTemplate; import org.thingsboard.server.queue.common.TbProtoJsQueueMsg; import org.thingsboard.server.queue.common.TbProtoQueueMsg; -import org.thingsboard.server.queue.discovery.TopicService; import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; +import org.thingsboard.server.queue.discovery.TopicService; import org.thingsboard.server.queue.settings.TbQueueCoreSettings; import org.thingsboard.server.queue.settings.TbQueueRemoteJsInvokeSettings; import org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings; @@ -159,6 +159,10 @@ public class AwsSqsTbRuleEngineQueueFactory implements TbRuleEngineQueueFactory return new TbAwsSqsProducerTemplate<>(otaAdmin, sqsSettings, topicService.buildTopicName(coreSettings.getOtaPackageTopic())); } + @Override + public TbQueueProducer> createHousekeeperMsgProducer() { + return new TbAwsSqsProducerTemplate<>(coreAdmin, sqsSettings, topicService.buildTopicName(coreSettings.getHousekeeperTopic())); + } @PreDestroy private void destroy() { diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTbVersionControlQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTbVersionControlQueueFactory.java index 73f2142718..aa79349dac 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTbVersionControlQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTbVersionControlQueueFactory.java @@ -80,6 +80,11 @@ public class AwsSqsTbVersionControlQueueFactory implements TbVersionControlQueue ); } + @Override + public TbQueueProducer> createHousekeeperMsgProducer() { + return new TbAwsSqsProducerTemplate<>(coreAdmin, sqsSettings, topicService.buildTopicName(coreSettings.getHousekeeperTopic())); + } + @PreDestroy private void destroy() { if (coreAdmin != null) { diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTransportQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTransportQueueFactory.java index 582c6a7796..ec07847631 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTransportQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTransportQueueFactory.java @@ -131,6 +131,11 @@ public class AwsSqsTransportQueueFactory implements TbTransportQueueFactory { return new TbAwsSqsProducerTemplate<>(coreAdmin, sqsSettings, topicService.buildTopicName(coreSettings.getUsageStatsTopic())); } + @Override + public TbQueueProducer> createHousekeeperMsgProducer() { + return new TbAwsSqsProducerTemplate<>(coreAdmin, sqsSettings, topicService.buildTopicName(coreSettings.getHousekeeperTopic())); + } + @PreDestroy private void destroy() { if (coreAdmin != null) { diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/HousekeeperClientQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/HousekeeperClientQueueFactory.java new file mode 100644 index 0000000000..79c38a04e9 --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/HousekeeperClientQueueFactory.java @@ -0,0 +1,26 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.provider; + +import org.thingsboard.server.gen.transport.TransportProtos.ToHousekeeperServiceMsg; +import org.thingsboard.server.queue.TbQueueProducer; +import org.thingsboard.server.queue.common.TbProtoQueueMsg; + +public interface HousekeeperClientQueueFactory { + + TbQueueProducer> createHousekeeperMsgProducer(); + +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/InMemoryMonolithQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/InMemoryMonolithQueueFactory.java index a681038aa9..310bc75fe4 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/InMemoryMonolithQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/InMemoryMonolithQueueFactory.java @@ -160,6 +160,26 @@ public class InMemoryMonolithQueueFactory implements TbCoreQueueFactory, TbRuleE return new InMemoryTbQueueProducer<>(storage, topicService.buildTopicName(vcSettings.getTopic())); } + @Override + public TbQueueProducer> createHousekeeperMsgProducer() { + return new InMemoryTbQueueProducer<>(storage, topicService.buildTopicName(coreSettings.getHousekeeperTopic())); + } + + @Override + public TbQueueConsumer> createHousekeeperMsgConsumer() { + return new InMemoryTbQueueConsumer<>(storage, topicService.buildTopicName(coreSettings.getHousekeeperTopic())); + } + + @Override + public TbQueueProducer> createHousekeeperReprocessingMsgProducer() { + return new InMemoryTbQueueProducer<>(storage, topicService.buildTopicName(coreSettings.getHousekeeperReprocessingTopic())); + } + + @Override + public TbQueueConsumer> createHousekeeperReprocessingMsgConsumer() { + return new InMemoryTbQueueConsumer<>(storage, topicService.buildTopicName(coreSettings.getHousekeeperReprocessingTopic())); + } + @Scheduled(fixedRateString = "${queue.in_memory.stats.print-interval-ms:60000}") private void printInMemoryStats() { storage.printStats(); diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/InMemoryTbTransportQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/InMemoryTbTransportQueueFactory.java index f08a822466..4ee79aef60 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/InMemoryTbTransportQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/InMemoryTbTransportQueueFactory.java @@ -120,4 +120,9 @@ public class InMemoryTbTransportQueueFactory implements TbTransportQueueFactory return new InMemoryTbQueueProducer<>(storage, topicService.buildTopicName(coreSettings.getUsageStatsTopic())); } + @Override + public TbQueueProducer> createHousekeeperMsgProducer() { + return new InMemoryTbQueueProducer<>(storage, topicService.buildTopicName(coreSettings.getHousekeeperTopic())); + } + } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaMonolithQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaMonolithQueueFactory.java index b6602abe63..cd09f682c6 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaMonolithQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaMonolithQueueFactory.java @@ -25,6 +25,7 @@ import org.thingsboard.server.gen.js.JsInvokeProtos; import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToHousekeeperServiceMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToOtaPackageStateServiceMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineNotificationMsg; @@ -39,8 +40,8 @@ import org.thingsboard.server.queue.TbQueueRequestTemplate; import org.thingsboard.server.queue.common.DefaultTbQueueRequestTemplate; import org.thingsboard.server.queue.common.TbProtoJsQueueMsg; import org.thingsboard.server.queue.common.TbProtoQueueMsg; -import org.thingsboard.server.queue.discovery.TopicService; import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; +import org.thingsboard.server.queue.discovery.TopicService; import org.thingsboard.server.queue.kafka.TbKafkaAdmin; import org.thingsboard.server.queue.kafka.TbKafkaConsumerStatsService; import org.thingsboard.server.queue.kafka.TbKafkaConsumerTemplate; @@ -82,6 +83,8 @@ public class KafkaMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEngi private final TbQueueAdmin notificationAdmin; private final TbQueueAdmin fwUpdatesAdmin; private final TbQueueAdmin vcAdmin; + private final TbQueueAdmin housekeeperAdmin; + private final TbQueueAdmin housekeeperReprocessingAdmin; private final AtomicLong consumerCount = new AtomicLong(); @@ -115,6 +118,8 @@ public class KafkaMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEngi this.notificationAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getNotificationsConfigs()); this.fwUpdatesAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getFwUpdatesConfigs()); this.vcAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getVcConfigs()); + this.housekeeperAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getHousekeeperConfigs()); + this.housekeeperReprocessingAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getHousekeeperReprocessingConfigs()); } @Override @@ -346,6 +351,52 @@ public class KafkaMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEngi return requestBuilder.build(); } + @Override + public TbQueueProducer> createHousekeeperMsgProducer() { + return TbKafkaProducerTemplate.>builder() + .settings(kafkaSettings) + .clientId("monolith-housekeeper-producer-" + serviceInfoProvider.getServiceId()) + .defaultTopic(topicService.buildTopicName(coreSettings.getHousekeeperTopic())) + .admin(housekeeperAdmin) + .build(); + } + + @Override + public TbQueueConsumer> createHousekeeperMsgConsumer() { + return TbKafkaConsumerTemplate.>builder() + .settings(kafkaSettings) + .topic(topicService.buildTopicName(coreSettings.getHousekeeperTopic())) + .clientId("monolith-housekeeper-consumer-" + serviceInfoProvider.getServiceId()) + .groupId(topicService.buildTopicName("monolith-housekeeper-consumer")) + .decoder(msg -> new TbProtoQueueMsg<>(msg.getKey(), ToHousekeeperServiceMsg.parseFrom(msg.getData()), msg.getHeaders())) + .admin(housekeeperAdmin) + .statsService(consumerStatsService) + .build(); + } + + @Override + public TbQueueProducer> createHousekeeperReprocessingMsgProducer() { + return TbKafkaProducerTemplate.>builder() + .settings(kafkaSettings) + .clientId("monolith-housekeeper-reprocessing-producer-" + serviceInfoProvider.getServiceId()) + .defaultTopic(topicService.buildTopicName(coreSettings.getHousekeeperReprocessingTopic())) + .admin(housekeeperReprocessingAdmin) + .build(); + } + + @Override + public TbQueueConsumer> createHousekeeperReprocessingMsgConsumer() { + return TbKafkaConsumerTemplate.>builder() + .settings(kafkaSettings) + .topic(topicService.buildTopicName(coreSettings.getHousekeeperReprocessingTopic())) + .clientId("monolith-housekeeper-reprocessing-consumer-" + serviceInfoProvider.getServiceId()) + .groupId(topicService.buildTopicName("monolith-housekeeper-reprocessing-consumer")) + .decoder(msg -> new TbProtoQueueMsg<>(msg.getKey(), ToHousekeeperServiceMsg.parseFrom(msg.getData()), msg.getHeaders())) + .admin(housekeeperReprocessingAdmin) + .statsService(consumerStatsService) + .build(); + } + @PreDestroy private void destroy() { if (coreAdmin != null) { diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbCoreQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbCoreQueueFactory.java index a6c2b648e5..95fea30d7b 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbCoreQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbCoreQueueFactory.java @@ -23,6 +23,7 @@ import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.gen.js.JsInvokeProtos; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToHousekeeperServiceMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToOtaPackageStateServiceMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineNotificationMsg; @@ -38,8 +39,8 @@ import org.thingsboard.server.queue.TbQueueRequestTemplate; import org.thingsboard.server.queue.common.DefaultTbQueueRequestTemplate; import org.thingsboard.server.queue.common.TbProtoJsQueueMsg; import org.thingsboard.server.queue.common.TbProtoQueueMsg; -import org.thingsboard.server.queue.discovery.TopicService; import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; +import org.thingsboard.server.queue.discovery.TopicService; import org.thingsboard.server.queue.kafka.TbKafkaAdmin; import org.thingsboard.server.queue.kafka.TbKafkaConsumerStatsService; import org.thingsboard.server.queue.kafka.TbKafkaConsumerTemplate; @@ -80,6 +81,8 @@ public class KafkaTbCoreQueueFactory implements TbCoreQueueFactory { private final TbQueueAdmin notificationAdmin; private final TbQueueAdmin fwUpdatesAdmin; private final TbQueueAdmin vcAdmin; + private final TbQueueAdmin housekeeperAdmin; + private final TbQueueAdmin housekeeperReprocessingAdmin; public KafkaTbCoreQueueFactory(TopicService topicService, TbKafkaSettings kafkaSettings, @@ -112,6 +115,8 @@ public class KafkaTbCoreQueueFactory implements TbCoreQueueFactory { this.notificationAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getNotificationsConfigs()); this.fwUpdatesAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getFwUpdatesConfigs()); this.vcAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getVcConfigs()); + this.housekeeperAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getHousekeeperConfigs()); + this.housekeeperReprocessingAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getHousekeeperReprocessingConfigs()); } @Override @@ -303,6 +308,53 @@ public class KafkaTbCoreQueueFactory implements TbCoreQueueFactory { return requestBuilder.build(); } + @Override + public TbQueueProducer> createHousekeeperMsgProducer() { + return TbKafkaProducerTemplate.>builder() + .settings(kafkaSettings) + .clientId("tb-core-housekeeper-producer-" + serviceInfoProvider.getServiceId()) + .defaultTopic(topicService.buildTopicName(coreSettings.getHousekeeperTopic())) + .admin(housekeeperAdmin) + .build(); + } + + @Override + public TbQueueConsumer> createHousekeeperMsgConsumer() { + return TbKafkaConsumerTemplate.>builder() + .settings(kafkaSettings) + .topic(topicService.buildTopicName(coreSettings.getHousekeeperTopic())) + .clientId("tb-core-housekeeper-consumer-" + serviceInfoProvider.getServiceId()) + .groupId(topicService.buildTopicName("tb-core-housekeeper-consumer")) + .decoder(msg -> new TbProtoQueueMsg<>(msg.getKey(), ToHousekeeperServiceMsg.parseFrom(msg.getData()), msg.getHeaders())) + .admin(housekeeperAdmin) + .statsService(consumerStatsService) + .build(); + } + + @Override + public TbQueueProducer> createHousekeeperReprocessingMsgProducer() { + return TbKafkaProducerTemplate.>builder() + .settings(kafkaSettings) + .clientId("tb-core-housekeeper-reprocessing-producer-" + serviceInfoProvider.getServiceId()) + .defaultTopic(topicService.buildTopicName(coreSettings.getHousekeeperReprocessingTopic())) + .admin(housekeeperReprocessingAdmin) + .build(); + } + + @Override + public TbQueueConsumer> createHousekeeperReprocessingMsgConsumer() { + return TbKafkaConsumerTemplate.>builder() + .settings(kafkaSettings) + .topic(topicService.buildTopicName(coreSettings.getHousekeeperReprocessingTopic())) + .clientId("tb-core-housekeeper-reprocessing-consumer-" + serviceInfoProvider.getServiceId()) + .groupId(topicService.buildTopicName("tb-core-housekeeper-reprocessing-consumer")) + .decoder(msg -> new TbProtoQueueMsg<>(msg.getKey(), ToHousekeeperServiceMsg.parseFrom(msg.getData()), msg.getHeaders())) + .admin(housekeeperReprocessingAdmin) + .statsService(consumerStatsService) + .build(); + } + + @PreDestroy private void destroy() { if (coreAdmin != null) { diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbRuleEngineQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbRuleEngineQueueFactory.java index 798e39ca65..568380d7b5 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbRuleEngineQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbRuleEngineQueueFactory.java @@ -25,6 +25,7 @@ import org.thingsboard.server.gen.js.JsInvokeProtos; import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToHousekeeperServiceMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; @@ -72,6 +73,7 @@ public class KafkaTbRuleEngineQueueFactory implements TbRuleEngineQueueFactory { private final TbQueueAdmin jsExecutorResponseAdmin; private final TbQueueAdmin notificationAdmin; private final TbQueueAdmin fwUpdatesAdmin; + private final TbQueueAdmin housekeeperAdmin; private final AtomicLong consumerCount = new AtomicLong(); public KafkaTbRuleEngineQueueFactory(TopicService topicService, TbKafkaSettings kafkaSettings, @@ -97,6 +99,7 @@ public class KafkaTbRuleEngineQueueFactory implements TbRuleEngineQueueFactory { this.jsExecutorResponseAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getJsExecutorResponseConfigs()); this.notificationAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getNotificationsConfigs()); this.fwUpdatesAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getFwUpdatesConfigs()); + this.housekeeperAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getHousekeeperConfigs()); } @Override @@ -230,6 +233,16 @@ public class KafkaTbRuleEngineQueueFactory implements TbRuleEngineQueueFactory { return requestBuilder.build(); } + @Override + public TbQueueProducer> createHousekeeperMsgProducer() { + return TbKafkaProducerTemplate.>builder() + .settings(kafkaSettings) + .clientId("tb-rule-engine-housekeeper-producer-" + serviceInfoProvider.getServiceId()) + .defaultTopic(topicService.buildTopicName(coreSettings.getHousekeeperTopic())) + .admin(housekeeperAdmin) + .build(); + } + @PreDestroy private void destroy() { if (coreAdmin != null) { diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbTransportQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbTransportQueueFactory.java index 8102b16dbd..40904c7c65 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbTransportQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbTransportQueueFactory.java @@ -21,6 +21,7 @@ import org.springframework.stereotype.Component; import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToHousekeeperServiceMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToUsageStatsServiceMsg; @@ -66,6 +67,7 @@ public class KafkaTbTransportQueueFactory implements TbTransportQueueFactory { private final TbQueueAdmin transportApiRequestAdmin; private final TbQueueAdmin transportApiResponseAdmin; private final TbQueueAdmin notificationAdmin; + private final TbQueueAdmin housekeeperAdmin; public KafkaTbTransportQueueFactory(TbKafkaSettings kafkaSettings, TbServiceInfoProvider serviceInfoProvider, @@ -90,6 +92,7 @@ public class KafkaTbTransportQueueFactory implements TbTransportQueueFactory { this.transportApiRequestAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getTransportApiRequestConfigs()); this.transportApiResponseAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getTransportApiResponseConfigs()); this.notificationAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getNotificationsConfigs()); + this.housekeeperAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getHousekeeperConfigs()); } @Override @@ -173,6 +176,16 @@ public class KafkaTbTransportQueueFactory implements TbTransportQueueFactory { return requestBuilder.build(); } + @Override + public TbQueueProducer> createHousekeeperMsgProducer() { + return TbKafkaProducerTemplate.>builder() + .settings(kafkaSettings) + .clientId("tb-transport-housekeeper-producer-" + serviceInfoProvider.getServiceId()) + .defaultTopic(topicService.buildTopicName(coreSettings.getHousekeeperTopic())) + .admin(housekeeperAdmin) + .build(); + } + @PreDestroy private void destroy() { if (coreAdmin != null) { diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbVersionControlQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbVersionControlQueueFactory.java index 31ded1f3e3..c6633b030c 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbVersionControlQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbVersionControlQueueFactory.java @@ -17,7 +17,9 @@ package org.thingsboard.server.queue.provider; import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; import org.springframework.stereotype.Component; +import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToHousekeeperServiceMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToUsageStatsServiceMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToVersionControlServiceMsg; import org.thingsboard.server.queue.TbQueueAdmin; @@ -51,6 +53,7 @@ public class KafkaTbVersionControlQueueFactory implements TbVersionControlQueueF private final TbQueueAdmin coreAdmin; private final TbQueueAdmin vcAdmin; private final TbQueueAdmin notificationAdmin; + private final TbQueueAdmin housekeeperAdmin; public KafkaTbVersionControlQueueFactory(TbKafkaSettings kafkaSettings, TbServiceInfoProvider serviceInfoProvider, @@ -69,6 +72,7 @@ public class KafkaTbVersionControlQueueFactory implements TbVersionControlQueueF this.coreAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getCoreConfigs()); this.vcAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getVcConfigs()); this.notificationAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getNotificationsConfigs()); + this.housekeeperAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getHousekeeperConfigs()); } @@ -105,6 +109,16 @@ public class KafkaTbVersionControlQueueFactory implements TbVersionControlQueueF return requestBuilder.build(); } + @Override + public TbQueueProducer> createHousekeeperMsgProducer() { + return TbKafkaProducerTemplate.>builder() + .settings(kafkaSettings) + .clientId("tb-vc-housekeeper-producer-" + serviceInfoProvider.getServiceId()) + .defaultTopic(topicService.buildTopicName(coreSettings.getHousekeeperTopic())) + .admin(housekeeperAdmin) + .build(); + } + @PreDestroy private void destroy() { if (coreAdmin != null) { diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubMonolithQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubMonolithQueueFactory.java index 61c2614fc7..a3734daa91 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubMonolithQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubMonolithQueueFactory.java @@ -26,6 +26,7 @@ import org.thingsboard.server.gen.js.JsInvokeProtos.RemoteJsResponse; import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToHousekeeperServiceMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToOtaPackageStateServiceMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineNotificationMsg; @@ -40,8 +41,8 @@ import org.thingsboard.server.queue.TbQueueRequestTemplate; import org.thingsboard.server.queue.common.DefaultTbQueueRequestTemplate; import org.thingsboard.server.queue.common.TbProtoJsQueueMsg; import org.thingsboard.server.queue.common.TbProtoQueueMsg; -import org.thingsboard.server.queue.discovery.TopicService; import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; +import org.thingsboard.server.queue.discovery.TopicService; import org.thingsboard.server.queue.pubsub.TbPubSubAdmin; import org.thingsboard.server.queue.pubsub.TbPubSubConsumerTemplate; import org.thingsboard.server.queue.pubsub.TbPubSubProducerTemplate; @@ -227,6 +228,28 @@ public class PubSubMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEng return new TbPubSubProducerTemplate<>(vcAdmin, pubSubSettings, topicService.buildTopicName(vcSettings.getTopic())); } + @Override + public TbQueueProducer> createHousekeeperMsgProducer() { + return new TbPubSubProducerTemplate<>(coreAdmin, pubSubSettings, topicService.buildTopicName(coreSettings.getHousekeeperTopic())); + } + + @Override + public TbQueueConsumer> createHousekeeperMsgConsumer() { + return new TbPubSubConsumerTemplate<>(coreAdmin, pubSubSettings, topicService.buildTopicName(coreSettings.getHousekeeperTopic()), + msg -> new TbProtoQueueMsg<>(msg.getKey(), ToHousekeeperServiceMsg.parseFrom(msg.getData()), msg.getHeaders())); + } + + @Override + public TbQueueProducer> createHousekeeperReprocessingMsgProducer() { + return new TbPubSubProducerTemplate<>(coreAdmin, pubSubSettings, topicService.buildTopicName(coreSettings.getHousekeeperReprocessingTopic())); + } + + @Override + public TbQueueConsumer> createHousekeeperReprocessingMsgConsumer() { + return new TbPubSubConsumerTemplate<>(coreAdmin, pubSubSettings, topicService.buildTopicName(coreSettings.getHousekeeperReprocessingTopic()), + msg -> new TbProtoQueueMsg<>(msg.getKey(), ToHousekeeperServiceMsg.parseFrom(msg.getData()), msg.getHeaders())); + } + @PreDestroy private void destroy() { if (coreAdmin != null) { diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTbCoreQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTbCoreQueueFactory.java index 63975e3116..c48559467d 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTbCoreQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTbCoreQueueFactory.java @@ -24,6 +24,7 @@ import org.thingsboard.server.gen.js.JsInvokeProtos; import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToHousekeeperServiceMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToOtaPackageStateServiceMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineNotificationMsg; @@ -38,8 +39,8 @@ import org.thingsboard.server.queue.TbQueueRequestTemplate; import org.thingsboard.server.queue.common.DefaultTbQueueRequestTemplate; import org.thingsboard.server.queue.common.TbProtoJsQueueMsg; import org.thingsboard.server.queue.common.TbProtoQueueMsg; -import org.thingsboard.server.queue.discovery.TopicService; import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; +import org.thingsboard.server.queue.discovery.TopicService; import org.thingsboard.server.queue.pubsub.TbPubSubAdmin; import org.thingsboard.server.queue.pubsub.TbPubSubConsumerTemplate; import org.thingsboard.server.queue.pubsub.TbPubSubProducerTemplate; @@ -198,6 +199,28 @@ public class PubSubTbCoreQueueFactory implements TbCoreQueueFactory { return null; } + @Override + public TbQueueProducer> createHousekeeperMsgProducer() { + return new TbPubSubProducerTemplate<>(coreAdmin, pubSubSettings, topicService.buildTopicName(coreSettings.getHousekeeperTopic())); + } + + @Override + public TbQueueConsumer> createHousekeeperMsgConsumer() { + return new TbPubSubConsumerTemplate<>(coreAdmin, pubSubSettings, topicService.buildTopicName(coreSettings.getHousekeeperTopic()), + msg -> new TbProtoQueueMsg<>(msg.getKey(), ToHousekeeperServiceMsg.parseFrom(msg.getData()), msg.getHeaders())); + } + + @Override + public TbQueueProducer> createHousekeeperReprocessingMsgProducer() { + return new TbPubSubProducerTemplate<>(coreAdmin, pubSubSettings, topicService.buildTopicName(coreSettings.getHousekeeperReprocessingTopic())); + } + + @Override + public TbQueueConsumer> createHousekeeperReprocessingMsgConsumer() { + return new TbPubSubConsumerTemplate<>(coreAdmin, pubSubSettings, topicService.buildTopicName(coreSettings.getHousekeeperReprocessingTopic()), + msg -> new TbProtoQueueMsg<>(msg.getKey(), ToHousekeeperServiceMsg.parseFrom(msg.getData()), msg.getHeaders())); + } + @PreDestroy private void destroy() { if (coreAdmin != null) { diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTbRuleEngineQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTbRuleEngineQueueFactory.java index 9cf474a0e0..3895971655 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTbRuleEngineQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTbRuleEngineQueueFactory.java @@ -25,6 +25,8 @@ import org.thingsboard.server.gen.js.JsInvokeProtos; import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToHousekeeperServiceMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToOtaPackageStateServiceMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; @@ -36,8 +38,8 @@ import org.thingsboard.server.queue.TbQueueRequestTemplate; import org.thingsboard.server.queue.common.DefaultTbQueueRequestTemplate; import org.thingsboard.server.queue.common.TbProtoJsQueueMsg; import org.thingsboard.server.queue.common.TbProtoQueueMsg; -import org.thingsboard.server.queue.discovery.TopicService; import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; +import org.thingsboard.server.queue.discovery.TopicService; import org.thingsboard.server.queue.pubsub.TbPubSubAdmin; import org.thingsboard.server.queue.pubsub.TbPubSubConsumerTemplate; import org.thingsboard.server.queue.pubsub.TbPubSubProducerTemplate; @@ -157,10 +159,15 @@ public class PubSubTbRuleEngineQueueFactory implements TbRuleEngineQueueFactory } @Override - public TbQueueProducer> createToOtaPackageStateServiceMsgProducer() { + public TbQueueProducer> createToOtaPackageStateServiceMsgProducer() { return new TbPubSubProducerTemplate<>(coreAdmin, pubSubSettings, topicService.buildTopicName(coreSettings.getOtaPackageTopic())); } + @Override + public TbQueueProducer> createHousekeeperMsgProducer() { + return new TbPubSubProducerTemplate<>(coreAdmin, pubSubSettings, topicService.buildTopicName(coreSettings.getHousekeeperTopic())); + } + @PreDestroy private void destroy() { if (coreAdmin != null) { diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTbVersionControlQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTbVersionControlQueueFactory.java index b970490893..3faf35bbc7 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTbVersionControlQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTbVersionControlQueueFactory.java @@ -79,6 +79,11 @@ public class PubSubTbVersionControlQueueFactory implements TbVersionControlQueue ); } + @Override + public TbQueueProducer> createHousekeeperMsgProducer() { + return new TbPubSubProducerTemplate<>(coreAdmin, pubSubSettings, topicService.buildTopicName(coreSettings.getHousekeeperTopic())); + } + @PreDestroy private void destroy() { if (coreAdmin != null) { diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTransportQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTransportQueueFactory.java index 1f4d838c1e..9f993a553d 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTransportQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/PubSubTransportQueueFactory.java @@ -21,6 +21,7 @@ import org.springframework.stereotype.Component; import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToHousekeeperServiceMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToUsageStatsServiceMsg; @@ -131,6 +132,11 @@ public class PubSubTransportQueueFactory implements TbTransportQueueFactory { return new TbPubSubProducerTemplate<>(coreAdmin, pubSubSettings, topicService.buildTopicName(coreSettings.getUsageStatsTopic())); } + @Override + public TbQueueProducer> createHousekeeperMsgProducer() { + return new TbPubSubProducerTemplate<>(coreAdmin, pubSubSettings, topicService.buildTopicName(coreSettings.getHousekeeperTopic())); + } + @PreDestroy private void destroy() { if (coreAdmin != null) { diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqMonolithQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqMonolithQueueFactory.java index a3fbc5a360..40f4fdc2e5 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqMonolithQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqMonolithQueueFactory.java @@ -26,6 +26,7 @@ import org.thingsboard.server.gen.js.JsInvokeProtos.RemoteJsResponse; import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToHousekeeperServiceMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToOtaPackageStateServiceMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineNotificationMsg; @@ -40,8 +41,8 @@ import org.thingsboard.server.queue.TbQueueRequestTemplate; import org.thingsboard.server.queue.common.DefaultTbQueueRequestTemplate; import org.thingsboard.server.queue.common.TbProtoJsQueueMsg; import org.thingsboard.server.queue.common.TbProtoQueueMsg; -import org.thingsboard.server.queue.discovery.TopicService; import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; +import org.thingsboard.server.queue.discovery.TopicService; import org.thingsboard.server.queue.rabbitmq.TbRabbitMqAdmin; import org.thingsboard.server.queue.rabbitmq.TbRabbitMqConsumerTemplate; import org.thingsboard.server.queue.rabbitmq.TbRabbitMqProducerTemplate; @@ -224,6 +225,28 @@ public class RabbitMqMonolithQueueFactory implements TbCoreQueueFactory, TbRuleE return new TbRabbitMqProducerTemplate<>(vcAdmin, rabbitMqSettings, topicService.buildTopicName(vcSettings.getTopic())); } + @Override + public TbQueueProducer> createHousekeeperMsgProducer() { + return new TbRabbitMqProducerTemplate<>(coreAdmin, rabbitMqSettings, topicService.buildTopicName(coreSettings.getHousekeeperTopic())); + } + + @Override + public TbQueueConsumer> createHousekeeperMsgConsumer() { + return new TbRabbitMqConsumerTemplate<>(coreAdmin, rabbitMqSettings, topicService.buildTopicName(coreSettings.getHousekeeperTopic()), + msg -> new TbProtoQueueMsg<>(msg.getKey(), ToHousekeeperServiceMsg.parseFrom(msg.getData()), msg.getHeaders())); + } + + @Override + public TbQueueProducer> createHousekeeperReprocessingMsgProducer() { + return new TbRabbitMqProducerTemplate<>(coreAdmin, rabbitMqSettings, topicService.buildTopicName(coreSettings.getHousekeeperReprocessingTopic())); + } + + @Override + public TbQueueConsumer> createHousekeeperReprocessingMsgConsumer() { + return new TbRabbitMqConsumerTemplate<>(coreAdmin, rabbitMqSettings, topicService.buildTopicName(coreSettings.getHousekeeperReprocessingTopic()), + msg -> new TbProtoQueueMsg<>(msg.getKey(), ToHousekeeperServiceMsg.parseFrom(msg.getData()), msg.getHeaders())); + } + @PreDestroy private void destroy() { if (coreAdmin != null) { diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqTbCoreQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqTbCoreQueueFactory.java index af4d46c487..06fe34c458 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqTbCoreQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqTbCoreQueueFactory.java @@ -38,8 +38,8 @@ import org.thingsboard.server.queue.TbQueueRequestTemplate; import org.thingsboard.server.queue.common.DefaultTbQueueRequestTemplate; import org.thingsboard.server.queue.common.TbProtoJsQueueMsg; import org.thingsboard.server.queue.common.TbProtoQueueMsg; -import org.thingsboard.server.queue.discovery.TopicService; import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; +import org.thingsboard.server.queue.discovery.TopicService; import org.thingsboard.server.queue.rabbitmq.TbRabbitMqAdmin; import org.thingsboard.server.queue.rabbitmq.TbRabbitMqConsumerTemplate; import org.thingsboard.server.queue.rabbitmq.TbRabbitMqProducerTemplate; @@ -198,6 +198,28 @@ public class RabbitMqTbCoreQueueFactory implements TbCoreQueueFactory { return new TbRabbitMqProducerTemplate<>(coreAdmin, rabbitMqSettings, topicService.buildTopicName(coreSettings.getUsageStatsTopic())); } + @Override + public TbQueueProducer> createHousekeeperMsgProducer() { + return new TbRabbitMqProducerTemplate<>(coreAdmin, rabbitMqSettings, topicService.buildTopicName(coreSettings.getHousekeeperTopic())); + } + + @Override + public TbQueueConsumer> createHousekeeperMsgConsumer() { + return new TbRabbitMqConsumerTemplate<>(coreAdmin, rabbitMqSettings, topicService.buildTopicName(coreSettings.getHousekeeperTopic()), + msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportProtos.ToHousekeeperServiceMsg.parseFrom(msg.getData()), msg.getHeaders())); + } + + @Override + public TbQueueProducer> createHousekeeperReprocessingMsgProducer() { + return new TbRabbitMqProducerTemplate<>(coreAdmin, rabbitMqSettings, topicService.buildTopicName(coreSettings.getHousekeeperReprocessingTopic())); + } + + @Override + public TbQueueConsumer> createHousekeeperReprocessingMsgConsumer() { + return new TbRabbitMqConsumerTemplate<>(coreAdmin, rabbitMqSettings, topicService.buildTopicName(coreSettings.getHousekeeperReprocessingTopic()), + msg -> new TbProtoQueueMsg<>(msg.getKey(), TransportProtos.ToHousekeeperServiceMsg.parseFrom(msg.getData()), msg.getHeaders())); + } + @PreDestroy private void destroy() { if (coreAdmin != null) { diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqTbRuleEngineQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqTbRuleEngineQueueFactory.java index 5e76baaec9..8d7016404a 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqTbRuleEngineQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqTbRuleEngineQueueFactory.java @@ -160,6 +160,11 @@ public class RabbitMqTbRuleEngineQueueFactory implements TbRuleEngineQueueFactor return new TbRabbitMqProducerTemplate<>(coreAdmin, rabbitMqSettings, topicService.buildTopicName(coreSettings.getOtaPackageTopic())); } + @Override + public TbQueueProducer> createHousekeeperMsgProducer() { + return new TbRabbitMqProducerTemplate<>(coreAdmin, rabbitMqSettings, topicService.buildTopicName(coreSettings.getHousekeeperTopic())); + } + @PreDestroy private void destroy() { if (coreAdmin != null) { diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqTbVersionControlQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqTbVersionControlQueueFactory.java index 2be1531334..d2842a3492 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqTbVersionControlQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqTbVersionControlQueueFactory.java @@ -79,6 +79,11 @@ public class RabbitMqTbVersionControlQueueFactory implements TbVersionControlQue ); } + @Override + public TbQueueProducer> createHousekeeperMsgProducer() { + return new TbRabbitMqProducerTemplate<>(coreAdmin, rabbitMqSettings, topicService.buildTopicName(coreSettings.getHousekeeperTopic())); + } + @PreDestroy private void destroy() { if (coreAdmin != null) { diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqTransportQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqTransportQueueFactory.java index 0d4a37f437..13df3414d8 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqTransportQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/RabbitMqTransportQueueFactory.java @@ -132,6 +132,11 @@ public class RabbitMqTransportQueueFactory implements TbTransportQueueFactory { return new TbRabbitMqProducerTemplate<>(coreAdmin, rabbitMqSettings, topicService.buildTopicName(coreSettings.getUsageStatsTopic())); } + @Override + public TbQueueProducer> createHousekeeperMsgProducer() { + return new TbRabbitMqProducerTemplate<>(coreAdmin, rabbitMqSettings, topicService.buildTopicName(coreSettings.getHousekeeperTopic())); + } + @PreDestroy private void destroy() { if (coreAdmin != null) { diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusMonolithQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusMonolithQueueFactory.java index e2d11a6d62..c52ec7f7b2 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusMonolithQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusMonolithQueueFactory.java @@ -25,6 +25,7 @@ import org.thingsboard.server.gen.js.JsInvokeProtos; import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToHousekeeperServiceMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToOtaPackageStateServiceMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineNotificationMsg; @@ -44,8 +45,8 @@ import org.thingsboard.server.queue.azure.servicebus.TbServiceBusSettings; import org.thingsboard.server.queue.common.DefaultTbQueueRequestTemplate; import org.thingsboard.server.queue.common.TbProtoJsQueueMsg; import org.thingsboard.server.queue.common.TbProtoQueueMsg; -import org.thingsboard.server.queue.discovery.TopicService; import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; +import org.thingsboard.server.queue.discovery.TopicService; import org.thingsboard.server.queue.settings.TbQueueCoreSettings; import org.thingsboard.server.queue.settings.TbQueueRemoteJsInvokeSettings; import org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings; @@ -223,6 +224,28 @@ public class ServiceBusMonolithQueueFactory implements TbCoreQueueFactory, TbRul return new TbServiceBusProducerTemplate<>(vcAdmin, serviceBusSettings, topicService.buildTopicName(vcSettings.getTopic())); } + @Override + public TbQueueProducer> createHousekeeperMsgProducer() { + return new TbServiceBusProducerTemplate<>(coreAdmin, serviceBusSettings, topicService.buildTopicName(coreSettings.getHousekeeperTopic())); + } + + @Override + public TbQueueConsumer> createHousekeeperMsgConsumer() { + return new TbServiceBusConsumerTemplate<>(coreAdmin, serviceBusSettings, topicService.buildTopicName(coreSettings.getHousekeeperTopic()), + msg -> new TbProtoQueueMsg<>(msg.getKey(), ToHousekeeperServiceMsg.parseFrom(msg.getData()), msg.getHeaders())); + } + + @Override + public TbQueueProducer> createHousekeeperReprocessingMsgProducer() { + return new TbServiceBusProducerTemplate<>(coreAdmin, serviceBusSettings, topicService.buildTopicName(coreSettings.getHousekeeperReprocessingTopic())); + } + + @Override + public TbQueueConsumer> createHousekeeperReprocessingMsgConsumer() { + return new TbServiceBusConsumerTemplate<>(coreAdmin, serviceBusSettings, topicService.buildTopicName(coreSettings.getHousekeeperReprocessingTopic()), + msg -> new TbProtoQueueMsg<>(msg.getKey(), ToHousekeeperServiceMsg.parseFrom(msg.getData()), msg.getHeaders())); + } + @PreDestroy private void destroy() { if (coreAdmin != null) { diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusTbCoreQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusTbCoreQueueFactory.java index dd68cb57a5..37fcfad1bd 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusTbCoreQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusTbCoreQueueFactory.java @@ -24,6 +24,7 @@ import org.thingsboard.server.gen.js.JsInvokeProtos; import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToHousekeeperServiceMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToOtaPackageStateServiceMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineNotificationMsg; @@ -43,8 +44,8 @@ import org.thingsboard.server.queue.azure.servicebus.TbServiceBusSettings; import org.thingsboard.server.queue.common.DefaultTbQueueRequestTemplate; import org.thingsboard.server.queue.common.TbProtoJsQueueMsg; import org.thingsboard.server.queue.common.TbProtoQueueMsg; -import org.thingsboard.server.queue.discovery.TopicService; import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; +import org.thingsboard.server.queue.discovery.TopicService; import org.thingsboard.server.queue.settings.TbQueueCoreSettings; import org.thingsboard.server.queue.settings.TbQueueRemoteJsInvokeSettings; import org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings; @@ -198,6 +199,28 @@ public class ServiceBusTbCoreQueueFactory implements TbCoreQueueFactory { return null; } + @Override + public TbQueueProducer> createHousekeeperMsgProducer() { + return new TbServiceBusProducerTemplate<>(coreAdmin, serviceBusSettings, topicService.buildTopicName(coreSettings.getHousekeeperTopic())); + } + + @Override + public TbQueueConsumer> createHousekeeperMsgConsumer() { + return new TbServiceBusConsumerTemplate<>(coreAdmin, serviceBusSettings, topicService.buildTopicName(coreSettings.getHousekeeperTopic()), + msg -> new TbProtoQueueMsg<>(msg.getKey(), ToHousekeeperServiceMsg.parseFrom(msg.getData()), msg.getHeaders())); + } + + @Override + public TbQueueProducer> createHousekeeperReprocessingMsgProducer() { + return new TbServiceBusProducerTemplate<>(coreAdmin, serviceBusSettings, topicService.buildTopicName(coreSettings.getHousekeeperReprocessingTopic())); + } + + @Override + public TbQueueConsumer> createHousekeeperReprocessingMsgConsumer() { + return new TbServiceBusConsumerTemplate<>(coreAdmin, serviceBusSettings, topicService.buildTopicName(coreSettings.getHousekeeperReprocessingTopic()), + msg -> new TbProtoQueueMsg<>(msg.getKey(), ToHousekeeperServiceMsg.parseFrom(msg.getData()), msg.getHeaders())); + } + @PreDestroy private void destroy() { if (coreAdmin != null) { diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusTbRuleEngineQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusTbRuleEngineQueueFactory.java index 2a23383e13..cd2c52c21d 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusTbRuleEngineQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusTbRuleEngineQueueFactory.java @@ -41,8 +41,8 @@ import org.thingsboard.server.queue.azure.servicebus.TbServiceBusSettings; import org.thingsboard.server.queue.common.DefaultTbQueueRequestTemplate; import org.thingsboard.server.queue.common.TbProtoJsQueueMsg; import org.thingsboard.server.queue.common.TbProtoQueueMsg; -import org.thingsboard.server.queue.discovery.TopicService; import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; +import org.thingsboard.server.queue.discovery.TopicService; import org.thingsboard.server.queue.settings.TbQueueCoreSettings; import org.thingsboard.server.queue.settings.TbQueueRemoteJsInvokeSettings; import org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings; @@ -160,6 +160,11 @@ public class ServiceBusTbRuleEngineQueueFactory implements TbRuleEngineQueueFact return new TbServiceBusProducerTemplate<>(coreAdmin, serviceBusSettings, topicService.buildTopicName(coreSettings.getOtaPackageTopic())); } + @Override + public TbQueueProducer> createHousekeeperMsgProducer() { + return new TbServiceBusProducerTemplate<>(coreAdmin, serviceBusSettings, topicService.buildTopicName(coreSettings.getHousekeeperTopic())); + } + @PreDestroy private void destroy() { if (coreAdmin != null) { diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusTbVersionControlQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusTbVersionControlQueueFactory.java index 8762a906a3..c3447a1012 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusTbVersionControlQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusTbVersionControlQueueFactory.java @@ -79,6 +79,11 @@ public class ServiceBusTbVersionControlQueueFactory implements TbVersionControlQ ); } + @Override + public TbQueueProducer> createHousekeeperMsgProducer() { + return new TbServiceBusProducerTemplate<>(coreAdmin, serviceBusSettings, topicService.buildTopicName(coreSettings.getHousekeeperTopic())); + } + @PreDestroy private void destroy() { if (coreAdmin != null) { diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusTransportQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusTransportQueueFactory.java index d6cae4a682..f2ed799905 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusTransportQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/ServiceBusTransportQueueFactory.java @@ -133,6 +133,11 @@ public class ServiceBusTransportQueueFactory implements TbTransportQueueFactory return new TbServiceBusProducerTemplate<>(coreAdmin, serviceBusSettings, topicService.buildTopicName(coreSettings.getUsageStatsTopic())); } + @Override + public TbQueueProducer> createHousekeeperMsgProducer() { + return new TbServiceBusProducerTemplate<>(coreAdmin, serviceBusSettings, topicService.buildTopicName(coreSettings.getHousekeeperTopic())); + } + @PreDestroy private void destroy() { if (coreAdmin != null) { diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbCoreQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbCoreQueueFactory.java index 5aa7866033..20a67c2e09 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbCoreQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbCoreQueueFactory.java @@ -18,6 +18,7 @@ package org.thingsboard.server.queue.provider; import org.thingsboard.server.gen.js.JsInvokeProtos; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToHousekeeperServiceMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToOtaPackageStateServiceMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineNotificationMsg; @@ -36,7 +37,7 @@ import org.thingsboard.server.queue.common.TbProtoQueueMsg; * Responsible for initialization of various Producers and Consumers used by TB Core Node. * Implementation Depends on the queue queue.type from yml or TB_QUEUE_TYPE environment variable */ -public interface TbCoreQueueFactory extends TbUsageStatsClientQueueFactory { +public interface TbCoreQueueFactory extends TbUsageStatsClientQueueFactory, HousekeeperClientQueueFactory { /** * Used to push messages to instances of TB Transport Service @@ -130,4 +131,11 @@ public interface TbCoreQueueFactory extends TbUsageStatsClientQueueFactory { * @return */ TbQueueProducer> createVersionControlMsgProducer(); + + TbQueueConsumer> createHousekeeperMsgConsumer(); + + TbQueueProducer> createHousekeeperReprocessingMsgProducer(); + + TbQueueConsumer> createHousekeeperReprocessingMsgConsumer(); + } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbCoreQueueProducerProvider.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbCoreQueueProducerProvider.java index 8379fc48be..3dcd716a96 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbCoreQueueProducerProvider.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbCoreQueueProducerProvider.java @@ -18,6 +18,7 @@ package org.thingsboard.server.queue.provider; import org.springframework.stereotype.Service; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToHousekeeperServiceMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; @@ -41,6 +42,7 @@ public class TbCoreQueueProducerProvider implements TbQueueProducerProvider { private TbQueueProducer> toTbCoreNotifications; private TbQueueProducer> toUsageStats; private TbQueueProducer> toVersionControl; + private TbQueueProducer> toHousekeeper; public TbCoreQueueProducerProvider(TbCoreQueueFactory tbQueueProvider) { this.tbQueueProvider = tbQueueProvider; @@ -55,6 +57,7 @@ public class TbCoreQueueProducerProvider implements TbQueueProducerProvider { this.toTbCoreNotifications = tbQueueProvider.createTbCoreNotificationsMsgProducer(); this.toUsageStats = tbQueueProvider.createToUsageStatsServiceMsgProducer(); this.toVersionControl = tbQueueProvider.createVersionControlMsgProducer(); + this.toHousekeeper = tbQueueProvider.createHousekeeperMsgProducer(); } @Override @@ -91,4 +94,10 @@ public class TbCoreQueueProducerProvider implements TbQueueProducerProvider { public TbQueueProducer> getTbVersionControlMsgProducer() { return toVersionControl; } + + @Override + public TbQueueProducer> getHousekeeperMsgProducer() { + return toHousekeeper; + } + } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbQueueProducerProvider.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbQueueProducerProvider.java index 4dbdca4375..a752301cd8 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbQueueProducerProvider.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbQueueProducerProvider.java @@ -17,6 +17,7 @@ package org.thingsboard.server.queue.provider; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToHousekeeperServiceMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; @@ -78,4 +79,7 @@ public interface TbQueueProducerProvider { * @return */ TbQueueProducer> getTbVersionControlMsgProducer(); + + TbQueueProducer> getHousekeeperMsgProducer(); + } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbRuleEngineProducerProvider.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbRuleEngineProducerProvider.java index 1e1ccf3b76..768d6b9c24 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbRuleEngineProducerProvider.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbRuleEngineProducerProvider.java @@ -20,6 +20,7 @@ import org.springframework.stereotype.Service; import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToHousekeeperServiceMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; @@ -40,6 +41,7 @@ public class TbRuleEngineProducerProvider implements TbQueueProducerProvider { private TbQueueProducer> toRuleEngineNotifications; private TbQueueProducer> toTbCoreNotifications; private TbQueueProducer> toUsageStats; + private TbQueueProducer> toHousekeeper; public TbRuleEngineProducerProvider(TbRuleEngineQueueFactory tbQueueProvider) { this.tbQueueProvider = tbQueueProvider; @@ -53,6 +55,7 @@ public class TbRuleEngineProducerProvider implements TbQueueProducerProvider { this.toRuleEngineNotifications = tbQueueProvider.createRuleEngineNotificationsMsgProducer(); this.toTbCoreNotifications = tbQueueProvider.createTbCoreNotificationsMsgProducer(); this.toUsageStats = tbQueueProvider.createToUsageStatsServiceMsgProducer(); + this.toHousekeeper = tbQueueProvider.createHousekeeperMsgProducer(); } @Override @@ -89,4 +92,10 @@ public class TbRuleEngineProducerProvider implements TbQueueProducerProvider { public TbQueueProducer> getTbVersionControlMsgProducer() { throw new RuntimeException("Not Implemented! Should not be used by Rule Engine!"); } + + @Override + public TbQueueProducer> getHousekeeperMsgProducer() { + return toHousekeeper; + } + } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbRuleEngineQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbRuleEngineQueueFactory.java index cd4575af6a..f23c7e47f3 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbRuleEngineQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbRuleEngineQueueFactory.java @@ -32,7 +32,7 @@ import org.thingsboard.server.queue.common.TbProtoQueueMsg; * Responsible for initialization of various Producers and Consumers used by TB Core Node. * Implementation Depends on the queue queue.type from yml or TB_QUEUE_TYPE environment variable */ -public interface TbRuleEngineQueueFactory extends TbUsageStatsClientQueueFactory { +public interface TbRuleEngineQueueFactory extends TbUsageStatsClientQueueFactory, HousekeeperClientQueueFactory { /** * Used to push messages to instances of TB Transport Service diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbTransportQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbTransportQueueFactory.java index 909089f175..c1a0a0fd24 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbTransportQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbTransportQueueFactory.java @@ -26,7 +26,7 @@ import org.thingsboard.server.queue.TbQueueProducer; import org.thingsboard.server.queue.TbQueueRequestTemplate; import org.thingsboard.server.queue.common.TbProtoQueueMsg; -public interface TbTransportQueueFactory extends TbUsageStatsClientQueueFactory { +public interface TbTransportQueueFactory extends TbUsageStatsClientQueueFactory, HousekeeperClientQueueFactory { TbQueueRequestTemplate, TbProtoQueueMsg> createTransportApiRequestTemplate(); diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbTransportQueueProducerProvider.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbTransportQueueProducerProvider.java index 3096265e0f..010eaf404f 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbTransportQueueProducerProvider.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbTransportQueueProducerProvider.java @@ -20,6 +20,7 @@ import org.springframework.stereotype.Service; import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToHousekeeperServiceMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; @@ -38,6 +39,7 @@ public class TbTransportQueueProducerProvider implements TbQueueProducerProvider private TbQueueProducer> toTbCore; private TbQueueProducer> toTbCoreNotifications; private TbQueueProducer> toUsageStats; + private TbQueueProducer> toHousekeeper; public TbTransportQueueProducerProvider(TbTransportQueueFactory tbQueueProvider) { this.tbQueueProvider = tbQueueProvider; @@ -49,6 +51,7 @@ public class TbTransportQueueProducerProvider implements TbQueueProducerProvider this.toRuleEngine = tbQueueProvider.createRuleEngineMsgProducer(); this.toUsageStats = tbQueueProvider.createToUsageStatsServiceMsgProducer(); this.toTbCoreNotifications = tbQueueProvider.createTbCoreNotificationsMsgProducer(); + this.toHousekeeper = tbQueueProvider.createHousekeeperMsgProducer(); } @Override @@ -85,4 +88,10 @@ public class TbTransportQueueProducerProvider implements TbQueueProducerProvider public TbQueueProducer> getTbUsageStatsMsgProducer() { return toUsageStats; } + + @Override + public TbQueueProducer> getHousekeeperMsgProducer() { + return toHousekeeper; + } + } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbVersionControlProducerProvider.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbVersionControlProducerProvider.java index 0f1703f3d3..1e697c644a 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbVersionControlProducerProvider.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbVersionControlProducerProvider.java @@ -20,6 +20,7 @@ import org.springframework.stereotype.Service; import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToHousekeeperServiceMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; @@ -36,6 +37,7 @@ public class TbVersionControlProducerProvider implements TbQueueProducerProvider private final TbVersionControlQueueFactory tbQueueProvider; private TbQueueProducer> toTbCoreNotifications; private TbQueueProducer> toUsageStats; + private TbQueueProducer> toHousekeeper; public TbVersionControlProducerProvider(TbVersionControlQueueFactory tbQueueProvider) { this.tbQueueProvider = tbQueueProvider; @@ -45,6 +47,7 @@ public class TbVersionControlProducerProvider implements TbQueueProducerProvider public void init() { this.toTbCoreNotifications = tbQueueProvider.createTbCoreNotificationsMsgProducer(); this.toUsageStats = tbQueueProvider.createToUsageStatsServiceMsgProducer(); + this.toHousekeeper = tbQueueProvider.createHousekeeperMsgProducer(); } @Override @@ -54,7 +57,7 @@ public class TbVersionControlProducerProvider implements TbQueueProducerProvider @Override public TbQueueProducer> getRuleEngineMsgProducer() { - throw new RuntimeException("Not Implemented! Should not be used by Version Control Service!"); + throw new RuntimeException("Not Implemented! Should not be used by Version Control Service!"); } @Override @@ -81,4 +84,10 @@ public class TbVersionControlProducerProvider implements TbQueueProducerProvider public TbQueueProducer> getTbUsageStatsMsgProducer() { return toUsageStats; } + + @Override + public TbQueueProducer> getHousekeeperMsgProducer() { + return toHousekeeper; + } + } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbVersionControlQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbVersionControlQueueFactory.java index 168a466015..3917df24cd 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbVersionControlQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbVersionControlQueueFactory.java @@ -25,7 +25,7 @@ import org.thingsboard.server.queue.common.TbProtoQueueMsg; * Responsible for initialization of various Producers and Consumers used by TB Version Control Node. * Implementation Depends on the queue queue.type from yml or TB_QUEUE_TYPE environment variable */ -public interface TbVersionControlQueueFactory extends TbUsageStatsClientQueueFactory { +public interface TbVersionControlQueueFactory extends TbUsageStatsClientQueueFactory, HousekeeperClientQueueFactory { /** * Used to push notifications to other instances of TB Core Service diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/settings/TbQueueCoreSettings.java b/common/queue/src/main/java/org/thingsboard/server/queue/settings/TbQueueCoreSettings.java index 0a8c12eb8d..2aa01193b0 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/settings/TbQueueCoreSettings.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/settings/TbQueueCoreSettings.java @@ -34,6 +34,12 @@ public class TbQueueCoreSettings { @Value("${queue.core.usage-stats-topic:tb_usage_stats}") private String usageStatsTopic; + @Value("${queue.core.housekeeper.topic:tb_housekeeper}") + private String housekeeperTopic; + + @Value("${queue.core.housekeeper.reprocessing-topic:tb_housekeeper.reprocessing}") + private String housekeeperReprocessingTopic; + @Value("${queue.core.partitions}") private int partitions; } diff --git a/common/stats/src/main/java/org/thingsboard/server/common/stats/DefaultStatsFactory.java b/common/stats/src/main/java/org/thingsboard/server/common/stats/DefaultStatsFactory.java index be0c4fe19f..ca97792186 100644 --- a/common/stats/src/main/java/org/thingsboard/server/common/stats/DefaultStatsFactory.java +++ b/common/stats/src/main/java/org/thingsboard/server/common/stats/DefaultStatsFactory.java @@ -19,13 +19,13 @@ import io.micrometer.core.instrument.Counter; import io.micrometer.core.instrument.MeterRegistry; import io.micrometer.core.instrument.Tags; import io.micrometer.core.instrument.Timer; +import jakarta.annotation.PostConstruct; import org.apache.commons.lang3.ArrayUtils; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Service; import org.thingsboard.server.common.data.StringUtils; -import jakarta.annotation.PostConstruct; import java.util.concurrent.atomic.AtomicInteger; @Service @@ -63,13 +63,7 @@ public class DefaultStatsFactory implements StatsFactory { @Override public StatsCounter createStatsCounter(String key, String statsName, String... otherTags) { - String[] tags = new String[]{STATS_NAME_TAG, statsName}; - if (otherTags.length > 0) { - if (otherTags.length % 2 != 0) { - throw new IllegalArgumentException("Invalid tags array size"); - } - tags = ArrayUtils.addAll(tags, otherTags); - } + String[] tags = getTags(statsName, otherTags); return new StatsCounter( new AtomicInteger(0), metricsEnabled ? meterRegistry.counter(key, tags) : STUB_COUNTER, @@ -111,6 +105,24 @@ public class DefaultStatsFactory implements StatsFactory { return timerBuilder.register(meterRegistry); } + @Override + public StatsTimer createTimer(StatsType type, String name, String... tags) { + return new StatsTimer(name, Timer.builder(type.getName()) + .tags(getTags(name, tags)) + .register(meterRegistry)); + } + + private static String[] getTags(String statsName, String[] otherTags) { + String[] tags = new String[]{STATS_NAME_TAG, statsName}; + if (otherTags.length > 0) { + if (otherTags.length % 2 != 0) { + throw new IllegalArgumentException("Invalid tags array size"); + } + tags = ArrayUtils.addAll(tags, otherTags); + } + return tags; + } + private static class StubCounter implements Counter { @Override public void increment(double amount) { diff --git a/common/stats/src/main/java/org/thingsboard/server/common/stats/StatsFactory.java b/common/stats/src/main/java/org/thingsboard/server/common/stats/StatsFactory.java index b9bf0b97fb..a6cc4aa1d4 100644 --- a/common/stats/src/main/java/org/thingsboard/server/common/stats/StatsFactory.java +++ b/common/stats/src/main/java/org/thingsboard/server/common/stats/StatsFactory.java @@ -29,4 +29,6 @@ public interface StatsFactory { Timer createTimer(String key, String... tags); + StatsTimer createTimer(StatsType type, String name, String... tags); + } diff --git a/common/stats/src/main/java/org/thingsboard/server/common/stats/StatsTimer.java b/common/stats/src/main/java/org/thingsboard/server/common/stats/StatsTimer.java new file mode 100644 index 0000000000..ca74444619 --- /dev/null +++ b/common/stats/src/main/java/org/thingsboard/server/common/stats/StatsTimer.java @@ -0,0 +1,55 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.stats; + +import io.micrometer.core.instrument.Timer; +import lombok.Getter; + +import java.util.concurrent.TimeUnit; + +public class StatsTimer { + + @Getter + private final String name; + private final Timer timer; + + private int count; + private long totalTime; + + public StatsTimer(String name, Timer micrometerTimer) { + this.name = name; + this.timer = micrometerTimer; + } + + public void record(long timeMs) { + count++; + totalTime += timeMs; + timer.record(timeMs, TimeUnit.MILLISECONDS); + } + + public double getAvg() { + if (count == 0) { + return 0.0; + } + return (double) totalTime / count; + } + + public void reset() { + count = 0; + totalTime = 0; + } + +} diff --git a/common/stats/src/main/java/org/thingsboard/server/common/stats/StatsType.java b/common/stats/src/main/java/org/thingsboard/server/common/stats/StatsType.java index 318f0418de..6c854ac8d6 100644 --- a/common/stats/src/main/java/org/thingsboard/server/common/stats/StatsType.java +++ b/common/stats/src/main/java/org/thingsboard/server/common/stats/StatsType.java @@ -16,7 +16,12 @@ package org.thingsboard.server.common.stats; public enum StatsType { - RULE_ENGINE("ruleEngine"), CORE("core"), TRANSPORT("transport"), JS_INVOKE("jsInvoke"), RATE_EXECUTOR("rateExecutor"); + RULE_ENGINE("ruleEngine"), + CORE("core"), + TRANSPORT("transport"), + JS_INVOKE("jsInvoke"), + RATE_EXECUTOR("rateExecutor"), + HOUSEKEEPER("housekeeper"); private String name; diff --git a/common/util/src/main/java/org/thingsboard/common/util/ThingsBoardThreadFactory.java b/common/util/src/main/java/org/thingsboard/common/util/ThingsBoardThreadFactory.java index de097cfd6c..4d0876bf8e 100644 --- a/common/util/src/main/java/org/thingsboard/common/util/ThingsBoardThreadFactory.java +++ b/common/util/src/main/java/org/thingsboard/common/util/ThingsBoardThreadFactory.java @@ -51,6 +51,11 @@ public class ThingsBoardThreadFactory implements ThreadFactory { Thread.currentThread().setName(name); } + public static void addThreadNamePrefix(String prefix) { + String name = Thread.currentThread().getName(); + name = prefix + "-" + name; + Thread.currentThread().setName(name); + } @Override public Thread newThread(Runnable r) { diff --git a/dao/src/main/java/org/thingsboard/server/dao/Dao.java b/dao/src/main/java/org/thingsboard/server/dao/Dao.java index 8d408945fa..e7290cdced 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/Dao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/Dao.java @@ -43,6 +43,8 @@ public interface Dao { void removeAllByIds(Collection ids); + List findIdsByTenantIdAndIdOffset(TenantId tenantId, UUID idOffset, int limit); + default EntityType getEntityType() { return null; } } diff --git a/dao/src/main/java/org/thingsboard/server/dao/DaoUtil.java b/dao/src/main/java/org/thingsboard/server/dao/DaoUtil.java index 865306c73b..01a72ea58c 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/DaoUtil.java +++ b/dao/src/main/java/org/thingsboard/server/dao/DaoUtil.java @@ -18,6 +18,7 @@ package org.thingsboard.server.dao; import org.springframework.data.domain.Page; import org.springframework.data.domain.PageRequest; import org.springframework.data.domain.Pageable; +import org.springframework.data.domain.Slice; import org.springframework.util.CollectionUtils; import org.thingsboard.server.common.data.EntityInfo; import org.thingsboard.server.common.data.EntitySubtype; @@ -51,8 +52,17 @@ public abstract class DaoUtil { return new PageData<>(data, page.getTotalPages(), page.getTotalElements(), page.hasNext()); } - public static PageData pageToPageData(Page page) { - return new PageData<>(page.getContent(), page.getTotalPages(), page.getTotalElements(), page.hasNext()); + public static PageData pageToPageData(Slice slice) { + int totalPages; + long totalElements; + if (slice instanceof Page page) { + totalPages = page.getTotalPages(); + totalElements = page.getTotalElements(); + } else { + totalPages = 0; + totalElements = 0; + } + return new PageData<>(slice.getContent(), totalPages, totalElements, slice.hasNext()); } public static Pageable toPageable(PageLink pageLink) { diff --git a/dao/src/main/java/org/thingsboard/server/dao/alarm/AlarmDao.java b/dao/src/main/java/org/thingsboard/server/dao/alarm/AlarmDao.java index 19b02fc932..570256e98c 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/alarm/AlarmDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/alarm/AlarmDao.java @@ -38,6 +38,7 @@ import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.common.data.query.AlarmCountQuery; import org.thingsboard.server.common.data.query.AlarmData; import org.thingsboard.server.common.data.query.AlarmDataQuery; +import org.thingsboard.server.common.data.util.TbPair; import org.thingsboard.server.dao.Dao; import java.util.Collection; @@ -80,13 +81,15 @@ public interface AlarmDao extends Dao { PageData findAlarmIdsByAssigneeId(TenantId tenantId, UUID userId, PageLink pageLink); - PageData findAlarmIdsByOriginatorId(TenantId tenantId, EntityId originatorId, PageLink pageLink); + PageData> findAlarmIdsByOriginatorId(TenantId tenantId, EntityId originatorId, long createdTimeOffset, AlarmId idOffset, int limit); void createEntityAlarmRecord(EntityAlarm entityAlarm); List findEntityAlarmRecords(TenantId tenantId, AlarmId id); - void deleteEntityAlarmRecords(TenantId tenantId, EntityId entityId); + List findEntityAlarmRecordsByEntityId(TenantId tenantId, EntityId entityId); + + int deleteEntityAlarmRecords(TenantId tenantId, EntityId entityId); void deleteEntityAlarmRecordsByTenantId(TenantId tenantId); diff --git a/dao/src/main/java/org/thingsboard/server/dao/alarm/BaseAlarmService.java b/dao/src/main/java/org/thingsboard/server/dao/alarm/BaseAlarmService.java index d49cef22b7..f0fc18c827 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/alarm/BaseAlarmService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/alarm/BaseAlarmService.java @@ -58,6 +58,7 @@ import org.thingsboard.server.common.data.relation.EntityRelation; import org.thingsboard.server.common.data.relation.EntityRelationsQuery; import org.thingsboard.server.common.data.relation.EntitySearchDirection; import org.thingsboard.server.common.data.relation.RelationsSearchParameters; +import org.thingsboard.server.common.data.util.TbPair; import org.thingsboard.server.dao.entity.AbstractCachedEntityService; import org.thingsboard.server.dao.entity.EntityService; import org.thingsboard.server.dao.eventsourcing.ActionEntityEvent; @@ -75,6 +76,7 @@ import java.util.LinkedHashSet; import java.util.List; import java.util.Optional; import java.util.Set; +import java.util.UUID; import java.util.concurrent.ExecutionException; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -183,17 +185,23 @@ public class BaseAlarmService extends AbstractCachedEntityService findAlarmIdsByOriginatorId(TenantId tenantId, EntityId originatorId, PageLink pageLink) { - log.trace("[{}] Executing findAlarmsByOriginatorId [{}]", tenantId, originatorId); - return alarmDao.findAlarmIdsByOriginatorId(tenantId, originatorId, pageLink); + public List> findAlarmIdsByOriginatorId(TenantId tenantId, EntityId originatorId, long createdTimeOffset, AlarmId idOffset, int limit) { + log.trace("[{}] Executing findAlarmIdsByOriginatorIdAndIdOffset [{}][{}]", tenantId, originatorId, idOffset); + return alarmDao.findAlarmIdsByOriginatorId(tenantId, originatorId, createdTimeOffset, idOffset, limit).getData(); } @Override @@ -329,9 +337,9 @@ public class BaseAlarmService extends AbstractCachedEntityService INCORRECT_ASSET_PROFILE_ID + id); - AssetProfile assetProfile = assetProfileDao.findById(tenantId, assetProfileId.getId()); - if (assetProfile != null && assetProfile.isDefault()) { + deleteEntity(tenantId, assetProfileId, false); + } + + @Override + @Transactional + public void deleteEntity(TenantId tenantId, EntityId id, boolean force) { + AssetProfile assetProfile = assetProfileDao.findById(tenantId, id.getId()); + if (assetProfile == null) { + return; + } + if (!force && assetProfile.isDefault()) { throw new DataValidationException("Deletion of Default Asset Profile is prohibited!"); } - this.removeAssetProfile(tenantId, assetProfile); + removeAssetProfile(tenantId, assetProfile); } private void removeAssetProfile(TenantId tenantId, AssetProfile assetProfile) { AssetProfileId assetProfileId = assetProfile.getId(); try { - deleteEntityRelations(tenantId, assetProfileId); assetProfileDao.removeById(tenantId, assetProfileId.getId()); publishEvictEvent(new AssetProfileEvictEvent(assetProfile.getTenantId(), assetProfile.getName(), null, assetProfile.getId(), assetProfile.isDefault())); @@ -304,14 +312,13 @@ public class AssetProfileServiceImpl extends AbstractCachedEntityService> findEntity(TenantId tenantId, EntityId entityId) { - return Optional.ofNullable(findAssetProfileById(tenantId, new AssetProfileId(entityId.getId()))); + public void deleteByTenantId(TenantId tenantId) { + deleteAssetProfilesByTenantId(tenantId); } @Override - @Transactional - public void deleteEntity(TenantId tenantId, EntityId id) { - deleteAssetProfile(tenantId, (AssetProfileId) id); + public Optional> findEntity(TenantId tenantId, EntityId entityId) { + return Optional.ofNullable(findAssetProfileById(tenantId, new AssetProfileId(entityId.getId()))); } @Override diff --git a/dao/src/main/java/org/thingsboard/server/dao/asset/BaseAssetService.java b/dao/src/main/java/org/thingsboard/server/dao/asset/BaseAssetService.java index f7342e7634..d491956b25 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/asset/BaseAssetService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/asset/BaseAssetService.java @@ -199,19 +199,25 @@ public class BaseAssetService extends AbstractCachedEntityService INCORRECT_ASSET_ID + id); - if (entityViewService.existsByTenantIdAndEntityId(tenantId, assetId)) { + deleteEntity(tenantId, assetId, false); + } + + @Override + @Transactional + public void deleteEntity(TenantId tenantId, EntityId id, boolean force) { + if (!force && entityViewService.existsByTenantIdAndEntityId(tenantId, id)) { throw new DataValidationException("Can't delete asset that has entity views!"); } - Asset asset = assetDao.findById(tenantId, assetId.getId()); - alarmService.deleteEntityAlarmRelations(tenantId, assetId); + Asset asset = assetDao.findById(tenantId, id.getId()); + if (asset == null) { + return; + } deleteAsset(tenantId, asset); } private void deleteAsset(TenantId tenantId, Asset asset) { log.trace("Executing deleteAsset [{}]", asset.getId()); - relationService.deleteEntityRelations(tenantId, asset.getId()); - assetDao.removeById(tenantId, asset.getUuidId()); publishEvictEvent(new AssetCacheEvictEvent(asset.getTenantId(), asset.getName(), null)); @@ -277,6 +283,11 @@ public class BaseAssetService extends AbstractCachedEntityService findAssetsByTenantIdAndCustomerId(TenantId tenantId, CustomerId customerId, PageLink pageLink) { log.trace("Executing findAssetsByTenantIdAndCustomerId, tenantId [{}], customerId [{}], pageLink [{}]", tenantId, customerId, pageLink); @@ -471,12 +482,6 @@ public class BaseAssetService extends AbstractCachedEntityService findAllKeysByEntityIds(TenantId tenantId, List entityIds); List findAllKeysByEntityIdsAndAttributeType(TenantId tenantId, List entityIds, String attributeType); + + List> removeAllByEntityId(TenantId tenantId, EntityId entityId); + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/attributes/BaseAttributesService.java b/dao/src/main/java/org/thingsboard/server/dao/attributes/BaseAttributesService.java index 5869ad9261..1b1f2d005d 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/attributes/BaseAttributesService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/attributes/BaseAttributesService.java @@ -18,6 +18,7 @@ package org.thingsboard.server.dao.attributes; import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.tuple.Pair; import org.springframework.beans.factory.annotation.Value; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.context.annotation.Primary; @@ -160,4 +161,11 @@ public class BaseAttributesService implements AttributesService { validate(entityId, scope); return Futures.allAsList(attributesDao.removeAll(tenantId, entityId, scope, attributeKeys)); } + + @Override + public int removeAllByEntityId(TenantId tenantId, EntityId entityId) { + List> deleted = attributesDao.removeAllByEntityId(tenantId, entityId); + return deleted.size(); + } + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/attributes/CachedAttributesService.java b/dao/src/main/java/org/thingsboard/server/dao/attributes/CachedAttributesService.java index dcd13fb168..f51bc6f0b8 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/attributes/CachedAttributesService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/attributes/CachedAttributesService.java @@ -19,7 +19,9 @@ import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; import com.google.common.util.concurrent.ListeningExecutorService; import com.google.common.util.concurrent.MoreExecutors; +import jakarta.annotation.PostConstruct; import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.tuple.Pair; import org.springframework.beans.factory.annotation.Value; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.context.annotation.Primary; @@ -39,7 +41,6 @@ import org.thingsboard.server.dao.cache.CacheExecutorService; import org.thingsboard.server.dao.service.Validator; import org.thingsboard.server.dao.sql.JpaExecutorService; -import jakarta.annotation.PostConstruct; import java.util.ArrayList; import java.util.Collection; import java.util.HashMap; @@ -158,47 +159,47 @@ public class CachedAttributesService implements AttributesService { return Futures.transformAsync(cacheExecutor.submit(() -> findCachedAttributes(entityId, scope, attributeKeys)), wrappedCachedAttributes -> { - List cachedAttributes = wrappedCachedAttributes.values().stream() - .map(TbCacheValueWrapper::get) - .filter(Objects::nonNull) - .collect(Collectors.toList()); - if (wrappedCachedAttributes.size() == attributeKeys.size()) { - log.trace("[{}][{}] Found all attributes from cache: {}", entityId, scope, attributeKeys); - return Futures.immediateFuture(cachedAttributes); - } - - Set notFoundAttributeKeys = new HashSet<>(attributeKeys); - notFoundAttributeKeys.removeAll(wrappedCachedAttributes.keySet()); - - List notFoundKeys = notFoundAttributeKeys.stream().map(k -> new AttributeCacheKey(scope, entityId, k)).collect(Collectors.toList()); - - // DB call should run in DB executor, not in cache-related executor - return jpaExecutorService.submit(() -> { - var cacheTransaction = cache.newTransactionForKeys(notFoundKeys); - try { - log.trace("[{}][{}] Lookup attributes from db: {}", entityId, scope, notFoundAttributeKeys); - List result = attributesDao.find(tenantId, entityId, scope, notFoundAttributeKeys); - for (AttributeKvEntry foundInDbAttribute : result) { - AttributeCacheKey attributeCacheKey = new AttributeCacheKey(scope, entityId, foundInDbAttribute.getKey()); - cacheTransaction.putIfAbsent(attributeCacheKey, foundInDbAttribute); - notFoundAttributeKeys.remove(foundInDbAttribute.getKey()); - } - for (String key : notFoundAttributeKeys) { - cacheTransaction.putIfAbsent(new AttributeCacheKey(scope, entityId, key), null); - } - List mergedAttributes = new ArrayList<>(cachedAttributes); - mergedAttributes.addAll(result); - cacheTransaction.commit(); - log.trace("[{}][{}] Commit cache transaction: {}", entityId, scope, notFoundAttributeKeys); - return mergedAttributes; - } catch (Throwable e) { - cacheTransaction.rollback(); - log.debug("Could not find attributes from cache: [{}] [{}] [{}]", entityId, scope, notFoundAttributeKeys, e); - throw e; - } - }); - - }, MoreExecutors.directExecutor()); // cacheExecutor analyse and returns results or submit to DB executor + List cachedAttributes = wrappedCachedAttributes.values().stream() + .map(TbCacheValueWrapper::get) + .filter(Objects::nonNull) + .collect(Collectors.toList()); + if (wrappedCachedAttributes.size() == attributeKeys.size()) { + log.trace("[{}][{}] Found all attributes from cache: {}", entityId, scope, attributeKeys); + return Futures.immediateFuture(cachedAttributes); + } + + Set notFoundAttributeKeys = new HashSet<>(attributeKeys); + notFoundAttributeKeys.removeAll(wrappedCachedAttributes.keySet()); + + List notFoundKeys = notFoundAttributeKeys.stream().map(k -> new AttributeCacheKey(scope, entityId, k)).collect(Collectors.toList()); + + // DB call should run in DB executor, not in cache-related executor + return jpaExecutorService.submit(() -> { + var cacheTransaction = cache.newTransactionForKeys(notFoundKeys); + try { + log.trace("[{}][{}] Lookup attributes from db: {}", entityId, scope, notFoundAttributeKeys); + List result = attributesDao.find(tenantId, entityId, scope, notFoundAttributeKeys); + for (AttributeKvEntry foundInDbAttribute : result) { + AttributeCacheKey attributeCacheKey = new AttributeCacheKey(scope, entityId, foundInDbAttribute.getKey()); + cacheTransaction.putIfAbsent(attributeCacheKey, foundInDbAttribute); + notFoundAttributeKeys.remove(foundInDbAttribute.getKey()); + } + for (String key : notFoundAttributeKeys) { + cacheTransaction.putIfAbsent(new AttributeCacheKey(scope, entityId, key), null); + } + List mergedAttributes = new ArrayList<>(cachedAttributes); + mergedAttributes.addAll(result); + cacheTransaction.commit(); + log.trace("[{}][{}] Commit cache transaction: {}", entityId, scope, notFoundAttributeKeys); + return mergedAttributes; + } catch (Throwable e) { + cacheTransaction.rollback(); + log.debug("Could not find attributes from cache: [{}] [{}] [{}]", entityId, scope, notFoundAttributeKeys, e); + throw e; + } + }); + + }, MoreExecutors.directExecutor()); // cacheExecutor analyse and returns results or submit to DB executor } private Map> findCachedAttributes(EntityId entityId, AttributeScope scope, Collection attributeKeys) { @@ -305,4 +306,17 @@ public class CachedAttributesService implements AttributesService { }, cacheExecutor)).collect(Collectors.toList())); } + @Override + public int removeAllByEntityId(TenantId tenantId, EntityId entityId) { + List> result = attributesDao.removeAllByEntityId(tenantId, entityId); + result.forEach(deleted -> { + AttributeScope scope = deleted.getKey(); + String key = deleted.getValue(); + if (scope != null && key != null) { + cache.evict(new AttributeCacheKey(scope, entityId, key)); + } + }); + return result.size(); + } + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/customer/CustomerServiceImpl.java b/dao/src/main/java/org/thingsboard/server/dao/customer/CustomerServiceImpl.java index 04308cbb2e..e5f84b5e2c 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/customer/CustomerServiceImpl.java +++ b/dao/src/main/java/org/thingsboard/server/dao/customer/CustomerServiceImpl.java @@ -127,9 +127,20 @@ public class CustomerServiceImpl extends AbstractEntityService implements Custom public void deleteCustomer(TenantId tenantId, CustomerId customerId) { log.trace("Executing deleteCustomer [{}]", customerId); Validator.validateId(customerId, id -> INCORRECT_CUSTOMER_ID + id); + deleteEntity(tenantId, customerId, false); + } + + @Transactional + @Override + public void deleteEntity(TenantId tenantId, EntityId id, boolean force) { + CustomerId customerId = (CustomerId) id; Customer customer = findCustomerById(tenantId, customerId); if (customer == null) { - throw new IncorrectParameterException("Unable to delete non-existent customer."); + if (force) { + return; + } else { + throw new IncorrectParameterException("Unable to delete non-existent customer."); + } } dashboardService.unassignCustomerDashboards(tenantId, customerId); entityViewService.unassignCustomerEntityViews(customer.getTenantId(), customerId); @@ -137,7 +148,6 @@ public class CustomerServiceImpl extends AbstractEntityService implements Custom deviceService.unassignCustomerDevices(customer.getTenantId(), customerId); edgeService.unassignCustomerEdges(customer.getTenantId(), customerId); userService.deleteCustomerUsers(customer.getTenantId(), customerId); - deleteEntityRelations(tenantId, customerId); apiUsageStateService.deleteApiUsageStateByEntityId(customerId); customerDao.removeById(tenantId, customerId.getId()); countService.publishCountEntityEvictEvent(tenantId, EntityType.CUSTOMER); @@ -181,6 +191,11 @@ public class CustomerServiceImpl extends AbstractEntityService implements Custom customersByTenantRemover.removeEntities(tenantId, tenantId); } + @Override + public void deleteByTenantId(TenantId tenantId) { + deleteCustomersByTenantId(tenantId); + } + private PaginatedRemover customersByTenantRemover = new PaginatedRemover() { @@ -200,12 +215,6 @@ public class CustomerServiceImpl extends AbstractEntityService implements Custom return Optional.ofNullable(findCustomerById(tenantId, new CustomerId(entityId.getId()))); } - @Transactional - @Override - public void deleteEntity(TenantId tenantId, EntityId id) { - deleteCustomer(tenantId, (CustomerId) id); - } - @Override public long countByTenantId(TenantId tenantId) { return customerDao.countByTenantId(tenantId); diff --git a/dao/src/main/java/org/thingsboard/server/dao/dashboard/DashboardServiceImpl.java b/dao/src/main/java/org/thingsboard/server/dao/dashboard/DashboardServiceImpl.java index 7f90d82af7..167ae51fc3 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/dashboard/DashboardServiceImpl.java +++ b/dao/src/main/java/org/thingsboard/server/dao/dashboard/DashboardServiceImpl.java @@ -230,7 +230,6 @@ public class DashboardServiceImpl extends AbstractEntityService implements Dashb public void deleteDashboard(TenantId tenantId, DashboardId dashboardId) { log.trace("Executing deleteDashboard [{}]", dashboardId); Validator.validateId(dashboardId, id -> INCORRECT_DASHBOARD_ID + id); - deleteEntityRelations(tenantId, dashboardId); try { dashboardDao.removeById(tenantId, dashboardId.getId()); publishEvictEvent(new DashboardTitleEvictEvent(dashboardId)); @@ -246,6 +245,12 @@ public class DashboardServiceImpl extends AbstractEntityService implements Dashb } } + @Override + @Transactional + public void deleteEntity(TenantId tenantId, EntityId id, boolean force) { + deleteDashboard(tenantId, (DashboardId) id); + } + @Override public PageData findDashboardsByTenantId(TenantId tenantId, PageLink pageLink) { log.trace("Executing findDashboardsByTenantId, tenantId [{}], pageLink [{}]", tenantId, pageLink); @@ -269,6 +274,11 @@ public class DashboardServiceImpl extends AbstractEntityService implements Dashb tenantDashboardsRemover.removeEntities(tenantId, tenantId); } + @Override + public void deleteByTenantId(TenantId tenantId) { + deleteDashboardsByTenantId(tenantId); + } + @Override public PageData findDashboardsByTenantIdAndCustomerId(TenantId tenantId, CustomerId customerId, PageLink pageLink) { log.trace("Executing findDashboardsByTenantIdAndCustomerId, tenantId [{}], customerId [{}], pageLink [{}]", tenantId, customerId, pageLink); @@ -390,12 +400,6 @@ public class DashboardServiceImpl extends AbstractEntityService implements Dashb return dashboardDao.countByTenantId(tenantId); } - @Override - @Transactional - public void deleteEntity(TenantId tenantId, EntityId id) { - deleteDashboard(tenantId, (DashboardId) id); - } - @Override public EntityType getEntityType() { return EntityType.DASHBOARD; diff --git a/dao/src/main/java/org/thingsboard/server/dao/device/DeviceProfileServiceImpl.java b/dao/src/main/java/org/thingsboard/server/dao/device/DeviceProfileServiceImpl.java index 7e6d79d9a7..7a0f7a56c4 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/device/DeviceProfileServiceImpl.java +++ b/dao/src/main/java/org/thingsboard/server/dao/device/DeviceProfileServiceImpl.java @@ -18,7 +18,6 @@ package org.thingsboard.server.dao.device; import lombok.extern.slf4j.Slf4j; import org.hibernate.exception.ConstraintViolationException; import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.context.annotation.Lazy; import org.springframework.stereotype.Service; import org.springframework.transaction.annotation.Transactional; import org.springframework.transaction.event.TransactionalEventListener; @@ -47,7 +46,6 @@ import org.thingsboard.server.dao.entity.AbstractCachedEntityService; import org.thingsboard.server.dao.eventsourcing.DeleteEntityEvent; import org.thingsboard.server.dao.eventsourcing.SaveEntityEvent; import org.thingsboard.server.dao.exception.DataValidationException; -import org.thingsboard.server.dao.queue.QueueService; import org.thingsboard.server.dao.resource.ImageService; import org.thingsboard.server.dao.service.DataValidator; import org.thingsboard.server.dao.service.PaginatedRemover; @@ -91,10 +89,6 @@ public class DeviceProfileServiceImpl extends AbstractCachedEntityService deviceProfileValidator; - @Lazy - @Autowired - private QueueService queueService; - @Autowired private ImageService imageService; @@ -223,17 +217,25 @@ public class DeviceProfileServiceImpl extends AbstractCachedEntityService INCORRECT_DEVICE_PROFILE_ID + id); - DeviceProfile deviceProfile = deviceProfileDao.findById(tenantId, deviceProfileId.getId()); - if (deviceProfile != null && deviceProfile.isDefault()) { + deleteEntity(tenantId, deviceProfileId, false); + } + + @Override + @Transactional + public void deleteEntity(TenantId tenantId, EntityId id, boolean force) { + DeviceProfile deviceProfile = deviceProfileDao.findById(tenantId, id.getId()); + if (deviceProfile == null) { + return; + } + if (!force && deviceProfile.isDefault()) { throw new DataValidationException("Deletion of Default Device Profile is prohibited!"); } - this.removeDeviceProfile(tenantId, deviceProfile); + removeDeviceProfile(tenantId, deviceProfile); } private void removeDeviceProfile(TenantId tenantId, DeviceProfile deviceProfile) { DeviceProfileId deviceProfileId = deviceProfile.getId(); try { - deleteEntityRelations(tenantId, deviceProfileId); deviceProfileDao.removeById(tenantId, deviceProfileId.getId()); publishEvictEvent(new DeviceProfileEvictEvent(deviceProfile.getTenantId(), deviceProfile.getName(), null, deviceProfile.getId(), deviceProfile.isDefault(), @@ -359,14 +361,13 @@ public class DeviceProfileServiceImpl extends AbstractCachedEntityService> findEntity(TenantId tenantId, EntityId entityId) { - return Optional.ofNullable(findDeviceProfileById(tenantId, new DeviceProfileId(entityId.getId()))); + public void deleteByTenantId(TenantId tenantId) { + deleteDeviceProfilesByTenantId(tenantId); } @Override - @Transactional - public void deleteEntity(TenantId tenantId, EntityId id) { - deleteDeviceProfile(tenantId, (DeviceProfileId) id); + public Optional> findEntity(TenantId tenantId, EntityId entityId) { + return Optional.ofNullable(findDeviceProfileById(tenantId, new DeviceProfileId(entityId.getId()))); } @Override diff --git a/dao/src/main/java/org/thingsboard/server/dao/device/DeviceServiceImpl.java b/dao/src/main/java/org/thingsboard/server/dao/device/DeviceServiceImpl.java index ff00db3564..a616c14a69 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/device/DeviceServiceImpl.java +++ b/dao/src/main/java/org/thingsboard/server/dao/device/DeviceServiceImpl.java @@ -322,20 +322,26 @@ public class DeviceServiceImpl extends AbstractCachedEntityService INCORRECT_DEVICE_ID + id); - if (entityViewService.existsByTenantIdAndEntityId(tenantId, deviceId)) { + deleteEntity(tenantId, deviceId, false); + } + + @Override + @Transactional + public void deleteEntity(TenantId tenantId, EntityId id, boolean force) { + if (!force && entityViewService.existsByTenantIdAndEntityId(tenantId, id)) { throw new DataValidationException("Can't delete device that has entity views!"); } - Device device = deviceDao.findById(tenantId, deviceId.getId()); - alarmService.deleteEntityAlarmRelations(tenantId, deviceId); + Device device = deviceDao.findById(tenantId, id.getId()); + if (device == null) { + return; + } deleteDevice(tenantId, device); } private void deleteDevice(TenantId tenantId, Device device) { log.trace("Executing deleteDevice [{}]", device.getId()); deviceCredentialsService.deleteDeviceCredentialsByDeviceId(tenantId, device.getId()); - relationService.deleteEntityRelations(tenantId, device.getId()); - deviceDao.removeById(tenantId, device.getUuidId()); DeviceCacheEvictEvent deviceCacheEvictEvent = new DeviceCacheEvictEvent(device.getTenantId(), device.getId(), device.getName(), null); @@ -347,12 +353,11 @@ public class DeviceServiceImpl extends AbstractCachedEntityService findDevicesByTenantId(TenantId tenantId, PageLink pageLink) { log.trace("Executing findDevicesByTenantId, tenantId [{}], pageLink [{}]", tenantId, pageLink); - validateId(tenantId, id ->INCORRECT_TENANT_ID + id); + validateId(tenantId, id -> INCORRECT_TENANT_ID + id); validatePageLink(pageLink); return deviceDao.findDevicesByTenantId(tenantId.getId(), pageLink); } - @Override public PageData findDeviceInfosByFilter(DeviceInfoFilter filter, PageLink pageLink) { log.trace("Executing findDeviceInfosByFilter, filter [{}], pageLink [{}]", filter, pageLink); @@ -413,14 +418,14 @@ public class DeviceServiceImpl extends AbstractCachedEntityService findDevicesByIds(List deviceIds) { log.trace("Executing findDevicesByIdsAsync, deviceIds [{}]", deviceIds); - validateIds(deviceIds, ids-> "Incorrect deviceIds " + ids); + validateIds(deviceIds, ids -> "Incorrect deviceIds " + ids); return deviceDao.findDevicesByIds(toUUIDs(deviceIds)); } @Override public ListenableFuture> findDevicesByIdsAsync(List deviceIds) { log.trace("Executing findDevicesByIdsAsync, deviceIds [{}]", deviceIds); - validateIds(deviceIds, ids-> "Incorrect deviceIds " + ids); + validateIds(deviceIds, ids -> "Incorrect deviceIds " + ids); return deviceDao.findDevicesByIdsAsync(toUUIDs(deviceIds)); } @@ -432,6 +437,12 @@ public class DeviceServiceImpl extends AbstractCachedEntityService findDevicesByTenantIdAndCustomerId(TenantId tenantId, CustomerId customerId, PageLink pageLink) { log.trace("Executing findDevicesByTenantIdAndCustomerId, tenantId [{}], customerId [{}], pageLink [{}]", tenantId, customerId, pageLink); @@ -648,12 +659,6 @@ public class DeviceServiceImpl extends AbstractCachedEntityService tenantDevicesRemover = new PaginatedRemover<>() { @Override diff --git a/dao/src/main/java/org/thingsboard/server/dao/edge/EdgeServiceImpl.java b/dao/src/main/java/org/thingsboard/server/dao/edge/EdgeServiceImpl.java index 94aa3380ee..66cf74ba9c 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/edge/EdgeServiceImpl.java +++ b/dao/src/main/java/org/thingsboard/server/dao/edge/EdgeServiceImpl.java @@ -224,15 +224,21 @@ public class EdgeServiceImpl extends AbstractCachedEntityService INCORRECT_EDGE_ID + id); Edge edge = edgeDao.findById(tenantId, edgeId.getId()); - - deleteEntityRelations(tenantId, edgeId); - + if (edge == null) { + return; + } edgeDao.removeById(tenantId, edgeId.getId()); eventPublisher.publishEvent(DeleteEntityEvent.builder().tenantId(tenantId).entityId(edgeId).build()); publishEvictEvent(new EdgeCacheEvictEvent(edge.getTenantId(), edge.getName(), null)); } + @Override + @Transactional + public void deleteEntity(TenantId tenantId, EntityId id, boolean force) { + deleteEdge(tenantId, (EdgeId) id); + } + @Override public PageData findEdgesByTenantId(TenantId tenantId, PageLink pageLink) { log.trace("Executing findEdgesByTenantId, tenantId [{}], pageLink [{}]", tenantId, pageLink); @@ -282,6 +288,11 @@ public class EdgeServiceImpl extends AbstractCachedEntityService findEdgesByTenantIdAndCustomerId(TenantId tenantId, CustomerId customerId, PageLink pageLink) { log.trace("Executing findEdgesByTenantIdAndCustomerId, tenantId [{}], customerId [{}], pageLink [{}]", tenantId, customerId, pageLink); diff --git a/dao/src/main/java/org/thingsboard/server/dao/entity/AbstractEntityService.java b/dao/src/main/java/org/thingsboard/server/dao/entity/AbstractEntityService.java index 0baa497cbf..6b88f19699 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/entity/AbstractEntityService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/entity/AbstractEntityService.java @@ -31,6 +31,7 @@ import org.thingsboard.server.dao.alarm.AlarmService; import org.thingsboard.server.dao.edge.EdgeService; import org.thingsboard.server.dao.entityview.EntityViewService; import org.thingsboard.server.dao.exception.DataValidationException; +import org.thingsboard.server.dao.housekeeper.CleanUpService; import org.thingsboard.server.dao.relation.RelationService; import java.util.Collections; @@ -63,6 +64,10 @@ public abstract class AbstractEntityService { @Autowired(required = false) protected EdgeService edgeService; + @Autowired + @Lazy + protected CleanUpService cleanUpService; + protected void createRelation(TenantId tenantId, EntityRelation relation) { log.debug("Creating relation: {}", relation); relationService.saveRelation(tenantId, relation); @@ -73,11 +78,6 @@ public abstract class AbstractEntityService { relationService.deleteRelation(tenantId, relation); } - protected void deleteEntityRelations(TenantId tenantId, EntityId entityId) { - relationService.deleteEntityRelations(tenantId, entityId); - alarmService.deleteEntityAlarmRelations(tenantId, entityId); - } - protected static Optional extractConstraintViolationException(Exception t) { if (t instanceof ConstraintViolationException) { return Optional.of((ConstraintViolationException) t); diff --git a/dao/src/main/java/org/thingsboard/server/dao/entity/DefaultEntityServiceRegistry.java b/dao/src/main/java/org/thingsboard/server/dao/entity/DefaultEntityServiceRegistry.java index 8202d49db8..364d3a12f3 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/entity/DefaultEntityServiceRegistry.java +++ b/dao/src/main/java/org/thingsboard/server/dao/entity/DefaultEntityServiceRegistry.java @@ -24,6 +24,7 @@ import org.thingsboard.server.common.data.EntityType; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Optional; @Service @RequiredArgsConstructor @@ -48,7 +49,8 @@ public class DefaultEntityServiceRegistry implements EntityServiceRegistry { @Override public EntityDaoService getServiceByEntityType(EntityType entityType) { - return entityDaoServicesMap.get(entityType); + return Optional.ofNullable(entityDaoServicesMap.get(entityType)) + .orElseThrow(() -> new IllegalArgumentException("Unsupported entity type " + entityType)); } } diff --git a/dao/src/main/java/org/thingsboard/server/dao/entity/EntityDaoRegistry.java b/dao/src/main/java/org/thingsboard/server/dao/entity/EntityDaoRegistry.java new file mode 100644 index 0000000000..7b2f2d42c1 --- /dev/null +++ b/dao/src/main/java/org/thingsboard/server/dao/entity/EntityDaoRegistry.java @@ -0,0 +1,51 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao.entity; + +import lombok.extern.slf4j.Slf4j; +import org.springframework.stereotype.Service; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.dao.Dao; + +import java.util.EnumMap; +import java.util.List; +import java.util.Map; + +@Service +@Slf4j +public class EntityDaoRegistry { + + private final Map> daos = new EnumMap<>(EntityType.class); + + private EntityDaoRegistry(List> daos) { + daos.forEach(dao -> { + EntityType entityType = dao.getEntityType(); + if (entityType != null) { + this.daos.put(entityType, dao); + } + }); + } + + @SuppressWarnings("unchecked") + public Dao getDao(EntityType entityType) { + Dao dao = (Dao) daos.get(entityType); + if (dao == null) { + throw new IllegalArgumentException("Missing dao for entity type " + entityType); + } + return dao; + } + +} diff --git a/dao/src/main/java/org/thingsboard/server/dao/entityview/EntityViewServiceImpl.java b/dao/src/main/java/org/thingsboard/server/dao/entityview/EntityViewServiceImpl.java index 9d3c33dcc2..e56055b614 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/entityview/EntityViewServiceImpl.java +++ b/dao/src/main/java/org/thingsboard/server/dao/entityview/EntityViewServiceImpl.java @@ -327,13 +327,21 @@ public class EntityViewServiceImpl extends AbstractCachedEntityService INCORRECT_ENTITY_VIEW_ID + id); - deleteEntityRelations(tenantId, entityViewId); EntityView entityView = entityViewDao.findById(tenantId, entityViewId.getId()); + if (entityView == null) { + return; + } entityViewDao.removeById(tenantId, entityViewId.getId()); publishEvictEvent(new EntityViewEvictEvent(entityView.getTenantId(), entityView.getId(), entityView.getEntityId(), null, entityView.getName(), null)); eventPublisher.publishEvent(DeleteEntityEvent.builder().tenantId(tenantId).entityId(entityViewId).build()); } + @Override + @Transactional + public void deleteEntity(TenantId tenantId, EntityId id, boolean force) { + deleteEntityView(tenantId, (EntityViewId) id); + } + @Override public void deleteEntityViewsByTenantId(TenantId tenantId) { log.trace("Executing deleteEntityViewsByTenantId, tenantId [{}]", tenantId); @@ -341,6 +349,11 @@ public class EntityViewServiceImpl extends AbstractCachedEntityService> findEntityViewTypesByTenantId(TenantId tenantId) { log.trace("Executing findEntityViewTypesByTenantId, tenantId [{}]", tenantId); diff --git a/dao/src/main/java/org/thingsboard/server/dao/housekeeper/CleanUpService.java b/dao/src/main/java/org/thingsboard/server/dao/housekeeper/CleanUpService.java new file mode 100644 index 0000000000..d64e6ea536 --- /dev/null +++ b/dao/src/main/java/org/thingsboard/server/dao/housekeeper/CleanUpService.java @@ -0,0 +1,91 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao.housekeeper; + +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.stereotype.Component; +import org.springframework.transaction.annotation.Propagation; +import org.springframework.transaction.annotation.Transactional; +import org.springframework.transaction.event.TransactionalEventListener; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.User; +import org.thingsboard.server.common.data.housekeeper.HousekeeperTask; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.msg.housekeeper.HousekeeperClient; +import org.thingsboard.server.dao.eventsourcing.DeleteEntityEvent; +import org.thingsboard.server.dao.relation.RelationService; + +import java.util.EnumSet; +import java.util.Optional; +import java.util.Set; + +@Component +@RequiredArgsConstructor +@Slf4j +public class CleanUpService { + + private final Optional housekeeperClient; + private final RelationService relationService; + + private final Set skippedEntities = EnumSet.of( + EntityType.ALARM, EntityType.QUEUE, EntityType.TB_RESOURCE, EntityType.OTA_PACKAGE, + EntityType.NOTIFICATION_REQUEST, EntityType.NOTIFICATION_TEMPLATE, + EntityType.NOTIFICATION_TARGET, EntityType.NOTIFICATION_RULE + ); + + @TransactionalEventListener(fallbackExecution = true) // after transaction commit + @Transactional(propagation = Propagation.NOT_SUPPORTED) + public void handleEntityDeletionEvent(DeleteEntityEvent event) { + TenantId tenantId = event.getTenantId(); + EntityId entityId = event.getEntityId(); + EntityType entityType = entityId.getEntityType(); + try { + log.trace("[{}][{}][{}] Handling entity deletion event", tenantId, entityType, entityId.getId()); + if (!skippedEntities.contains(entityType)) { + cleanUpRelatedData(tenantId, entityId); + } + if (entityType == EntityType.USER) { + submitTask(HousekeeperTask.unassignAlarms((User) event.getEntity())); + } + } catch (Throwable e) { + log.error("[{}][{}][{}] Failed to handle entity deletion event", tenantId, entityType, entityId.getId(), e); + } + } + + public void cleanUpRelatedData(TenantId tenantId, EntityId entityId) { + log.debug("[{}][{}][{}] Cleaning up related data", tenantId, entityId.getEntityType(), entityId.getId()); + relationService.deleteEntityRelations(tenantId, entityId); + submitTask(HousekeeperTask.deleteAttributes(tenantId, entityId)); + submitTask(HousekeeperTask.deleteTelemetry(tenantId, entityId)); + submitTask(HousekeeperTask.deleteEvents(tenantId, entityId)); + submitTask(HousekeeperTask.deleteAlarms(tenantId, entityId)); + } + + public void removeTenantEntities(TenantId tenantId, EntityType... entityTypes) { + for (EntityType entityType : entityTypes) { + submitTask(HousekeeperTask.deleteTenantEntities(tenantId, entityType)); + } + } + + private void submitTask(HousekeeperTask task) { + housekeeperClient.ifPresent(housekeeperClient -> { + housekeeperClient.submitTask(task); + }); + } + +} diff --git a/dao/src/main/java/org/thingsboard/server/dao/notification/DefaultNotificationRequestService.java b/dao/src/main/java/org/thingsboard/server/dao/notification/DefaultNotificationRequestService.java index a77be093db..ebeb39ed91 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/notification/DefaultNotificationRequestService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/notification/DefaultNotificationRequestService.java @@ -93,6 +93,16 @@ public class DefaultNotificationRequestService implements NotificationRequestSer eventPublisher.publishEvent(DeleteEntityEvent.builder().tenantId(tenantId).entity(request).entityId(request.getId()).build()); } + @Override + public void deleteEntity(TenantId tenantId, EntityId id, boolean force) { + if (force) { + notificationRequestDao.removeById(tenantId, id.getId()); + } else { + NotificationRequest notificationRequest = findNotificationRequestById(tenantId, (NotificationRequestId) id); + deleteNotificationRequest(tenantId, notificationRequest); + } + } + @Override public PageData findScheduledNotificationRequests(PageLink pageLink) { return notificationRequestDao.findAllByStatus(NotificationRequestStatus.SCHEDULED, pageLink); @@ -109,6 +119,11 @@ public class DefaultNotificationRequestService implements NotificationRequestSer notificationRequestDao.removeByTenantId(tenantId); } + @Override + public void deleteByTenantId(TenantId tenantId) { + deleteNotificationRequestsByTenantId(tenantId); + } + @Override public Optional> findEntity(TenantId tenantId, EntityId entityId) { return Optional.ofNullable(findNotificationRequestById(tenantId, new NotificationRequestId(entityId.getId()))); diff --git a/dao/src/main/java/org/thingsboard/server/dao/notification/DefaultNotificationRuleService.java b/dao/src/main/java/org/thingsboard/server/dao/notification/DefaultNotificationRuleService.java index 7dbecbb725..ccf8d83887 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/notification/DefaultNotificationRuleService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/notification/DefaultNotificationRuleService.java @@ -94,19 +94,24 @@ public class DefaultNotificationRuleService extends AbstractEntityService implem eventPublisher.publishEvent(DeleteEntityEvent.builder().tenantId(tenantId).entityId(id).build()); } + @Override + public void deleteEntity(TenantId tenantId, EntityId id, boolean force) { + deleteNotificationRuleById(tenantId, (NotificationRuleId) id); + } + @Override public void deleteNotificationRulesByTenantId(TenantId tenantId) { notificationRuleDao.removeByTenantId(tenantId); } @Override - public Optional> findEntity(TenantId tenantId, EntityId entityId) { - return Optional.ofNullable(findNotificationRuleById(tenantId, new NotificationRuleId(entityId.getId()))); + public void deleteByTenantId(TenantId tenantId) { + deleteNotificationRulesByTenantId(tenantId); } @Override - public void deleteEntity(TenantId tenantId, EntityId id) { - deleteNotificationRuleById(tenantId, (NotificationRuleId) id); + public Optional> findEntity(TenantId tenantId, EntityId entityId) { + return Optional.ofNullable(findNotificationRuleById(tenantId, new NotificationRuleId(entityId.getId()))); } @Override diff --git a/dao/src/main/java/org/thingsboard/server/dao/notification/DefaultNotificationSettingsService.java b/dao/src/main/java/org/thingsboard/server/dao/notification/DefaultNotificationSettingsService.java index 5b6f464ea6..10438c484a 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/notification/DefaultNotificationSettingsService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/notification/DefaultNotificationSettingsService.java @@ -172,17 +172,15 @@ public class DefaultNotificationSettingsService implements NotificationSettingsS defaultNotifications.create(tenantId, DefaultNotifications.entitiesLimitForSysadmin, sysAdmins.getId()); defaultNotifications.create(tenantId, DefaultNotifications.entitiesLimitForTenant, affectedTenantAdmins.getId()); - defaultNotifications.create(tenantId, DefaultNotifications.apiFeatureWarningForSysadmin, sysAdmins.getId()); defaultNotifications.create(tenantId, DefaultNotifications.apiFeatureWarningForTenant, affectedTenantAdmins.getId()); defaultNotifications.create(tenantId, DefaultNotifications.apiFeatureDisabledForSysadmin, sysAdmins.getId()); defaultNotifications.create(tenantId, DefaultNotifications.apiFeatureDisabledForTenant, affectedTenantAdmins.getId()); - defaultNotifications.create(tenantId, DefaultNotifications.exceededRateLimits, affectedTenantAdmins.getId()); defaultNotifications.create(tenantId, DefaultNotifications.exceededPerEntityRateLimits, affectedTenantAdmins.getId()); defaultNotifications.create(tenantId, DefaultNotifications.exceededRateLimitsForSysadmin, sysAdmins.getId()); - defaultNotifications.create(tenantId, DefaultNotifications.newPlatformVersion, sysAdmins.getId()); + defaultNotifications.create(tenantId, DefaultNotifications.taskProcessingFailure, tenantAdmins.getId()); return; } @@ -205,19 +203,19 @@ public class DefaultNotificationSettingsService implements NotificationSettingsS @Override public void updateDefaultNotificationConfigs(TenantId tenantId) { if (tenantId.isSysTenantId()) { - if (notificationTemplateService.findNotificationTemplatesByTenantIdAndNotificationTypes(tenantId, - List.of(NotificationType.RATE_LIMITS), new PageLink(1)).getTotalElements() > 0) { - return; - } - NotificationTarget sysAdmins = notificationTargetService.findNotificationTargetsByTenantIdAndUsersFilterType(tenantId, UsersFilterType.SYSTEM_ADMINISTRATORS).stream() .findFirst().orElseGet(() -> createTarget(tenantId, "System administrators", new SystemAdministratorsFilter(), "All system administrators")); NotificationTarget affectedTenantAdmins = notificationTargetService.findNotificationTargetsByTenantIdAndUsersFilterType(tenantId, UsersFilterType.AFFECTED_TENANT_ADMINISTRATORS).stream() .findFirst().orElseGet(() -> createTarget(tenantId, "Affected tenant's administrators", new AffectedTenantAdministratorsFilter(), "")); - defaultNotifications.create(tenantId, DefaultNotifications.exceededRateLimits, affectedTenantAdmins.getId()); - defaultNotifications.create(tenantId, DefaultNotifications.exceededPerEntityRateLimits, affectedTenantAdmins.getId()); - defaultNotifications.create(tenantId, DefaultNotifications.exceededRateLimitsForSysadmin, sysAdmins.getId()); + if (!isNotificationConfigured(tenantId, NotificationType.RATE_LIMITS)) { + defaultNotifications.create(tenantId, DefaultNotifications.exceededRateLimits, affectedTenantAdmins.getId()); + defaultNotifications.create(tenantId, DefaultNotifications.exceededPerEntityRateLimits, affectedTenantAdmins.getId()); + defaultNotifications.create(tenantId, DefaultNotifications.exceededRateLimitsForSysadmin, sysAdmins.getId()); + } + if (!isNotificationConfigured(tenantId, NotificationType.TASK_PROCESSING_FAILURE)) { + defaultNotifications.create(tenantId, DefaultNotifications.taskProcessingFailure, sysAdmins.getId()); + } } else { var requiredNotificationTypes = List.of(NotificationType.EDGE_CONNECTION, NotificationType.EDGE_COMMUNICATION_FAILURE); var existingNotificationTypes = notificationTemplateService.findNotificationTemplatesByTenantIdAndNotificationTypes( @@ -251,6 +249,10 @@ public class DefaultNotificationSettingsService implements NotificationSettingsS } } + private boolean isNotificationConfigured(TenantId tenantId, NotificationType... notificationTypes) { + return notificationTemplateService.countNotificationTemplatesByTenantIdAndNotificationTypes(tenantId, List.of(notificationTypes)) > 0; + } + private NotificationTarget createTarget(TenantId tenantId, String name, UsersFilter filter, String description) { NotificationTarget target = new NotificationTarget(); target.setTenantId(tenantId); diff --git a/dao/src/main/java/org/thingsboard/server/dao/notification/DefaultNotificationTargetService.java b/dao/src/main/java/org/thingsboard/server/dao/notification/DefaultNotificationTargetService.java index 94c339335b..a6ab6e08ff 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/notification/DefaultNotificationTargetService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/notification/DefaultNotificationTargetService.java @@ -193,10 +193,16 @@ public class DefaultNotificationTargetService extends AbstractEntityService impl @Override public void deleteNotificationTargetById(TenantId tenantId, NotificationTargetId id) { - if (notificationRequestDao.existsByTenantIdAndStatusAndTargetId(tenantId, NotificationRequestStatus.SCHEDULED, id)) { + deleteEntity(tenantId, id, false); + } + + @Override + public void deleteEntity(TenantId tenantId, EntityId id, boolean force) { + NotificationTargetId targetId = (NotificationTargetId) id; + if (!force && notificationRequestDao.existsByTenantIdAndStatusAndTargetId(tenantId, NotificationRequestStatus.SCHEDULED, targetId)) { throw new IllegalArgumentException("Recipients group is referenced by scheduled notification request"); } - if (notificationRuleDao.existsByTenantIdAndTargetId(tenantId, id)) { + if (!force && notificationRuleDao.existsByTenantIdAndTargetId(tenantId, targetId)) { throw new IllegalArgumentException("Recipients group is being used in notification rule"); } notificationTargetDao.removeById(tenantId, id.getId()); @@ -208,6 +214,11 @@ public class DefaultNotificationTargetService extends AbstractEntityService impl notificationTargetDao.removeByTenantId(tenantId); } + @Override + public void deleteByTenantId(TenantId tenantId) { + deleteNotificationTargetsByTenantId(tenantId); + } + @Override public long countNotificationTargetsByTenantId(TenantId tenantId) { return notificationTargetDao.countByTenantId(tenantId); @@ -218,11 +229,6 @@ public class DefaultNotificationTargetService extends AbstractEntityService impl return Optional.ofNullable(findNotificationTargetById(tenantId, new NotificationTargetId(entityId.getId()))); } - @Override - public void deleteEntity(TenantId tenantId, EntityId id) { - deleteNotificationTargetById(tenantId, (NotificationTargetId) id); - } - @Override public EntityType getEntityType() { return EntityType.NOTIFICATION_TARGET; diff --git a/dao/src/main/java/org/thingsboard/server/dao/notification/DefaultNotificationTemplateService.java b/dao/src/main/java/org/thingsboard/server/dao/notification/DefaultNotificationTemplateService.java index 58ec572b7f..14b9d176df 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/notification/DefaultNotificationTemplateService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/notification/DefaultNotificationTemplateService.java @@ -74,9 +74,19 @@ public class DefaultNotificationTemplateService extends AbstractEntityService im return notificationTemplateDao.findByTenantIdAndNotificationTypesAndPageLink(tenantId, notificationTypes, pageLink); } + @Override + public int countNotificationTemplatesByTenantIdAndNotificationTypes(TenantId tenantId, List notificationTypes) { + return notificationTemplateDao.countByTenantIdAndNotificationTypes(tenantId, notificationTypes); + } + @Override public void deleteNotificationTemplateById(TenantId tenantId, NotificationTemplateId id) { - if (notificationRequestDao.existsByTenantIdAndStatusAndTemplateId(tenantId, NotificationRequestStatus.SCHEDULED, id)) { + deleteEntity(tenantId, id, false); + } + + @Override + public void deleteEntity(TenantId tenantId, EntityId id, boolean force) { + if (!force && notificationRequestDao.existsByTenantIdAndStatusAndTemplateId(tenantId, NotificationRequestStatus.SCHEDULED, (NotificationTemplateId) id)) { throw new IllegalArgumentException("Notification template is referenced by scheduled notification request"); } try { @@ -96,13 +106,13 @@ public class DefaultNotificationTemplateService extends AbstractEntityService im } @Override - public Optional> findEntity(TenantId tenantId, EntityId entityId) { - return Optional.ofNullable(findNotificationTemplateById(tenantId, new NotificationTemplateId(entityId.getId()))); + public void deleteByTenantId(TenantId tenantId) { + deleteNotificationTemplatesByTenantId(tenantId); } @Override - public void deleteEntity(TenantId tenantId, EntityId id) { - deleteNotificationTemplateById(tenantId, (NotificationTemplateId) id); + public Optional> findEntity(TenantId tenantId, EntityId entityId) { + return Optional.ofNullable(findNotificationTemplateById(tenantId, new NotificationTemplateId(entityId.getId()))); } @Override diff --git a/dao/src/main/java/org/thingsboard/server/dao/notification/DefaultNotifications.java b/dao/src/main/java/org/thingsboard/server/dao/notification/DefaultNotifications.java index fd17618bd1..4f617c2b9d 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/notification/DefaultNotifications.java +++ b/dao/src/main/java/org/thingsboard/server/dao/notification/DefaultNotifications.java @@ -50,6 +50,7 @@ import org.thingsboard.server.common.data.notification.rule.trigger.config.Notif import org.thingsboard.server.common.data.notification.rule.trigger.config.NotificationRuleTriggerType; import org.thingsboard.server.common.data.notification.rule.trigger.config.RateLimitsNotificationRuleTriggerConfig; import org.thingsboard.server.common.data.notification.rule.trigger.config.RuleEngineComponentLifecycleEventNotificationRuleTriggerConfig; +import org.thingsboard.server.common.data.notification.rule.trigger.config.TaskProcessingFailureNotificationRuleTriggerConfig; import org.thingsboard.server.common.data.notification.template.NotificationTemplate; import org.thingsboard.server.common.data.notification.template.NotificationTemplateConfig; import org.thingsboard.server.common.data.notification.template.WebDeliveryMethodNotificationTemplate; @@ -358,6 +359,19 @@ public class DefaultNotifications { .build()) .build(); + public static final DefaultNotification taskProcessingFailure = DefaultNotification.builder() + .name("Task processing failure notification") + .type(NotificationType.TASK_PROCESSING_FAILURE) + .subject("Failed to process ${taskType}") + .text("Failed to process ${taskDescription} for tenant ${tenantId}: ${error}") + .icon("warning").color(YELLOW_COLOR) + .rule(DefaultRule.builder() + .name("Task processing failure") + .triggerConfig(TaskProcessingFailureNotificationRuleTriggerConfig.builder().build()) + .description("Send notification to system admins when task processing fails") + .build()) + .build(); + public static final DefaultNotification jwtSigningKeyIssue = DefaultNotification.builder() .name("JWT Signing Key issue notification") .type(NotificationType.GENERAL) diff --git a/dao/src/main/java/org/thingsboard/server/dao/notification/NotificationTemplateDao.java b/dao/src/main/java/org/thingsboard/server/dao/notification/NotificationTemplateDao.java index d9a80ce3f5..136bc4d472 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/notification/NotificationTemplateDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/notification/NotificationTemplateDao.java @@ -30,6 +30,8 @@ public interface NotificationTemplateDao extends Dao, Expo PageData findByTenantIdAndNotificationTypesAndPageLink(TenantId tenantId, List notificationTypes, PageLink pageLink); + int countByTenantIdAndNotificationTypes(TenantId tenantId, List notificationTypes); + void removeByTenantId(TenantId tenantId); } diff --git a/dao/src/main/java/org/thingsboard/server/dao/ota/BaseOtaPackageService.java b/dao/src/main/java/org/thingsboard/server/dao/ota/BaseOtaPackageService.java index b2fa1a3788..06c449880d 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/ota/BaseOtaPackageService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/ota/BaseOtaPackageService.java @@ -218,6 +218,11 @@ public class BaseOtaPackageService extends AbstractCachedEntityService tenantOtaPackageRemover = new PaginatedRemover<>() { diff --git a/dao/src/main/java/org/thingsboard/server/dao/queue/BaseQueueService.java b/dao/src/main/java/org/thingsboard/server/dao/queue/BaseQueueService.java index d930ab7249..e3c1ef68aa 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/queue/BaseQueueService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/queue/BaseQueueService.java @@ -83,6 +83,11 @@ public class BaseQueueService extends AbstractEntityService implements QueueServ } } + @Override + public void deleteEntity(TenantId tenantId, EntityId id, boolean force) { + deleteQueue(tenantId, (QueueId) id); + } + @Override public List findQueuesByTenantId(TenantId tenantId) { log.trace("Executing findQueues, tenantId: [{}]", tenantId); @@ -126,6 +131,11 @@ public class BaseQueueService extends AbstractEntityService implements QueueServ tenantQueuesRemover.removeEntities(tenantId, tenantId); } + @Override + public void deleteByTenantId(TenantId tenantId) { + deleteQueuesByTenantId(tenantId); + } + @Override public Optional> findEntity(TenantId tenantId, EntityId entityId) { return Optional.ofNullable(findQueueById(tenantId, new QueueId(entityId.getId()))); diff --git a/dao/src/main/java/org/thingsboard/server/dao/queue/BaseQueueStatsService.java b/dao/src/main/java/org/thingsboard/server/dao/queue/BaseQueueStatsService.java index 2274a782bb..6352ac1a4c 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/queue/BaseQueueStatsService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/queue/BaseQueueStatsService.java @@ -78,6 +78,11 @@ public class BaseQueueStatsService extends AbstractEntityService implements Queu queueStatsDao.deleteByTenantId(tenantId); } + @Override + public void deleteEntity(TenantId tenantId, EntityId id, boolean force) { + queueStatsDao.removeById(tenantId, id.getId()); + } + @Override public Optional> findEntity(TenantId tenantId, EntityId entityId) { return Optional.ofNullable(findQueueStatsById(tenantId, new QueueStatsId(entityId.getId()))); @@ -87,4 +92,5 @@ public class BaseQueueStatsService extends AbstractEntityService implements Queu public EntityType getEntityType() { return EntityType.QUEUE_STATS; } + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/resource/BaseResourceService.java b/dao/src/main/java/org/thingsboard/server/dao/resource/BaseResourceService.java index 37b030996f..723d0e8861 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/resource/BaseResourceService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/resource/BaseResourceService.java @@ -159,6 +159,11 @@ public class BaseResourceService extends AbstractCachedEntityService findAllTenantResourcesByTenantId(TbResourceInfoFilter filter, PageLink pageLink) { TenantId tenantId = filter.getTenantId(); @@ -204,13 +209,13 @@ public class BaseResourceService extends AbstractCachedEntityService> findEntity(TenantId tenantId, EntityId entityId) { - return Optional.ofNullable(findResourceInfoById(tenantId, new TbResourceId(entityId.getId()))); + public void deleteByTenantId(TenantId tenantId) { + deleteResourcesByTenantId(tenantId); } @Override - public void deleteEntity(TenantId tenantId, EntityId id) { - deleteResource(tenantId, (TbResourceId) id); + public Optional> findEntity(TenantId tenantId, EntityId entityId) { + return Optional.ofNullable(findResourceInfoById(tenantId, new TbResourceId(entityId.getId()))); } @Override diff --git a/dao/src/main/java/org/thingsboard/server/dao/rpc/BaseRpcService.java b/dao/src/main/java/org/thingsboard/server/dao/rpc/BaseRpcService.java index 2b314e82df..c79de9c6ac 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/rpc/BaseRpcService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/rpc/BaseRpcService.java @@ -59,6 +59,11 @@ public class BaseRpcService implements RpcService { rpcDao.removeById(tenantId, rpcId.getId()); } + @Override + public void deleteEntity(TenantId tenantId, EntityId id, boolean force) { + deleteRpc(tenantId, (RpcId) id); + } + @Override public void deleteAllRpcByTenantId(TenantId tenantId) { log.trace("Executing deleteAllRpcByTenantId, tenantId [{}]", tenantId); @@ -66,6 +71,11 @@ public class BaseRpcService implements RpcService { tenantRpcRemover.removeEntities(tenantId, tenantId); } + @Override + public void deleteByTenantId(TenantId tenantId) { + deleteAllRpcByTenantId(tenantId); + } + @Override public Rpc findById(TenantId tenantId, RpcId rpcId) { log.trace("Executing findById, tenantId [{}], rpcId [{}]", tenantId, rpcId); diff --git a/dao/src/main/java/org/thingsboard/server/dao/rule/BaseRuleChainService.java b/dao/src/main/java/org/thingsboard/server/dao/rule/BaseRuleChainService.java index ae3929f5fb..bebb3891cb 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/rule/BaseRuleChainService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/rule/BaseRuleChainService.java @@ -197,7 +197,6 @@ public class BaseRuleChainService extends AbstractEntityService implements RuleC List updatedRuleNodes = new ArrayList<>(); List existingRuleNodes = getRuleChainNodes(tenantId, ruleChainMetaData.getRuleChainId()); for (RuleNode existingNode : existingRuleNodes) { - deleteEntityRelations(tenantId, existingNode.getId()); Integer index = ruleNodeIndexMap.get(existingNode.getId()); RuleNode newRuleNode = null; if (index != null) { @@ -452,12 +451,30 @@ public class BaseRuleChainService extends AbstractEntityService implements RuleC checkRuleNodesAndDelete(tenantId, ruleChain, referencingRuleChainIds); } + @Override + @Transactional + public void deleteEntity(TenantId tenantId, EntityId id, boolean force) { + if (force) { + RuleChain ruleChain = findRuleChainById(tenantId, (RuleChainId) id); + checkRuleNodesAndDelete(tenantId, ruleChain, null); + } else { + deleteRuleChainById(tenantId, (RuleChainId) id); + } + } + + @Transactional @Override public void deleteRuleChainsByTenantId(TenantId tenantId) { Validator.validateId(tenantId, "Incorrect tenant id for delete rule chains request."); tenantRuleChainsRemover.removeEntities(tenantId, tenantId); } + @Transactional + @Override + public void deleteByTenantId(TenantId tenantId) { + deleteRuleChainsByTenantId(tenantId); + } + @Override public RuleChainData exportTenantRuleChains(TenantId tenantId, PageLink pageLink) { Validator.validateId(tenantId, "Incorrect tenant id for search rule chain request."); @@ -789,10 +806,10 @@ public class BaseRuleChainService extends AbstractEntityService implements RuleC private void deleteRuleNodes(TenantId tenantId, List ruleNodes) { List ruleNodeIds = ruleNodes.stream().map(RuleNode::getId).collect(Collectors.toList()); - for (var node : ruleNodes) { - deleteEntityRelations(tenantId, node.getId()); - } ruleNodeDao.deleteByIdIn(ruleNodeIds); + for (var nodeId : ruleNodeIds) { + cleanUpService.cleanUpRelatedData(tenantId, nodeId); + } } @Override @@ -802,7 +819,6 @@ public class BaseRuleChainService extends AbstractEntityService implements RuleC for (EntityRelation relation : nodeRelations) { deleteRuleNode(tenantId, relation.getTo()); } - deleteEntityRelations(tenantId, ruleChainId); } @@ -819,12 +835,6 @@ public class BaseRuleChainService extends AbstractEntityService implements RuleC return ruleChainDao.countByTenantId(tenantId); } - @Override - @Transactional - public void deleteEntity(TenantId tenantId, EntityId id) { - deleteRuleChainById(tenantId, (RuleChainId) id); - } - @Override public EntityType getEntityType() { return EntityType.RULE_CHAIN; @@ -839,8 +849,8 @@ public class BaseRuleChainService extends AbstractEntityService implements RuleC } private void deleteRuleNode(TenantId tenantId, EntityId entityId) { - deleteEntityRelations(tenantId, entityId); ruleNodeDao.removeById(tenantId, entityId.getId()); + cleanUpService.cleanUpRelatedData(tenantId, entityId); } private final PaginatedRemover tenantRuleChainsRemover = diff --git a/dao/src/main/java/org/thingsboard/server/dao/settings/AdminSettingsServiceImpl.java b/dao/src/main/java/org/thingsboard/server/dao/settings/AdminSettingsServiceImpl.java index 13aaba8751..da0ca9c577 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/settings/AdminSettingsServiceImpl.java +++ b/dao/src/main/java/org/thingsboard/server/dao/settings/AdminSettingsServiceImpl.java @@ -29,7 +29,7 @@ import org.thingsboard.server.dao.service.Validator; @Service @Slf4j public class AdminSettingsServiceImpl implements AdminSettingsService { - + @Autowired private AdminSettingsDao adminSettingsDao; @@ -40,7 +40,7 @@ public class AdminSettingsServiceImpl implements AdminSettingsService { public AdminSettings findAdminSettingsById(TenantId tenantId, AdminSettingsId adminSettingsId) { log.trace("Executing findAdminSettingsById [{}]", adminSettingsId); Validator.validateId(adminSettingsId, id -> "Incorrect adminSettingsId " + id); - return adminSettingsDao.findById(tenantId, adminSettingsId.getId()); + return adminSettingsDao.findById(tenantId, adminSettingsId.getId()); } @Override @@ -59,15 +59,15 @@ public class AdminSettingsServiceImpl implements AdminSettingsService { public AdminSettings saveAdminSettings(TenantId tenantId, AdminSettings adminSettings) { log.trace("Executing saveAdminSettings [{}]", adminSettings); adminSettingsValidator.validate(adminSettings, data -> tenantId); - if (adminSettings.getKey().equals("mail")){ + if (adminSettings.getKey().equals("mail")) { AdminSettings mailSettings = findAdminSettingsByKey(tenantId, "mail"); if (mailSettings != null) { JsonNode newJsonValue = adminSettings.getJsonValue(); JsonNode oldJsonValue = mailSettings.getJsonValue(); - if (!newJsonValue.has("password") && oldJsonValue.has("password")){ - ((ObjectNode) newJsonValue).put("password", oldJsonValue.get("password").asText()); + if (!newJsonValue.has("password") && oldJsonValue.has("password")) { + ((ObjectNode) newJsonValue).put("password", oldJsonValue.get("password").asText()); } - if (!newJsonValue.has("refreshToken") && oldJsonValue.has("refreshToken")){ + if (!newJsonValue.has("refreshToken") && oldJsonValue.has("refreshToken")) { ((ObjectNode) newJsonValue).put("refreshToken", oldJsonValue.get("refreshToken").asText()); } dropTokenIfProviderInfoChanged(newJsonValue, oldJsonValue); @@ -92,12 +92,12 @@ public class AdminSettingsServiceImpl implements AdminSettingsService { } private void dropTokenIfProviderInfoChanged(JsonNode newJsonValue, JsonNode oldJsonValue) { - if (newJsonValue.has("enableOauth2") && newJsonValue.get("enableOauth2").asBoolean()){ + if (newJsonValue.has("enableOauth2") && newJsonValue.get("enableOauth2").asBoolean()) { if (!newJsonValue.get("providerId").equals(oldJsonValue.get("providerId")) || !newJsonValue.get("clientId").equals(oldJsonValue.get("clientId")) || !newJsonValue.get("clientSecret").equals(oldJsonValue.get("clientSecret")) || !newJsonValue.get("redirectUri").equals(oldJsonValue.get("redirectUri")) || - (newJsonValue.has("providerTenantId") && !newJsonValue.get("providerTenantId").equals(oldJsonValue.get("providerTenantId")))){ + (newJsonValue.has("providerTenantId") && !newJsonValue.get("providerTenantId").equals(oldJsonValue.get("providerTenantId")))) { ((ObjectNode) newJsonValue).put("tokenGenerated", false); ((ObjectNode) newJsonValue).remove("refreshToken"); ((ObjectNode) newJsonValue).remove("refreshTokenExpires"); diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/JpaAbstractDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/JpaAbstractDao.java index fdb96195fb..973cb4b5b2 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/JpaAbstractDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/JpaAbstractDao.java @@ -19,12 +19,15 @@ import com.datastax.oss.driver.api.core.uuid.Uuids; import com.google.common.collect.Lists; import com.google.common.util.concurrent.ListenableFuture; import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Autowired; import org.springframework.data.jpa.repository.JpaRepository; +import org.springframework.jdbc.core.JdbcTemplate; import org.springframework.transaction.annotation.Transactional; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.dao.Dao; import org.thingsboard.server.dao.DaoUtil; import org.thingsboard.server.dao.model.BaseEntity; +import org.thingsboard.server.dao.model.ModelConstants; import org.thingsboard.server.dao.util.SqlDao; import java.util.Collection; @@ -41,6 +44,9 @@ public abstract class JpaAbstractDao, D> extends JpaAbstractDaoListeningExecutorService implements Dao { + @Autowired + protected JdbcTemplate jdbcTemplate; + protected abstract Class getEntityClass(); protected abstract JpaRepository getRepository(); @@ -123,4 +129,23 @@ public abstract class JpaAbstractDao, D> return DaoUtil.convertDataList(entities); } + @Override + public List findIdsByTenantIdAndIdOffset(TenantId tenantId, UUID idOffset, int limit) { + String query = "SELECT id FROM " + getEntityType().getTableName() + " WHERE " + getTenantIdColumn() + " = ? "; + Object[] params; + if (idOffset == null) { + params = new Object[]{tenantId.getId(), limit}; + } else { + query += " AND id > ? "; + params = new Object[]{tenantId.getId(), idOffset, limit}; + } + query += " ORDER BY id LIMIT ?"; + + return jdbcTemplate.queryForList(query, UUID.class, params); + } + + protected String getTenantIdColumn() { + return ModelConstants.TENANT_ID_COLUMN; + } + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/alarm/AlarmRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/alarm/AlarmRepository.java index 735234ffaa..8353334017 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/alarm/AlarmRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/alarm/AlarmRepository.java @@ -17,13 +17,14 @@ package org.thingsboard.server.dao.sql.alarm; import org.springframework.data.domain.Page; import org.springframework.data.domain.Pageable; +import org.springframework.data.domain.Slice; import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.data.jpa.repository.Modifying; import org.springframework.data.jpa.repository.Query; import org.springframework.data.repository.query.Param; import org.springframework.transaction.annotation.Transactional; import org.thingsboard.server.common.data.alarm.AlarmSeverity; -import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.util.TbPair; import org.thingsboard.server.dao.model.sql.AlarmEntity; import org.thingsboard.server.dao.model.sql.AlarmInfoEntity; @@ -333,8 +334,21 @@ public interface AlarmRepository extends JpaRepository { @Query("SELECT a.id FROM AlarmEntity a WHERE a.tenantId = :tenantId AND a.assigneeId = :assigneeId") Page findAlarmIdsByAssigneeId(@Param("tenantId") UUID tenantId, @Param("assigneeId") UUID assigneeId, Pageable pageable); - @Query("SELECT a.id FROM AlarmEntity a WHERE a.tenantId = :tenantId AND a.originatorId = :originatorId") - Page findAlarmIdsByOriginatorId(@Param("tenantId") UUID tenantId, @Param("originatorId") UUID originatorId, Pageable pageable); + // using Slice so that count query is not executed + @Query("SELECT new org.thingsboard.server.common.data.util.TbPair(a.id, a.createdTime) " + + "FROM AlarmEntity a WHERE a.originatorId = :originatorId " + + "AND (a.createdTime > :createdTimeOffset OR " + + "(a.createdTime = :createdTimeOffset AND a.id > :idOffset))") + Slice> findAlarmIdsByOriginatorId(@Param("originatorId") UUID originatorId, + @Param("createdTimeOffset") long createdTimeOffset, + @Param("idOffset") UUID idOffset, + Pageable pageable); + + // using Slice so that count query is not executed + @Query("SELECT new org.thingsboard.server.common.data.util.TbPair(a.id, a.createdTime) " + + "FROM AlarmEntity a WHERE a.originatorId = :originatorId") + Slice> findAlarmIdsByOriginatorId(@Param("originatorId") UUID originatorId, + Pageable pageable); @Query(value = "SELECT create_or_update_active_alarm(:t_id, :c_id, :a_id, :a_created_ts, :a_o_id, :a_o_type, :a_type, :a_severity, " + ":a_start_ts, :a_end_ts, :a_details, :a_propagate, :a_propagate_to_owner, " + diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/alarm/EntityAlarmRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/alarm/EntityAlarmRepository.java index 8373999758..ccf82e1b5b 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/alarm/EntityAlarmRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/alarm/EntityAlarmRepository.java @@ -33,11 +33,13 @@ public interface EntityAlarmRepository extends JpaRepository findAllByEntityId(UUID entityId); + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/alarm/JpaAlarmDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/alarm/JpaAlarmDao.java index 90a5b1f8d4..b0bf2cc666 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/alarm/JpaAlarmDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/alarm/JpaAlarmDao.java @@ -21,6 +21,8 @@ import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.data.domain.Page; import org.springframework.data.domain.PageRequest; +import org.springframework.data.domain.Pageable; +import org.springframework.data.domain.Slice; import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.stereotype.Component; import org.springframework.util.CollectionUtils; @@ -48,9 +50,11 @@ import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.id.UserId; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; +import org.thingsboard.server.common.data.page.SortOrder; import org.thingsboard.server.common.data.query.AlarmCountQuery; import org.thingsboard.server.common.data.query.AlarmData; import org.thingsboard.server.common.data.query.AlarmDataQuery; +import org.thingsboard.server.common.data.util.TbPair; import org.thingsboard.server.dao.DaoUtil; import org.thingsboard.server.dao.alarm.AlarmDao; import org.thingsboard.server.dao.model.ModelConstants; @@ -69,6 +73,7 @@ import java.util.Optional; import java.util.Set; import java.util.UUID; +import static org.thingsboard.server.common.data.page.SortOrder.Direction.ASC; import static org.thingsboard.server.dao.DaoUtil.convertTenantEntityTypesToDto; import static org.thingsboard.server.dao.DaoUtil.toPageable; @@ -297,9 +302,15 @@ public class JpaAlarmDao extends JpaAbstractDao implements A } @Override - public PageData findAlarmIdsByOriginatorId(TenantId tenantId, EntityId originatorId, PageLink pageLink) { - return DaoUtil.pageToPageData(alarmRepository.findAlarmIdsByOriginatorId(tenantId.getId(), originatorId.getId(), DaoUtil.toPageable(pageLink))) - .mapData(AlarmId::new); + public PageData> findAlarmIdsByOriginatorId(TenantId tenantId, EntityId originatorId, long createdTimeOffset, AlarmId idOffset, int limit) { + Slice> result; + Pageable pageRequest = toPageable(new PageLink(limit), List.of(SortOrder.of("createdTime", ASC), SortOrder.of("id", ASC))); + if (idOffset == null) { + result = alarmRepository.findAlarmIdsByOriginatorId(originatorId.getId(), pageRequest); + } else { + result = alarmRepository.findAlarmIdsByOriginatorId(originatorId.getId(), createdTimeOffset, idOffset.getId(), pageRequest); + } + return DaoUtil.pageToPageData(result); } @Override @@ -315,8 +326,13 @@ public class JpaAlarmDao extends JpaAbstractDao implements A } @Override - public void deleteEntityAlarmRecords(TenantId tenantId, EntityId entityId) { - entityAlarmRepository.deleteByEntityId(entityId.getId()); + public List findEntityAlarmRecordsByEntityId(TenantId tenantId, EntityId entityId) { + return DaoUtil.convertDataList(entityAlarmRepository.findAllByEntityId(entityId.getId())); + } + + @Override + public int deleteEntityAlarmRecords(TenantId tenantId, EntityId entityId) { + return entityAlarmRepository.deleteByEntityId(entityId.getId()); } @Override diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/attributes/JpaAttributeDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/attributes/JpaAttributeDao.java index 39358ab5fd..3bf8e43c70 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/attributes/JpaAttributeDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/attributes/JpaAttributeDao.java @@ -22,9 +22,11 @@ import com.google.common.util.concurrent.MoreExecutors; import jakarta.annotation.PostConstruct; import jakarta.annotation.PreDestroy; import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.tuple.Pair; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Component; +import org.springframework.transaction.annotation.Transactional; import org.thingsboard.server.common.data.AttributeScope; import org.thingsboard.server.common.data.id.DeviceProfileId; import org.thingsboard.server.common.data.id.EntityId; @@ -34,6 +36,7 @@ import org.thingsboard.server.common.stats.StatsFactory; import org.thingsboard.server.dao.DaoUtil; import org.thingsboard.server.dao.attributes.AttributesDao; import org.thingsboard.server.dao.dictionary.KeyDictionaryDao; +import org.thingsboard.server.dao.model.ModelConstants; import org.thingsboard.server.dao.model.sql.AttributeKvCompositeKey; import org.thingsboard.server.dao.model.sql.AttributeKvEntity; import org.thingsboard.server.dao.sql.JpaAbstractDaoListeningExecutorService; @@ -146,8 +149,7 @@ public class JpaAttributeDao extends JpaAbstractDaoListeningExecutorService impl entityId.getId(), attributeScope.getId()); attributes.forEach(attributeKvEntity -> attributeKvEntity.setStrKey(keyDictionaryDao.getKey(attributeKvEntity.getId().getAttributeKey()))); - return DaoUtil.convertDataList(Lists.newArrayList( - attributes)); + return DaoUtil.convertDataList(Lists.newArrayList(attributes)); } @Override @@ -204,6 +206,16 @@ public class JpaAttributeDao extends JpaAbstractDaoListeningExecutorService impl return futuresList; } + @Transactional + @Override + public List> removeAllByEntityId(TenantId tenantId, EntityId entityId) { + return jdbcTemplate.queryForList("DELETE FROM attribute_kv WHERE entity_id = ? " + + "RETURNING attribute_type, attribute_key", entityId.getId()).stream() + .map(row -> Pair.of(AttributeScope.valueOf((Integer) row.get(ModelConstants.ATTRIBUTE_TYPE_COLUMN)), + keyDictionaryDao.getKey((Integer) row.get(ModelConstants.ATTRIBUTE_KEY_COLUMN)))) + .collect(Collectors.toList()); + } + private AttributeKvCompositeKey getAttributeKvCompositeKey(EntityId entityId, Integer attributeType, Integer attributeKey) { return new AttributeKvCompositeKey( entityId.getId(), diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/notification/JpaNotificationTemplateDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/notification/JpaNotificationTemplateDao.java index 8ee6c3df3a..8950ab1e35 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/notification/JpaNotificationTemplateDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/notification/JpaNotificationTemplateDao.java @@ -53,6 +53,11 @@ public class JpaNotificationTemplateDao extends JpaAbstractDao notificationTypes) { + return notificationTemplateRepository.countByTenantIdAndNotificationTypes(tenantId.getId(), notificationTypes); + } + @Override public void removeByTenantId(TenantId tenantId) { notificationTemplateRepository.deleteByTenantId(tenantId.getId()); diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/notification/NotificationTemplateRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/notification/NotificationTemplateRepository.java index b3e416a491..bb45c10d06 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/notification/NotificationTemplateRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/notification/NotificationTemplateRepository.java @@ -42,6 +42,11 @@ public interface NotificationTemplateRepository extends JpaRepository notificationTypes); + @Transactional @Modifying @Query("DELETE FROM NotificationTemplateEntity t WHERE t.tenantId = :tenantId") diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/queue/JpaQueueDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/queue/JpaQueueDao.java index 9e0723d9fc..799d1ff521 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/queue/JpaQueueDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/queue/JpaQueueDao.java @@ -21,6 +21,7 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.stereotype.Component; import org.thingsboard.server.common.data.DataConstants; +import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; @@ -86,4 +87,10 @@ public class JpaQueueDao extends JpaAbstractDao implements Q return DaoUtil.toPageData(queueRepository .findByTenantId(tenantId.getId(), pageLink.getTextSearch(), DaoUtil.toPageable(pageLink))); } -} \ No newline at end of file + + @Override + public EntityType getEntityType() { + return EntityType.QUEUE; + } + +} diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/queue/JpaQueueStatsDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/queue/JpaQueueStatsDao.java index ac57d54e90..1cfdbf0b38 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/queue/JpaQueueStatsDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/queue/JpaQueueStatsDao.java @@ -19,6 +19,7 @@ import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.stereotype.Component; +import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.queue.QueueStats; import org.thingsboard.server.dao.DaoUtil; @@ -63,4 +64,9 @@ public class JpaQueueStatsDao extends JpaAbstractDao findByTypeAndPath(TenantId tenantId, UserSettingsType type, String... path) { log.trace("findByTypeAndPath [{}][{}][{}]", tenantId, type, path); diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/user/UserSettingsRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/user/UserSettingsRepository.java index 7f50b5ad5b..436062ef4c 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/user/UserSettingsRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/user/UserSettingsRepository.java @@ -16,15 +16,23 @@ package org.thingsboard.server.dao.sql.user; import org.springframework.data.jpa.repository.JpaRepository; +import org.springframework.data.jpa.repository.Modifying; import org.springframework.data.jpa.repository.Query; import org.springframework.data.repository.query.Param; +import org.springframework.transaction.annotation.Transactional; import org.thingsboard.server.common.data.settings.UserSettingsCompositeKey; import org.thingsboard.server.dao.model.sql.UserSettingsEntity; import java.util.List; +import java.util.UUID; public interface UserSettingsRepository extends JpaRepository { + @Transactional + @Modifying + @Query("DELETE FROM UserSettingsEntity s WHERE s.userId = :userId") + void deleteByUserId(@Param("userId") UUID userId); + @Query(value = "SELECT * FROM user_settings WHERE type = :type AND (settings #> :path) IS NOT NULL", nativeQuery = true) List findByTypeAndPathExisting(@Param("type") String type, @Param("path") String[] path); diff --git a/dao/src/main/java/org/thingsboard/server/dao/tenant/TenantProfileServiceImpl.java b/dao/src/main/java/org/thingsboard/server/dao/tenant/TenantProfileServiceImpl.java index 7a6f5cd8f8..acbc7041fb 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/tenant/TenantProfileServiceImpl.java +++ b/dao/src/main/java/org/thingsboard/server/dao/tenant/TenantProfileServiceImpl.java @@ -133,7 +133,6 @@ public class TenantProfileServiceImpl extends AbstractCachedEntityService tenantValidator; - - @Lazy - @Autowired - private QueueService queueService; - - @Lazy - @Autowired - private QueueStatsService queueStatsService; - @Autowired private AdminSettingsService adminSettingsService; - @Autowired private NotificationSettingsService notificationSettingsService; - - @Autowired - private NotificationRequestService notificationRequestService; - - @Autowired - private NotificationRuleService notificationRuleService; - - @Autowired - private NotificationTemplateService notificationTemplateService; - @Autowired - private NotificationTargetService notificationTargetService; - + private TenantDataValidator tenantValidator; @Autowired protected TbTransactionalCache existsTenantCache; @@ -226,46 +150,27 @@ public class TenantServiceImpl extends AbstractCachedEntityService INCORRECT_TENANT_ID + id); - entityViewService.deleteEntityViewsByTenantId(tenantId); - widgetsBundleService.deleteWidgetsBundlesByTenantId(tenantId); - widgetTypeService.deleteWidgetTypesByTenantId(tenantId); - assetService.deleteAssetsByTenantId(tenantId); - assetProfileService.deleteAssetProfilesByTenantId(tenantId); - deviceService.deleteDevicesByTenantId(tenantId); - deviceProfileService.deleteDeviceProfilesByTenantId(tenantId); - dashboardService.deleteDashboardsByTenantId(tenantId); - customerService.deleteCustomersByTenantId(tenantId); - edgeService.deleteEdgesByTenantId(tenantId); - userService.deleteTenantAdmins(tenantId); - ruleChainService.deleteRuleChainsByTenantId(tenantId); - apiUsageStateService.deleteApiUsageStateByTenantId(tenantId); - resourceService.deleteResourcesByTenantId(tenantId); - otaPackageService.deleteOtaPackagesByTenantId(tenantId); - rpcService.deleteAllRpcByTenantId(tenantId); - queueService.deleteQueuesByTenantId(tenantId); - notificationRequestService.deleteNotificationRequestsByTenantId(tenantId); - notificationRuleService.deleteNotificationRulesByTenantId(tenantId); - notificationTemplateService.deleteNotificationTemplatesByTenantId(tenantId); - notificationTargetService.deleteNotificationTargetsByTenantId(tenantId); - notificationSettingsService.deleteNotificationSettings(tenantId); + + userService.deleteAllByTenantId(tenantId); adminSettingsService.deleteAdminSettingsByTenantId(tenantId); + notificationSettingsService.deleteNotificationSettings(tenantId); tenantDao.removeById(tenantId, tenantId.getId()); publishEvictEvent(new TenantEvictEvent(tenantId, true)); - eventPublisher.publishEvent(DeleteEntityEvent.builder().tenantId(tenantId) - .entity(tenant).entityId(tenantId).build()); - relationService.deleteEntityRelations(tenantId, tenantId); - alarmService.deleteEntityAlarmRecordsByTenantId(tenantId); - queueStatsService.deleteByTenantId(tenantId); + + cleanUpService.removeTenantEntities(tenantId, // don't forget to implement deleteEntity from EntityDaoService when adding entity type to this list + EntityType.ENTITY_VIEW, EntityType.WIDGETS_BUNDLE, EntityType.WIDGET_TYPE, + EntityType.ASSET, EntityType.ASSET_PROFILE, EntityType.DEVICE, EntityType.DEVICE_PROFILE, + EntityType.DASHBOARD, EntityType.EDGE, EntityType.RULE_CHAIN, EntityType.API_USAGE_STATE, + EntityType.TB_RESOURCE, EntityType.OTA_PACKAGE, EntityType.RPC, EntityType.QUEUE, + EntityType.NOTIFICATION_REQUEST, EntityType.NOTIFICATION_RULE, EntityType.NOTIFICATION_TEMPLATE, + EntityType.NOTIFICATION_TARGET, EntityType.QUEUE_STATS, EntityType.CUSTOMER + ); + eventPublisher.publishEvent(DeleteEntityEvent.builder().tenantId(tenantId).entityId(tenantId).entity(tenant).build()); } @Override diff --git a/dao/src/main/java/org/thingsboard/server/dao/usagerecord/ApiUsageStateServiceImpl.java b/dao/src/main/java/org/thingsboard/server/dao/usagerecord/ApiUsageStateServiceImpl.java index 52195cf8db..d901ae950e 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/usagerecord/ApiUsageStateServiceImpl.java +++ b/dao/src/main/java/org/thingsboard/server/dao/usagerecord/ApiUsageStateServiceImpl.java @@ -18,6 +18,7 @@ package org.thingsboard.server.dao.usagerecord; import lombok.extern.slf4j.Slf4j; import org.springframework.context.annotation.Lazy; import org.springframework.stereotype.Service; +import org.springframework.transaction.annotation.Transactional; import org.thingsboard.server.common.data.ApiFeature; import org.thingsboard.server.common.data.ApiUsageRecordKey; import org.thingsboard.server.common.data.ApiUsageState; @@ -35,6 +36,7 @@ import org.thingsboard.server.common.data.kv.StringDataEntry; import org.thingsboard.server.common.data.kv.TsKvEntry; import org.thingsboard.server.common.data.tenant.profile.TenantProfileConfiguration; import org.thingsboard.server.dao.entity.AbstractEntityService; +import org.thingsboard.server.dao.eventsourcing.DeleteEntityEvent; import org.thingsboard.server.dao.eventsourcing.SaveEntityEvent; import org.thingsboard.server.dao.service.DataValidator; import org.thingsboard.server.dao.tenant.TenantProfileDao; @@ -69,18 +71,26 @@ public class ApiUsageStateServiceImpl extends AbstractEntityService implements A this.apiUsageStateValidator = apiUsageStateValidator; } - @Override - public void deleteApiUsageStateByTenantId(TenantId tenantId) { - log.trace("Executing deleteUsageRecordsByTenantId [{}]", tenantId); - validateId(tenantId, id -> INCORRECT_TENANT_ID + id); - apiUsageStateDao.deleteApiUsageStateByTenantId(tenantId); - } - + @Transactional @Override public void deleteApiUsageStateByEntityId(EntityId entityId) { log.trace("Executing deleteApiUsageStateByEntityId [{}]", entityId); validateId(entityId.getId(), id -> "Invalid entity id " + id); - apiUsageStateDao.deleteApiUsageStateByEntityId(entityId); + ApiUsageState apiUsageState = findApiUsageStateByEntityId(entityId); + deleteApiUsageState(apiUsageState); + } + + private void deleteApiUsageState(ApiUsageState apiUsageState) { + if (apiUsageState != null) { + apiUsageStateDao.removeById(apiUsageState.getTenantId(), apiUsageState.getUuidId()); + eventPublisher.publishEvent(DeleteEntityEvent.builder().tenantId(apiUsageState.getTenantId()).entityId(apiUsageState.getId()).build()); + } + } + + @Override + public void deleteEntity(TenantId tenantId, EntityId id, boolean force) { + ApiUsageState apiUsageState = findApiUsageStateById(tenantId, (ApiUsageStateId) id); + deleteApiUsageState(apiUsageState); } @Override @@ -176,6 +186,12 @@ public class ApiUsageStateServiceImpl extends AbstractEntityService implements A return Optional.ofNullable(findApiUsageStateById(tenantId, new ApiUsageStateId(entityId.getId()))); } + @Transactional + @Override + public void deleteByTenantId(TenantId tenantId) { + deleteApiUsageStateByEntityId(tenantId); + } + @Override public EntityType getEntityType() { return EntityType.API_USAGE_STATE; diff --git a/dao/src/main/java/org/thingsboard/server/dao/user/UserServiceImpl.java b/dao/src/main/java/org/thingsboard/server/dao/user/UserServiceImpl.java index 4b804353ad..90084e1a3f 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/user/UserServiceImpl.java +++ b/dao/src/main/java/org/thingsboard/server/dao/user/UserServiceImpl.java @@ -262,7 +262,7 @@ public class UserServiceImpl extends AbstractEntityService implements UserServic validateId(userId, id -> INCORRECT_USER_ID + id); userCredentialsDao.removeByUserId(tenantId, userId); userAuthSettingsDao.removeByUserId(userId); - deleteEntityRelations(tenantId, userId); + userSettingsDao.removeByUserId(tenantId, userId); userDao.removeById(tenantId, userId.getId()); eventPublisher.publishEvent(new UserCredentialsInvalidationEvent(userId)); @@ -321,6 +321,17 @@ public class UserServiceImpl extends AbstractEntityService implements UserServic tenantAdminsRemover.removeEntities(tenantId, tenantId); } + @Override + public void deleteAllByTenantId(TenantId tenantId) { + log.trace("Executing deleteByTenantId, tenantId [{}]", tenantId); + usersRemover.removeEntities(tenantId, tenantId); + } + + @Override + public void deleteByTenantId(TenantId tenantId) { + deleteAllByTenantId(tenantId); + } + @Override public PageData findCustomerUsers(TenantId tenantId, CustomerId customerId, PageLink pageLink) { log.trace("Executing findCustomerUsers, tenantId [{}], customerId [{}], pageLink [{}]", tenantId, customerId, pageLink); @@ -508,6 +519,18 @@ public class UserServiceImpl extends AbstractEntityService implements UserServic } }; + private final PaginatedRemover usersRemover = new PaginatedRemover<>() { + @Override + protected PageData findEntities(TenantId tenantId, TenantId id, PageLink pageLink) { + return findUsersByTenantId(tenantId, pageLink); + } + + @Override + protected void removeEntity(TenantId tenantId, User user) { + deleteUser(tenantId, user); + } + }; + @Override public Optional> findEntity(TenantId tenantId, EntityId entityId) { return Optional.ofNullable(findUserById(tenantId, new UserId(entityId.getId()))); diff --git a/dao/src/main/java/org/thingsboard/server/dao/user/UserSettingsDao.java b/dao/src/main/java/org/thingsboard/server/dao/user/UserSettingsDao.java index 23734118a9..bc93a23568 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/user/UserSettingsDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/user/UserSettingsDao.java @@ -16,6 +16,7 @@ package org.thingsboard.server.dao.user; import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.id.UserId; import org.thingsboard.server.common.data.settings.UserSettings; import org.thingsboard.server.common.data.settings.UserSettingsCompositeKey; import org.thingsboard.server.common.data.settings.UserSettingsType; @@ -30,6 +31,8 @@ public interface UserSettingsDao { void removeById(TenantId tenantId, UserSettingsCompositeKey key); + void removeByUserId(TenantId tenantId, UserId userId); + List findByTypeAndPath(TenantId tenantId, UserSettingsType type, String... path); } diff --git a/dao/src/main/java/org/thingsboard/server/dao/widget/WidgetTypeServiceImpl.java b/dao/src/main/java/org/thingsboard/server/dao/widget/WidgetTypeServiceImpl.java index 5321e4ba1c..f2506a2d9b 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/widget/WidgetTypeServiceImpl.java +++ b/dao/src/main/java/org/thingsboard/server/dao/widget/WidgetTypeServiceImpl.java @@ -115,6 +115,11 @@ public class WidgetTypeServiceImpl implements WidgetTypeService { eventPublisher.publishEvent(DeleteEntityEvent.builder().tenantId(tenantId).entityId(widgetTypeId).build()); } + @Override + public void deleteEntity(TenantId tenantId, EntityId id, boolean force) { + deleteWidgetType(tenantId, (WidgetTypeId) id); + } + @Override public PageData findSystemWidgetTypesByPageLink(TenantId tenantId, boolean fullSearch, DeprecatedFilter deprecatedFilter, List widgetTypes, PageLink pageLink) { log.trace("Executing findSystemWidgetTypesByPageLink, fullSearch [{}], deprecatedFilter [{}], widgetTypes [{}], pageLink [{}]", fullSearch, deprecatedFilter, widgetTypes, pageLink); @@ -226,6 +231,11 @@ public class WidgetTypeServiceImpl implements WidgetTypeService { tenantWidgetTypeRemover.removeEntities(tenantId, tenantId); } + @Override + public void deleteByTenantId(TenantId tenantId) { + deleteWidgetTypesByTenantId(tenantId); + } + @Override public Optional> findEntity(TenantId tenantId, EntityId entityId) { return Optional.ofNullable(findWidgetTypeById(tenantId, new WidgetTypeId(entityId.getId()))); diff --git a/dao/src/main/java/org/thingsboard/server/dao/widget/WidgetsBundleServiceImpl.java b/dao/src/main/java/org/thingsboard/server/dao/widget/WidgetsBundleServiceImpl.java index 2e67406df1..ae53c70ec8 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/widget/WidgetsBundleServiceImpl.java +++ b/dao/src/main/java/org/thingsboard/server/dao/widget/WidgetsBundleServiceImpl.java @@ -92,12 +92,21 @@ public class WidgetsBundleServiceImpl implements WidgetsBundleService { public void deleteWidgetsBundle(TenantId tenantId, WidgetsBundleId widgetsBundleId) { log.trace("Executing deleteWidgetsBundle [{}]", widgetsBundleId); Validator.validateId(widgetsBundleId, id -> "Incorrect widgetsBundleId " + id); - WidgetsBundle widgetsBundle = findWidgetsBundleById(tenantId, widgetsBundleId); + deleteEntity(tenantId, widgetsBundleId, false); + } + + @Override + public void deleteEntity(TenantId tenantId, EntityId id, boolean force) { + WidgetsBundle widgetsBundle = findWidgetsBundleById(tenantId, (WidgetsBundleId) id); if (widgetsBundle == null) { - throw new IncorrectParameterException("Unable to delete non-existent widgets bundle."); + if (force) { + return; + } else { + throw new IncorrectParameterException("Unable to delete non-existent widgets bundle."); + } } - eventPublisher.publishEvent(DeleteEntityEvent.builder().tenantId(tenantId).entityId(widgetsBundleId).build()); - widgetsBundleDao.removeById(tenantId, widgetsBundleId.getId()); + eventPublisher.publishEvent(DeleteEntityEvent.builder().tenantId(tenantId).entityId(id).build()); + widgetsBundleDao.removeById(tenantId, id.getId()); } @Override @@ -180,13 +189,13 @@ public class WidgetsBundleServiceImpl implements WidgetsBundleService { } @Override - public Optional> findEntity(TenantId tenantId, EntityId entityId) { - return Optional.ofNullable(findWidgetsBundleById(tenantId, new WidgetsBundleId(entityId.getId()))); + public void deleteByTenantId(TenantId tenantId) { + deleteWidgetsBundlesByTenantId(tenantId); } @Override - public void deleteEntity(TenantId tenantId, EntityId id) { - deleteWidgetsBundle(tenantId, (WidgetsBundleId) id); + public Optional> findEntity(TenantId tenantId, EntityId entityId) { + return Optional.ofNullable(findWidgetsBundleById(tenantId, new WidgetsBundleId(entityId.getId()))); } @Override diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/AbstractServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/AbstractServiceTest.java index 5eddc78661..6127a4192a 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/AbstractServiceTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/AbstractServiceTest.java @@ -42,6 +42,8 @@ import org.thingsboard.server.common.data.device.profile.DefaultDeviceProfileTra import org.thingsboard.server.common.data.device.profile.DeviceProfileData; import org.thingsboard.server.common.data.edge.Edge; import org.thingsboard.server.common.data.event.RuleNodeDebugEvent; +import org.thingsboard.server.common.data.housekeeper.TenantEntitiesDeletionHousekeeperTask; +import org.thingsboard.server.common.data.housekeeper.HousekeeperTaskType; import org.thingsboard.server.common.data.id.DeviceProfileId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.HasId; @@ -51,6 +53,9 @@ import org.thingsboard.server.common.data.ota.OtaPackageType; import org.thingsboard.server.dao.audit.AuditLogLevelFilter; import org.thingsboard.server.dao.audit.AuditLogLevelMask; import org.thingsboard.server.dao.audit.AuditLogLevelProperties; +import org.thingsboard.server.dao.entity.EntityDaoService; +import org.thingsboard.server.dao.entity.EntityServiceRegistry; +import org.thingsboard.server.common.msg.housekeeper.HousekeeperClient; import org.thingsboard.server.dao.tenant.TenantService; import java.io.IOException; @@ -77,6 +82,9 @@ public abstract class AbstractServiceTest { @Autowired protected TenantService tenantService; + @Autowired + protected EntityServiceRegistry entityServiceRegistry; + protected TenantId tenantId; @Before @@ -131,6 +139,16 @@ public abstract class AbstractServiceTest { return new AuditLogLevelFilter(props); } + @Bean + public HousekeeperClient housekeeperClient() { + return task -> { + if (task.getTaskType() == HousekeeperTaskType.DELETE_TENANT_ENTITIES) { + EntityDaoService entityService = entityServiceRegistry.getServiceByEntityType(((TenantEntitiesDeletionHousekeeperTask) task).getEntityType()); + entityService.deleteByTenantId(task.getTenantId()); + } + }; + } + protected DeviceProfile createDeviceProfile(TenantId tenantId, String name) { DeviceProfile deviceProfile = new DeviceProfile(); deviceProfile.setTenantId(tenantId); diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/EntityDaoRegistryTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/EntityDaoRegistryTest.java new file mode 100644 index 0000000000..8524efc4a0 --- /dev/null +++ b/dao/src/test/java/org/thingsboard/server/dao/service/EntityDaoRegistryTest.java @@ -0,0 +1,76 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao.service; + +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.exception.ExceptionUtils; +import org.junit.Test; +import org.springframework.beans.factory.annotation.Autowired; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.dao.Dao; +import org.thingsboard.server.dao.entity.EntityDaoRegistry; + +import java.util.UUID; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.fail; +import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; + +@Slf4j +@DaoSqlTest +public class EntityDaoRegistryTest extends AbstractServiceTest { + + @Autowired + EntityDaoRegistry entityDaoRegistry; + + @Test + public void givenAllEntityTypes_whenGetDao_thenAllPresent() { + for (EntityType entityType : EntityType.values()) { + Dao dao = assertDoesNotThrow(() -> entityDaoRegistry.getDao(entityType)); + assertThat(dao).isNotNull(); + } + } + + @Test + public void givenAllDaos_whenFindById_thenOk() { + for (EntityType entityType : EntityType.values()) { + Dao dao = entityDaoRegistry.getDao(entityType); + assertDoesNotThrow(() -> { + dao.findById(TenantId.SYS_TENANT_ID, UUID.randomUUID()); + }); + } + } + + @Test + public void givenAllDaos_whenFindIdsByTenantIdAndIdOffset_thenOk() { + for (EntityType entityType : EntityType.values()) { + Dao dao = entityDaoRegistry.getDao(entityType); + try { + dao.findIdsByTenantIdAndIdOffset(TenantId.SYS_TENANT_ID, null, 10); + dao.findIdsByTenantIdAndIdOffset(TenantId.SYS_TENANT_ID, UUID.randomUUID(), 10); + } catch (Exception e) { + String error = ExceptionUtils.getRootCauseMessage(e); + if (error.contains("tenant_id")) { + log.debug("[{}] Ignoring not found tenant_id column", entityType); + } else { + fail("findIdsByTenantIdAndIdOffset for " + entityType + " dao threw error: " + error); + } + } + } + } + +} diff --git a/msa/vc-executor/src/main/resources/tb-vc-executor.yml b/msa/vc-executor/src/main/resources/tb-vc-executor.yml index 289baaa97a..a92a7eea2b 100644 --- a/msa/vc-executor/src/main/resources/tb-vc-executor.yml +++ b/msa/vc-executor/src/main/resources/tb-vc-executor.yml @@ -137,6 +137,8 @@ queue: notifications: "${TB_QUEUE_KAFKA_NOTIFICATIONS_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:1048576000;partitions:1;min.insync.replicas:1}" # Kafka properties for Core topics version-control: "${TB_QUEUE_KAFKA_VC_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:1048576000;partitions:1;min.insync.replicas:1}" + # Kafka properties for Housekeeper tasks topic + housekeeper: "${TB_QUEUE_KAFKA_HOUSEKEEPER_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:1048576000;partitions:10;min.insync.replicas:1}" consumer-stats: # Prints lag between consumer group offset and last messages offset in Kafka topics enabled: "${TB_QUEUE_KAFKA_CONSUMER_STATS_ENABLED:true}" @@ -250,6 +252,9 @@ queue: enabled: "${TB_QUEUE_CORE_STATS_ENABLED:true}" # Statistics printing interval for Core microservices print-interval-ms: "${TB_QUEUE_CORE_STATS_PRINT_INTERVAL_MS:60000}" + housekeeper: + # Topic name for Housekeeper tasks + topic: "${TB_HOUSEKEEPER_TOPIC:tb_housekeeper}" vc: # Default topic name for Kafka, RabbitMQ, etc. topic: "${TB_QUEUE_VC_TOPIC:tb_version_control}" diff --git a/transport/coap/src/main/resources/tb-coap-transport.yml b/transport/coap/src/main/resources/tb-coap-transport.yml index 86ed3a269b..17a6a53aca 100644 --- a/transport/coap/src/main/resources/tb-coap-transport.yml +++ b/transport/coap/src/main/resources/tb-coap-transport.yml @@ -299,6 +299,8 @@ queue: transport-api: "${TB_QUEUE_KAFKA_TA_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:1048576000;partitions:10;min.insync.replicas:1}" # Kafka properties for Notifications topics notifications: "${TB_QUEUE_KAFKA_NOTIFICATIONS_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:1048576000;partitions:1;min.insync.replicas:1}" + # Kafka properties for Housekeeper tasks topic + housekeeper: "${TB_QUEUE_KAFKA_HOUSEKEEPER_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:1048576000;partitions:10;min.insync.replicas:1}" aws_sqs: # Use the default credentials provider for AWS SQS use_default_credential_provider_chain: "${TB_QUEUE_AWS_SQS_USE_DEFAULT_CREDENTIAL_PROVIDER_CHAIN:false}" @@ -420,6 +422,9 @@ queue: enabled: "${TB_QUEUE_CORE_STATS_ENABLED:false}" # Statistics printing interval for Core microservices print-interval-ms: "${TB_QUEUE_CORE_STATS_PRINT_INTERVAL_MS:10000}" + housekeeper: + # Topic name for Housekeeper tasks + topic: "${TB_HOUSEKEEPER_TOPIC:tb_housekeeper}" js: # JS Eval request topic request_topic: "${REMOTE_JS_EVAL_REQUEST_TOPIC:js_eval.requests}" diff --git a/transport/http/src/main/resources/tb-http-transport.yml b/transport/http/src/main/resources/tb-http-transport.yml index 8311dd90a1..4252dfac06 100644 --- a/transport/http/src/main/resources/tb-http-transport.yml +++ b/transport/http/src/main/resources/tb-http-transport.yml @@ -272,6 +272,8 @@ queue: transport-api: "${TB_QUEUE_KAFKA_TA_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:1048576000;partitions:10;min.insync.replicas:1}" # Kafka properties for Notifications topics notifications: "${TB_QUEUE_KAFKA_NOTIFICATIONS_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:1048576000;partitions:1;min.insync.replicas:1}" + # Kafka properties for Housekeeper tasks topic + housekeeper: "${TB_QUEUE_KAFKA_HOUSEKEEPER_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:1048576000;partitions:10;min.insync.replicas:1}" aws_sqs: # Use default credentials provider for AWS SQS use_default_credential_provider_chain: "${TB_QUEUE_AWS_SQS_USE_DEFAULT_CREDENTIAL_PROVIDER_CHAIN:false}" @@ -394,6 +396,9 @@ queue: enabled: "${TB_QUEUE_CORE_STATS_ENABLED:false}" # Statistics printing interval for Core microservices print-interval-ms: "${TB_QUEUE_CORE_STATS_PRINT_INTERVAL_MS:10000}" + housekeeper: + # Topic name for Housekeeper tasks + topic: "${TB_HOUSEKEEPER_TOPIC:tb_housekeeper}" js: # JS Eval request topic request_topic: "${REMOTE_JS_EVAL_REQUEST_TOPIC:js_eval.requests}" diff --git a/transport/lwm2m/src/main/resources/tb-lwm2m-transport.yml b/transport/lwm2m/src/main/resources/tb-lwm2m-transport.yml index 1070f17746..cf1ba51240 100644 --- a/transport/lwm2m/src/main/resources/tb-lwm2m-transport.yml +++ b/transport/lwm2m/src/main/resources/tb-lwm2m-transport.yml @@ -377,6 +377,8 @@ queue: transport-api: "${TB_QUEUE_KAFKA_TA_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:1048576000;partitions:10;min.insync.replicas:1}" # Kafka properties for Notifications topics notifications: "${TB_QUEUE_KAFKA_NOTIFICATIONS_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:1048576000;partitions:1;min.insync.replicas:1}" + # Kafka properties for Housekeeper tasks topic + housekeeper: "${TB_QUEUE_KAFKA_HOUSEKEEPER_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:1048576000;partitions:10;min.insync.replicas:1}" aws_sqs: # Use the default credentials provider for AWS SQS use_default_credential_provider_chain: "${TB_QUEUE_AWS_SQS_USE_DEFAULT_CREDENTIAL_PROVIDER_CHAIN:false}" @@ -496,6 +498,9 @@ queue: enabled: "${TB_QUEUE_CORE_STATS_ENABLED:false}" # Statistics printing interval for Core microservices print-interval-ms: "${TB_QUEUE_CORE_STATS_PRINT_INTERVAL_MS:10000}" + housekeeper: + # Topic name for Housekeeper tasks + topic: "${TB_HOUSEKEEPER_TOPIC:tb_housekeeper}" js: # JS Eval request topic request_topic: "${REMOTE_JS_EVAL_REQUEST_TOPIC:js_eval.requests}" diff --git a/transport/mqtt/src/main/resources/tb-mqtt-transport.yml b/transport/mqtt/src/main/resources/tb-mqtt-transport.yml index 86c7890f28..d210138852 100644 --- a/transport/mqtt/src/main/resources/tb-mqtt-transport.yml +++ b/transport/mqtt/src/main/resources/tb-mqtt-transport.yml @@ -304,6 +304,8 @@ queue: transport-api: "${TB_QUEUE_KAFKA_TA_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:1048576000;partitions:10;min.insync.replicas:1}" # Kafka properties for Notifications topics notifications: "${TB_QUEUE_KAFKA_NOTIFICATIONS_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:1048576000;partitions:1;min.insync.replicas:1}" + # Kafka properties for Housekeeper tasks topic + housekeeper: "${TB_QUEUE_KAFKA_HOUSEKEEPER_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:1048576000;partitions:10;min.insync.replicas:1}" aws_sqs: # Use the default credentials provider for AWS SQS use_default_credential_provider_chain: "${TB_QUEUE_AWS_SQS_USE_DEFAULT_CREDENTIAL_PROVIDER_CHAIN:false}" @@ -425,6 +427,9 @@ queue: enabled: "${TB_QUEUE_CORE_STATS_ENABLED:false}" # Statistics printing interval for Core microservices print-interval-ms: "${TB_QUEUE_CORE_STATS_PRINT_INTERVAL_MS:10000}" + housekeeper: + # Topic name for Housekeeper tasks + topic: "${TB_HOUSEKEEPER_TOPIC:tb_housekeeper}" js: # JS Eval request topic request_topic: "${REMOTE_JS_EVAL_REQUEST_TOPIC:js_eval.requests}" diff --git a/transport/snmp/src/main/resources/tb-snmp-transport.yml b/transport/snmp/src/main/resources/tb-snmp-transport.yml index 72bef5183b..76f6abdb61 100644 --- a/transport/snmp/src/main/resources/tb-snmp-transport.yml +++ b/transport/snmp/src/main/resources/tb-snmp-transport.yml @@ -254,6 +254,8 @@ queue: transport-api: "${TB_QUEUE_KAFKA_TA_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:1048576000;partitions:10;min.insync.replicas:1}" # Kafka properties for Notifications topics notifications: "${TB_QUEUE_KAFKA_NOTIFICATIONS_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:1048576000;partitions:1;min.insync.replicas:1}" + # Kafka properties for Housekeeper tasks topic + housekeeper: "${TB_QUEUE_KAFKA_HOUSEKEEPER_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:1048576000;partitions:10;min.insync.replicas:1}" consumer-stats: # Prints lag between consumer group offset and last messages offset in Kafka topics enabled: "${TB_QUEUE_KAFKA_CONSUMER_STATS_ENABLED:true}" @@ -382,6 +384,9 @@ queue: enabled: "${TB_QUEUE_CORE_STATS_ENABLED:false}" # Statistics printing interval for Core microservices print-interval-ms: "${TB_QUEUE_CORE_STATS_PRINT_INTERVAL_MS:10000}" + housekeeper: + # Topic name for Housekeeper tasks + topic: "${TB_HOUSEKEEPER_TOPIC:tb_housekeeper}" js: # JS Eval request topic request_topic: "${REMOTE_JS_EVAL_REQUEST_TOPIC:js_eval.requests}" diff --git a/ui-ngx/src/app/modules/home/pages/notification/rule/rule-notification-dialog.component.html b/ui-ngx/src/app/modules/home/pages/notification/rule/rule-notification-dialog.component.html index 5b477d0345..f5a2ddba17 100644 --- a/ui-ngx/src/app/modules/home/pages/notification/rule/rule-notification-dialog.component.html +++ b/ui-ngx/src/app/modules/home/pages/notification/rule/rule-notification-dialog.component.html @@ -581,6 +581,19 @@ + + + {{ 'notification.task-processing-failure-trigger-settings' | translate }} +
+
+ + notification.description + + +
+
+ diff --git a/ui-ngx/src/app/modules/home/pages/notification/rule/rule-notification-dialog.component.ts b/ui-ngx/src/app/modules/home/pages/notification/rule/rule-notification-dialog.component.ts index 89488b7517..bc6935b4e7 100644 --- a/ui-ngx/src/app/modules/home/pages/notification/rule/rule-notification-dialog.component.ts +++ b/ui-ngx/src/app/modules/home/pages/notification/rule/rule-notification-dialog.component.ts @@ -101,6 +101,7 @@ export class RuleNotificationDialogComponent extends rateLimitsTemplateForm: FormGroup; edgeCommunicationFailureTemplateForm: FormGroup; edgeConnectionTemplateForm: FormGroup; + taskProcessingFailureTemplateForm: FormGroup; triggerType = TriggerType; triggerTypes: TriggerType[]; @@ -337,6 +338,12 @@ export class RuleNotificationDialogComponent extends }) }); + this.taskProcessingFailureTemplateForm = this.fb.group({ + triggerConfig: this.fb.group({ + taskTypes: [] + }) + }); + this.triggerTypeFormsMap = new Map([ [TriggerType.ALARM, this.alarmTemplateForm], [TriggerType.ALARM_COMMENT, this.alarmCommentTemplateForm], @@ -349,7 +356,8 @@ export class RuleNotificationDialogComponent extends [TriggerType.NEW_PLATFORM_VERSION, this.newPlatformVersionTemplateForm], [TriggerType.RATE_LIMITS, this.rateLimitsTemplateForm], [TriggerType.EDGE_COMMUNICATION_FAILURE, this.edgeCommunicationFailureTemplateForm], - [TriggerType.EDGE_CONNECTION, this.edgeConnectionTemplateForm] + [TriggerType.EDGE_CONNECTION, this.edgeConnectionTemplateForm], + [TriggerType.TASK_PROCESSING_FAILURE, this.taskProcessingFailureTemplateForm] ]); if (data.isAdd || data.isCopy) { @@ -488,7 +496,8 @@ export class RuleNotificationDialogComponent extends TriggerType.ENTITIES_LIMIT, TriggerType.API_USAGE_LIMIT, TriggerType.NEW_PLATFORM_VERSION, - TriggerType.RATE_LIMITS + TriggerType.RATE_LIMITS, + TriggerType.TASK_PROCESSING_FAILURE ]); if (this.isSysAdmin()) { diff --git a/ui-ngx/src/app/modules/home/pages/notification/template/template-notification-dialog.component.ts b/ui-ngx/src/app/modules/home/pages/notification/template/template-notification-dialog.component.ts index c0ae4f1abb..ccbcb6bf22 100644 --- a/ui-ngx/src/app/modules/home/pages/notification/template/template-notification-dialog.component.ts +++ b/ui-ngx/src/app/modules/home/pages/notification/template/template-notification-dialog.component.ts @@ -179,7 +179,8 @@ export class TemplateNotificationDialogComponent NotificationType.ENTITIES_LIMIT, NotificationType.API_USAGE_LIMIT, NotificationType.NEW_PLATFORM_VERSION, - NotificationType.RATE_LIMITS + NotificationType.RATE_LIMITS, + NotificationType.TASK_PROCESSING_FAILURE ]); if (this.isSysAdmin()) { diff --git a/ui-ngx/src/app/shared/models/notification.models.ts b/ui-ngx/src/app/shared/models/notification.models.ts index b8097d18cb..cfb1312209 100644 --- a/ui-ngx/src/app/shared/models/notification.models.ts +++ b/ui-ngx/src/app/shared/models/notification.models.ts @@ -523,7 +523,8 @@ export enum NotificationType { RULE_NODE = 'RULE_NODE', RATE_LIMITS = 'RATE_LIMITS', EDGE_CONNECTION = 'EDGE_CONNECTION', - EDGE_COMMUNICATION_FAILURE = 'EDGE_COMMUNICATION_FAILURE' + EDGE_COMMUNICATION_FAILURE = 'EDGE_COMMUNICATION_FAILURE', + TASK_PROCESSING_FAILURE = 'TASK_PROCESSING_FAILURE' } export const NotificationTypeIcons = new Map([ @@ -535,6 +536,7 @@ export const NotificationTypeIcons = new Map([ [NotificationType.RULE_ENGINE_COMPONENT_LIFECYCLE_EVENT, 'settings_ethernet'], [NotificationType.ENTITIES_LIMIT, 'data_thresholding'], [NotificationType.API_USAGE_LIMIT, 'insert_chart'], + [NotificationType.TASK_PROCESSING_FAILURE, 'warning'] ]); export const AlarmSeverityNotificationColors = new Map( @@ -646,7 +648,13 @@ export const NotificationTemplateTypeTranslateMap = new Map([ @@ -676,7 +685,8 @@ export const TriggerTypeTranslationMap = new Map([ [TriggerType.NEW_PLATFORM_VERSION, 'notification.trigger.new-platform-version'], [TriggerType.RATE_LIMITS, 'notification.trigger.rate-limits'], [TriggerType.EDGE_CONNECTION, 'notification.trigger.edge-connection'], - [TriggerType.EDGE_COMMUNICATION_FAILURE, 'notification.trigger.edge-communication-failure'] + [TriggerType.EDGE_COMMUNICATION_FAILURE, 'notification.trigger.edge-communication-failure'], + [TriggerType.TASK_PROCESSING_FAILURE, 'notification.trigger.task-processing-failure'] ]); export interface NotificationUserSettings { diff --git a/ui-ngx/src/assets/help/en_US/notification/task_processing_failure.md b/ui-ngx/src/assets/help/en_US/notification/task_processing_failure.md new file mode 100644 index 0000000000..0bcf107298 --- /dev/null +++ b/ui-ngx/src/assets/help/en_US/notification/task_processing_failure.md @@ -0,0 +1,46 @@ +#### Task processing failure notification templatization + +
+
+ +Notification subject and message fields support templatization. +The list of available templatization parameters depends on the template type. +See the available types and parameters below: + +Available template parameters: + +* `taskType` - the task type, e.g. 'telemetry deletion'; +* `taskDescription` - the task description, e.g. 'telemetry deletion for device c4d93dc0-63a1-11ee-aa6d-f7cbc0a71325'; +* `error` - the error stacktrace +* `tenantId` - the tenant id; +* `entityType` - the type of the entity to which the task is related; +* `entityId` - the id of the entity to which the task is related; +* `attempt` - the number of attempts processing the task + +Parameter names must be wrapped using `${...}`. For example: `${entityType}`. +You may also modify the value of the parameter with one of the suffixes: + +* `upperCase`, for example - `${entityType:upperCase}` +* `lowerCase`, for example - `${entityType:lowerCase}` +* `capitalize`, for example - `${entityType:capitalize}` + +
+ +##### Examples + +Let's assume that telemetry deletion failed for some device. +The following template: + +```text +Failed to process ${taskType} for ${entityType:lowerCase} ${entityId} +{:copy-code} +``` + +will be transformed to: + +```text +Failed to process telemetry deletion for device c4d93dc0-63a1-11ee-aa6d-f7cbc0a71325 +``` + +
+
diff --git a/ui-ngx/src/assets/locale/locale.constant-en_US.json b/ui-ngx/src/assets/locale/locale.constant-en_US.json index f8f3001610..20f4a2a118 100644 --- a/ui-ngx/src/assets/locale/locale.constant-en_US.json +++ b/ui-ngx/src/assets/locale/locale.constant-en_US.json @@ -3355,6 +3355,7 @@ "api-usage-trigger-settings": "API usage trigger settings", "new-platform-version-trigger-settings": "New platform version trigger settings", "rate-limits-trigger-settings": "Exceeded rate limits trigger settings", + "task-processing-failure-trigger-settings": "Task processing failure trigger settings", "at-least-one-should-be-selected": "At least one should be selected", "basic-settings": "Basic settings", "button-text": "Button text", @@ -3565,7 +3566,8 @@ "new-platform-version": "New platform version", "rate-limits": "Exceeded rate limits", "edge-communication-failure": "Edge communication failure", - "edge-connection": "Edge connection" + "edge-connection": "Edge connection", + "task-processing-failure": "Task processing failure" }, "templates": "Templates", "notification-templates": "Notifications / Templates", @@ -3588,6 +3590,7 @@ "rate-limits": "Exceeded rate limits", "edge-connection": "Edge connection", "edge-communication-failure": "Edge communication failure", + "task-processing-failure": "Task processing failure", "trigger": "Trigger", "trigger-required": "Trigger is required" },