diff --git a/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java b/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java index f82db6ebe7..bade75df00 100644 --- a/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java +++ b/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java @@ -246,6 +246,7 @@ public class ActorSystemContext { private RuleNodeStateService ruleNodeStateService; @Autowired + @Getter private PartitionService partitionService; @Autowired diff --git a/application/src/main/java/org/thingsboard/server/actors/app/AppActor.java b/application/src/main/java/org/thingsboard/server/actors/app/AppActor.java index 1461654216..98e0bf7c25 100644 --- a/application/src/main/java/org/thingsboard/server/actors/app/AppActor.java +++ b/application/src/main/java/org/thingsboard/server/actors/app/AppActor.java @@ -46,6 +46,7 @@ import org.thingsboard.server.dao.tenant.TenantService; import org.thingsboard.server.service.transport.msg.TransportToDeviceActorMsgWrapper; import java.util.HashSet; +import java.util.Optional; import java.util.Set; @Slf4j @@ -129,8 +130,11 @@ public class AppActor extends ContextAwareActor { PageDataIterable tenantIterator = new PageDataIterable<>(tenantService::findTenants, ENTITY_PACK_LIMIT); for (Tenant tenant : tenantIterator) { log.debug("[{}] Creating tenant actor", tenant.getId()); - getOrCreateTenantActor(tenant.getId()); - log.debug("[{}] Tenant actor created.", tenant.getId()); + getOrCreateTenantActor(tenant.getId()).ifPresentOrElse(tenantActor -> { + log.debug("[{}] Tenant actor created.", tenant.getId()); + }, () -> { + log.debug("[{}] Skipped actor creation", tenant.getId()); + }); } } log.info("Main system actor started."); @@ -143,11 +147,9 @@ public class AppActor extends ContextAwareActor { if (TenantId.SYS_TENANT_ID.equals(msg.getTenantId())) { msg.getMsg().getCallback().onFailure(new RuleEngineException("Message has system tenant id!")); } else { - if (!deletedTenants.contains(msg.getTenantId())) { - getOrCreateTenantActor(msg.getTenantId()).tell(msg); - } else { - msg.getMsg().getCallback().onSuccess(); - } + getOrCreateTenantActor(msg.getTenantId()).ifPresentOrElse(actor -> { + actor.tell(msg); + }, () -> msg.getMsg().getCallback().onSuccess()); } } @@ -164,12 +166,13 @@ public class AppActor extends ContextAwareActor { log.info("[{}] Handling tenant deleted notification: {}", msg.getTenantId(), msg); deletedTenants.add(tenantId); ctx.stop(new TbEntityActorId(tenantId)); - } else { - target = getOrCreateTenantActor(msg.getTenantId()); + return; } - } else { - target = getOrCreateTenantActor(msg.getTenantId()); } + target = getOrCreateTenantActor(msg.getTenantId()).orElseGet(() -> { + log.debug("Ignoring component lifecycle msg for tenant {} because it is not managed by this service", msg.getTenantId()); + return null; + }); } if (target != null) { target.tellWithHighPriority(msg); @@ -179,24 +182,28 @@ public class AppActor extends ContextAwareActor { } private void onToDeviceActorMsg(TenantAwareMsg msg, boolean priority) { - if (!deletedTenants.contains(msg.getTenantId())) { - TbActorRef tenantActor = getOrCreateTenantActor(msg.getTenantId()); + getOrCreateTenantActor(msg.getTenantId()).ifPresentOrElse(tenantActor -> { if (priority) { tenantActor.tellWithHighPriority(msg); } else { tenantActor.tell(msg); } - } else { + }, () -> { if (msg instanceof TransportToDeviceActorMsgWrapper) { ((TransportToDeviceActorMsgWrapper) msg).getCallback().onSuccess(); } - } + }); } - private TbActorRef getOrCreateTenantActor(TenantId tenantId) { - return ctx.getOrCreateChildActor(new TbEntityActorId(tenantId), + private Optional getOrCreateTenantActor(TenantId tenantId) { + if (deletedTenants.contains(tenantId)) { + return Optional.empty(); + } + return Optional.ofNullable(ctx.getOrCreateChildActor(new TbEntityActorId(tenantId), () -> DefaultActorService.TENANT_DISPATCHER_NAME, - () -> new TenantActor.ActorCreator(systemContext, tenantId)); + () -> new TenantActor.ActorCreator(systemContext, tenantId), + () -> systemContext.getServiceInfoProvider().isService(ServiceType.TB_CORE) || + systemContext.getPartitionService().isManagedByCurrentService(tenantId))); } private void onToEdgeSessionMsg(EdgeSessionMsg msg) { @@ -204,7 +211,7 @@ public class AppActor extends ContextAwareActor { if (ModelConstants.SYSTEM_TENANT.equals(msg.getTenantId())) { log.warn("Message has system tenant id: {}", msg); } else { - target = getOrCreateTenantActor(msg.getTenantId()); + target = getOrCreateTenantActor(msg.getTenantId()).orElse(null); } if (target != null) { target.tellWithHighPriority(msg); diff --git a/application/src/main/java/org/thingsboard/server/actors/ruleChain/RuleChainActorMessageProcessor.java b/application/src/main/java/org/thingsboard/server/actors/ruleChain/RuleChainActorMessageProcessor.java index de617367dd..cbfd78180b 100644 --- a/application/src/main/java/org/thingsboard/server/actors/ruleChain/RuleChainActorMessageProcessor.java +++ b/application/src/main/java/org/thingsboard/server/actors/ruleChain/RuleChainActorMessageProcessor.java @@ -167,7 +167,8 @@ public class RuleChainActorMessageProcessor extends ComponentMsgProcessor DefaultActorService.RULE_DISPATCHER_NAME, - () -> new RuleNodeActor.ActorCreator(systemContext, tenantId, entityId, ruleChainName, ruleNode.getId())); + () -> new RuleNodeActor.ActorCreator(systemContext, tenantId, entityId, ruleChainName, ruleNode.getId()), + () -> true); } private void initRoutes(RuleChain ruleChain, List ruleNodeList) { diff --git a/application/src/main/java/org/thingsboard/server/actors/ruleChain/RuleChainManagerActor.java b/application/src/main/java/org/thingsboard/server/actors/ruleChain/RuleChainManagerActor.java index 7f919754fc..987554c683 100644 --- a/application/src/main/java/org/thingsboard/server/actors/ruleChain/RuleChainManagerActor.java +++ b/application/src/main/java/org/thingsboard/server/actors/ruleChain/RuleChainManagerActor.java @@ -94,7 +94,8 @@ public abstract class RuleChainManagerActor extends ContextAwareActor { } else { return new RuleChainActor.ActorCreator(systemContext, tenantId, ruleChain); } - }); + }, + () -> true); } protected TbActorRef getEntityActorRef(EntityId entityId) { diff --git a/application/src/main/java/org/thingsboard/server/actors/tenant/TenantActor.java b/application/src/main/java/org/thingsboard/server/actors/tenant/TenantActor.java index 11a5895fef..c9f5448df3 100644 --- a/application/src/main/java/org/thingsboard/server/actors/tenant/TenantActor.java +++ b/application/src/main/java/org/thingsboard/server/actors/tenant/TenantActor.java @@ -32,7 +32,6 @@ import org.thingsboard.server.actors.service.DefaultActorService; import org.thingsboard.server.common.data.ApiUsageState; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.Tenant; -import org.thingsboard.server.common.data.TenantProfile; import org.thingsboard.server.common.data.edge.Edge; import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.EdgeId; @@ -82,21 +81,21 @@ public class TenantActor extends RuleChainManagerActor { cantFindTenant = true; log.info("[{}] Started tenant actor for missing tenant.", tenantId); } else { - TenantProfile tenantProfile = systemContext.getTenantProfileCache().get(tenant.getTenantProfileId()); - isCore = systemContext.getServiceInfoProvider().isService(ServiceType.TB_CORE); isRuleEngine = systemContext.getServiceInfoProvider().isService(ServiceType.TB_RULE_ENGINE); if (isRuleEngine) { - try { - if (getApiUsageState().isReExecEnabled()) { - log.debug("[{}] Going to init rule chains", tenantId); - initRuleChains(); - } else { - log.info("[{}] Skip init of the rule chains due to API limits", tenantId); + if (systemContext.getPartitionService().isManagedByCurrentService(tenantId)) { + try { + if (getApiUsageState().isReExecEnabled()) { + log.debug("[{}] Going to init rule chains", tenantId); + initRuleChains(); + } else { + log.info("[{}] Skip init of the rule chains due to API limits", tenantId); + } + } catch (Exception e) { + log.info("Failed to check ApiUsage \"ReExecEnabled\"!!!", e); + cantFindTenant = true; } - } catch (Exception e) { - log.info("Failed to check ApiUsage \"ReExecEnabled\"!!!", e); - cantFindTenant = true; } } log.debug("[{}] Tenant actor started.", tenantId); @@ -270,7 +269,8 @@ public class TenantActor extends RuleChainManagerActor { private TbActorRef getOrCreateDeviceActor(DeviceId deviceId) { return ctx.getOrCreateChildActor(new TbEntityActorId(deviceId), () -> DefaultActorService.DEVICE_DISPATCHER_NAME, - () -> new DeviceActorCreator(systemContext, tenantId, deviceId)); + () -> new DeviceActorCreator(systemContext, tenantId, deviceId), + () -> true); } private void onToEdgeSessionMsg(EdgeSessionMsg msg) { diff --git a/application/src/main/java/org/thingsboard/server/controller/NotificationController.java b/application/src/main/java/org/thingsboard/server/controller/NotificationController.java index 0f4a288e33..f72b2dacae 100644 --- a/application/src/main/java/org/thingsboard/server/controller/NotificationController.java +++ b/application/src/main/java/org/thingsboard/server/controller/NotificationController.java @@ -44,6 +44,7 @@ import org.thingsboard.server.common.data.notification.NotificationRequest; import org.thingsboard.server.common.data.notification.NotificationRequestInfo; import org.thingsboard.server.common.data.notification.NotificationRequestPreview; import org.thingsboard.server.common.data.notification.settings.NotificationSettings; +import org.thingsboard.server.common.data.notification.settings.UserNotificationSettings; import org.thingsboard.server.common.data.notification.targets.NotificationRecipient; import org.thingsboard.server.common.data.notification.targets.NotificationTarget; import org.thingsboard.server.common.data.notification.targets.NotificationTargetType; @@ -297,7 +298,7 @@ public class NotificationController extends BaseController { if (targetType == NotificationTargetType.PLATFORM_USERS) { PageData recipients = notificationTargetService.findRecipientsForNotificationTargetConfig(user.getTenantId(), (PlatformUsersNotificationTargetConfig) target.getConfiguration(), new PageLink(recipientsPreviewSize, 0, null, - new SortOrder("createdTime", SortOrder.Direction.DESC))); + SortOrder.BY_CREATED_TIME_DESC)); recipientsCount = (int) recipients.getTotalElements(); recipientsPart = recipients.getData().stream().map(r -> (NotificationRecipient) r).collect(Collectors.toList()); } else { @@ -431,10 +432,23 @@ public class NotificationController extends BaseController { notes = "Returns the list of delivery methods that are properly configured and are allowed to be used for sending notifications." + SYSTEM_OR_TENANT_AUTHORITY_PARAGRAPH) @GetMapping("/notification/deliveryMethods") - @PreAuthorize("hasAnyAuthority('SYS_ADMIN', 'TENANT_ADMIN')") + @PreAuthorize("hasAnyAuthority('SYS_ADMIN', 'TENANT_ADMIN', 'CUSTOMER_USER')") public Set getAvailableDeliveryMethods(@AuthenticationPrincipal SecurityUser user) throws ThingsboardException { - accessControlService.checkPermission(user, Resource.ADMIN_SETTINGS, Operation.READ); return notificationCenter.getAvailableDeliveryMethods(user.getTenantId()); } + + @PostMapping("/notification/settings/user") + @PreAuthorize("hasAnyAuthority('SYS_ADMIN', 'TENANT_ADMIN', 'CUSTOMER_USER')") + public UserNotificationSettings saveUserNotificationSettings(@RequestBody @Valid UserNotificationSettings settings, + @AuthenticationPrincipal SecurityUser user) { + return notificationSettingsService.saveUserNotificationSettings(user.getTenantId(), user.getId(), settings); + } + + @GetMapping("/notification/settings/user") + @PreAuthorize("hasAnyAuthority('SYS_ADMIN', 'TENANT_ADMIN', 'CUSTOMER_USER')") + public UserNotificationSettings getUserNotificationSettings(@AuthenticationPrincipal SecurityUser user) { + return notificationSettingsService.getUserNotificationSettings(user.getTenantId(), user.getId(), true); + } + } diff --git a/application/src/main/java/org/thingsboard/server/controller/QueueController.java b/application/src/main/java/org/thingsboard/server/controller/QueueController.java index 0e79ae7955..faeea1a4aa 100644 --- a/application/src/main/java/org/thingsboard/server/controller/QueueController.java +++ b/application/src/main/java/org/thingsboard/server/controller/QueueController.java @@ -126,7 +126,6 @@ public class QueueController extends BaseController { @PreAuthorize("hasAnyAuthority('SYS_ADMIN')") @RequestMapping(value = "/queues", params = {"serviceType"}, method = RequestMethod.POST) @ResponseBody - public Queue saveQueue(@ApiParam(value = "A JSON value representing the queue.") @RequestBody Queue queue, @ApiParam(value = QUEUE_SERVICE_TYPE_DESCRIPTION, allowableValues = QUEUE_SERVICE_TYPE_ALLOWABLE_VALUES, required = true) diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/EdgeGrpcSession.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/EdgeGrpcSession.java index 4f0f5d277a..3c1ec796f3 100644 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/EdgeGrpcSession.java +++ b/application/src/main/java/org/thingsboard/server/service/edge/rpc/EdgeGrpcSession.java @@ -39,12 +39,16 @@ import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.common.data.page.SortOrder; import org.thingsboard.server.common.data.page.TimePageLink; import org.thingsboard.server.gen.edge.v1.AlarmUpdateMsg; +import org.thingsboard.server.gen.edge.v1.AssetProfileUpdateMsg; +import org.thingsboard.server.gen.edge.v1.AssetUpdateMsg; import org.thingsboard.server.gen.edge.v1.AttributesRequestMsg; import org.thingsboard.server.gen.edge.v1.ConnectRequestMsg; import org.thingsboard.server.gen.edge.v1.ConnectResponseCode; import org.thingsboard.server.gen.edge.v1.ConnectResponseMsg; +import org.thingsboard.server.gen.edge.v1.DashboardUpdateMsg; import org.thingsboard.server.gen.edge.v1.DeviceCredentialsRequestMsg; import org.thingsboard.server.gen.edge.v1.DeviceCredentialsUpdateMsg; +import org.thingsboard.server.gen.edge.v1.DeviceProfileUpdateMsg; import org.thingsboard.server.gen.edge.v1.DeviceRpcCallMsg; import org.thingsboard.server.gen.edge.v1.DeviceUpdateMsg; import org.thingsboard.server.gen.edge.v1.DownlinkMsg; @@ -53,6 +57,7 @@ import org.thingsboard.server.gen.edge.v1.EdgeConfiguration; import org.thingsboard.server.gen.edge.v1.EdgeUpdateMsg; import org.thingsboard.server.gen.edge.v1.EdgeVersion; import org.thingsboard.server.gen.edge.v1.EntityDataProto; +import org.thingsboard.server.gen.edge.v1.EntityViewUpdateMsg; import org.thingsboard.server.gen.edge.v1.EntityViewsRequestMsg; import org.thingsboard.server.gen.edge.v1.RelationRequestMsg; import org.thingsboard.server.gen.edge.v1.RelationUpdateMsg; @@ -151,8 +156,8 @@ public final class EdgeGrpcSession implements Closeable { } if (connected) { if (requestMsg.getMsgType().equals(RequestMsgType.SYNC_REQUEST_RPC_MESSAGE)) { - if (requestMsg.hasSyncRequestMsg() && requestMsg.getSyncRequestMsg().getSyncRequired()) { - boolean fullSync = true; + if (requestMsg.hasSyncRequestMsg()) { + boolean fullSync = false; if (requestMsg.getSyncRequestMsg().hasFullSync()) { fullSync = requestMsg.getSyncRequestMsg().getFullSync(); } @@ -654,6 +659,11 @@ public final class EdgeGrpcSession implements Closeable { result.addAll(ctx.getTelemetryProcessor().processTelemetryMsg(edge.getTenantId(), entityData)); } } + if (uplinkMsg.getDeviceProfileUpdateMsgCount() > 0) { + for (DeviceProfileUpdateMsg deviceProfileUpdateMsg : uplinkMsg.getDeviceProfileUpdateMsgList()) { + result.add(ctx.getDeviceProfileProcessor().processDeviceProfileMsgFromEdge(edge.getTenantId(), edge, deviceProfileUpdateMsg)); + } + } if (uplinkMsg.getDeviceUpdateMsgCount() > 0) { for (DeviceUpdateMsg deviceUpdateMsg : uplinkMsg.getDeviceUpdateMsgList()) { result.add(ctx.getDeviceProcessor().processDeviceMsgFromEdge(edge.getTenantId(), edge, deviceUpdateMsg)); @@ -664,16 +674,36 @@ public final class EdgeGrpcSession implements Closeable { result.add(ctx.getDeviceProcessor().processDeviceCredentialsMsg(edge.getTenantId(), deviceCredentialsUpdateMsg)); } } + if (uplinkMsg.getAssetProfileUpdateMsgCount() > 0) { + for (AssetProfileUpdateMsg assetProfileUpdateMsg : uplinkMsg.getAssetProfileUpdateMsgList()) { + result.add(ctx.getAssetProfileProcessor().processAssetProfileMsgFromEdge(edge.getTenantId(), edge, assetProfileUpdateMsg)); + } + } + if (uplinkMsg.getAssetUpdateMsgCount() > 0) { + for (AssetUpdateMsg assetUpdateMsg : uplinkMsg.getAssetUpdateMsgList()) { + result.add(ctx.getAssetProcessor().processAssetMsgFromEdge(edge.getTenantId(), edge, assetUpdateMsg)); + } + } if (uplinkMsg.getAlarmUpdateMsgCount() > 0) { for (AlarmUpdateMsg alarmUpdateMsg : uplinkMsg.getAlarmUpdateMsgList()) { result.add(ctx.getAlarmProcessor().processAlarmMsg(edge.getTenantId(), alarmUpdateMsg)); } } + if (uplinkMsg.getEntityViewUpdateMsgCount() > 0) { + for (EntityViewUpdateMsg entityViewUpdateMsg : uplinkMsg.getEntityViewUpdateMsgList()) { + result.add(ctx.getEntityViewProcessor().processEntityViewMsgFromEdge(edge.getTenantId(), edge, entityViewUpdateMsg)); + } + } if (uplinkMsg.getRelationUpdateMsgCount() > 0) { for (RelationUpdateMsg relationUpdateMsg : uplinkMsg.getRelationUpdateMsgList()) { result.add(ctx.getRelationProcessor().processRelationMsg(edge.getTenantId(), relationUpdateMsg)); } } + if (uplinkMsg.getDashboardUpdateMsgCount() > 0) { + for (DashboardUpdateMsg dashboardUpdateMsg : uplinkMsg.getDashboardUpdateMsgList()) { + result.add(ctx.getDashboardProcessor().processDashboardMsgFromEdge(edge.getTenantId(), edge, dashboardUpdateMsg)); + } + } if (uplinkMsg.getRuleChainMetadataRequestMsgCount() > 0) { for (RuleChainMetadataRequestMsg ruleChainMetadataRequestMsg : uplinkMsg.getRuleChainMetadataRequestMsgList()) { result.add(ctx.getEdgeRequestsService().processRuleChainMetadataRequestMsg(edge.getTenantId(), edge, ruleChainMetadataRequestMsg)); diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/EdgeSyncCursor.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/EdgeSyncCursor.java index bf3e804f7e..1eab7c3232 100644 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/EdgeSyncCursor.java +++ b/application/src/main/java/org/thingsboard/server/service/edge/rpc/EdgeSyncCursor.java @@ -51,8 +51,6 @@ public class EdgeSyncCursor { fetchers.add(new QueuesEdgeEventFetcher(ctx.getQueueService())); fetchers.add(new RuleChainsEdgeEventFetcher(ctx.getRuleChainService())); fetchers.add(new AdminSettingsEdgeEventFetcher(ctx.getAdminSettingsService(), ctx.getFreemarkerConfig())); - fetchers.add(new DeviceProfilesEdgeEventFetcher(ctx.getDeviceProfileService())); - fetchers.add(new AssetProfilesEdgeEventFetcher(ctx.getAssetProfileService())); fetchers.add(new TenantAdminUsersEdgeEventFetcher(ctx.getUserService())); Customer publicCustomer = ctx.getCustomerService().findOrCreatePublicCustomer(edge.getTenantId()); fetchers.add(new CustomerEdgeEventFetcher(publicCustomer.getId())); @@ -61,6 +59,8 @@ public class EdgeSyncCursor { fetchers.add(new CustomerUsersEdgeEventFetcher(ctx.getUserService(), edge.getCustomerId())); } } + fetchers.add(new DeviceProfilesEdgeEventFetcher(ctx.getDeviceProfileService())); + fetchers.add(new AssetProfilesEdgeEventFetcher(ctx.getAssetProfileService())); fetchers.add(new DevicesEdgeEventFetcher(ctx.getDeviceService())); fetchers.add(new AssetsEdgeEventFetcher(ctx.getAssetService())); fetchers.add(new EntityViewsEdgeEventFetcher(ctx.getEntityViewService())); diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/BaseEdgeProcessor.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/BaseEdgeProcessor.java index 56f4f06fa3..ea2236374e 100644 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/BaseEdgeProcessor.java +++ b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/BaseEdgeProcessor.java @@ -23,9 +23,14 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.context.annotation.Lazy; import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.server.cluster.TbClusterService; +import org.thingsboard.server.common.data.Dashboard; import org.thingsboard.server.common.data.Device; +import org.thingsboard.server.common.data.DeviceProfile; import org.thingsboard.server.common.data.EdgeUtils; import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.EntityView; +import org.thingsboard.server.common.data.asset.Asset; +import org.thingsboard.server.common.data.asset.AssetProfile; import org.thingsboard.server.common.data.edge.Edge; import org.thingsboard.server.common.data.edge.EdgeEvent; import org.thingsboard.server.common.data.edge.EdgeEventActionType; @@ -43,8 +48,11 @@ import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.id.UserId; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; +import org.thingsboard.server.common.data.relation.EntityRelation; +import org.thingsboard.server.common.data.relation.RelationTypeGroup; import org.thingsboard.server.common.data.rule.RuleChain; import org.thingsboard.server.common.data.rule.RuleChainConnectionInfo; +import org.thingsboard.server.common.msg.TbMsgMetaData; import org.thingsboard.server.dao.alarm.AlarmService; import org.thingsboard.server.dao.asset.AssetProfileService; import org.thingsboard.server.dao.asset.AssetService; @@ -106,6 +114,7 @@ import java.util.concurrent.locks.ReentrantLock; public abstract class BaseEdgeProcessor { protected static final Lock deviceCreationLock = new ReentrantLock(); + protected static final Lock assetCreationLock = new ReentrantLock(); protected static final int DEFAULT_PAGE_SIZE = 100; @@ -197,6 +206,21 @@ public abstract class BaseEdgeProcessor { @Autowired protected DataValidator deviceValidator; + @Autowired + protected DataValidator deviceProfileValidator; + + @Autowired + protected DataValidator assetValidator; + + @Autowired + protected DataValidator assetProfileValidator; + + @Autowired + protected DataValidator dashboardValidator; + + @Autowired + protected DataValidator entityViewValidator; + @Autowired protected EdgeMsgConstructor edgeMsgConstructor; @@ -520,4 +544,28 @@ public abstract class BaseEdgeProcessor { return false; } } + + protected void createRelationFromEdge(TenantId tenantId, EdgeId edgeId, EntityId entityId) { + EntityRelation relation = new EntityRelation(); + relation.setFrom(edgeId); + relation.setTo(entityId); + relation.setTypeGroup(RelationTypeGroup.COMMON); + relation.setType(EntityRelation.EDGE_TYPE); + relationService.saveRelation(tenantId, relation); + } + + protected TbMsgMetaData getActionTbMsgMetaData(Edge edge, CustomerId customerId) { + TbMsgMetaData metaData = getTbMsgMetaData(edge); + if (customerId != null && !customerId.isNullUid()) { + metaData.putValue("customerId", customerId.toString()); + } + return metaData; + } + + protected TbMsgMetaData getTbMsgMetaData(Edge edge) { + TbMsgMetaData metaData = new TbMsgMetaData(); + metaData.putValue("edgeId", edge.getId().toString()); + metaData.putValue("edgeName", edge.getName()); + return metaData; + } } diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/asset/AssetEdgeProcessor.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/asset/AssetEdgeProcessor.java index d7824a7467..3e61a538c5 100644 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/asset/AssetEdgeProcessor.java +++ b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/asset/AssetEdgeProcessor.java @@ -15,23 +15,113 @@ */ package org.thingsboard.server.service.edge.rpc.processor.asset; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.node.ObjectNode; +import com.google.common.util.concurrent.Futures; +import com.google.common.util.concurrent.ListenableFuture; import lombok.extern.slf4j.Slf4j; +import org.springframework.data.util.Pair; import org.springframework.stereotype.Component; +import org.thingsboard.common.util.JacksonUtil; +import org.thingsboard.server.common.data.DataConstants; import org.thingsboard.server.common.data.EdgeUtils; import org.thingsboard.server.common.data.asset.Asset; import org.thingsboard.server.common.data.asset.AssetProfile; +import org.thingsboard.server.common.data.edge.Edge; import org.thingsboard.server.common.data.edge.EdgeEvent; +import org.thingsboard.server.common.data.edge.EdgeEventActionType; +import org.thingsboard.server.common.data.edge.EdgeEventType; import org.thingsboard.server.common.data.id.AssetId; +import org.thingsboard.server.common.data.id.CustomerId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.msg.TbMsgType; +import org.thingsboard.server.common.msg.TbMsg; +import org.thingsboard.server.common.msg.TbMsgDataType; +import org.thingsboard.server.dao.asset.BaseAssetService; +import org.thingsboard.server.dao.exception.DataValidationException; import org.thingsboard.server.gen.edge.v1.AssetUpdateMsg; import org.thingsboard.server.gen.edge.v1.DownlinkMsg; import org.thingsboard.server.gen.edge.v1.UpdateMsgType; +import org.thingsboard.server.queue.TbQueueCallback; +import org.thingsboard.server.queue.TbQueueMsgMetadata; import org.thingsboard.server.queue.util.TbCoreComponent; -import org.thingsboard.server.service.edge.rpc.processor.BaseEdgeProcessor; + +import java.util.UUID; @Component @Slf4j @TbCoreComponent -public class AssetEdgeProcessor extends BaseEdgeProcessor { +public class AssetEdgeProcessor extends BaseAssetProcessor { + + public ListenableFuture processAssetMsgFromEdge(TenantId tenantId, Edge edge, AssetUpdateMsg assetUpdateMsg) { + log.trace("[{}] executing processAssetMsgFromEdge [{}] from edge [{}]", tenantId, assetUpdateMsg, edge.getName()); + AssetId assetId = new AssetId(new UUID(assetUpdateMsg.getIdMSB(), assetUpdateMsg.getIdLSB())); + try { + edgeSynchronizationManager.getSync().set(true); + + switch (assetUpdateMsg.getMsgType()) { + case ENTITY_CREATED_RPC_MESSAGE: + case ENTITY_UPDATED_RPC_MESSAGE: + saveOrUpdateAsset(tenantId, assetId, assetUpdateMsg, edge); + return Futures.immediateFuture(null); + case ENTITY_DELETED_RPC_MESSAGE: + Asset assetToDelete = assetService.findAssetById(tenantId, assetId); + if (assetToDelete != null) { + assetService.unassignAssetFromEdge(tenantId, assetId, edge.getId()); + } + return Futures.immediateFuture(null); + case UNRECOGNIZED: + default: + return handleUnsupportedMsgType(assetUpdateMsg.getMsgType()); + } + } catch (DataValidationException e) { + if (e.getMessage().contains("limit reached")) { + log.warn("[{}] Number of allowed asset violated {}", tenantId, assetUpdateMsg, e); + return Futures.immediateFuture(null); + } else { + return Futures.immediateFailedFuture(e); + } + } finally { + edgeSynchronizationManager.getSync().remove(); + } + } + + private void saveOrUpdateAsset(TenantId tenantId, AssetId assetId, AssetUpdateMsg assetUpdateMsg, Edge edge) { + CustomerId customerId = safeGetCustomerId(assetUpdateMsg.getCustomerIdMSB(), assetUpdateMsg.getCustomerIdLSB()); + Pair resultPair = super.saveOrUpdateAsset(tenantId, assetId, assetUpdateMsg, customerId); + Boolean created = resultPair.getFirst(); + if (created) { + createRelationFromEdge(tenantId, edge.getId(), assetId); + pushAssetCreatedEventToRuleEngine(tenantId, edge, assetId); + assetService.assignAssetToEdge(tenantId, assetId, edge.getId()); + } + Boolean assetNameUpdated = resultPair.getSecond(); + if (assetNameUpdated) { + saveEdgeEvent(tenantId, edge.getId(), EdgeEventType.ASSET, EdgeEventActionType.UPDATED, assetId, null); + } + } + + private void pushAssetCreatedEventToRuleEngine(TenantId tenantId, Edge edge, AssetId assetId) { + try { + Asset asset = assetService.findAssetById(tenantId, assetId); + ObjectNode entityNode = JacksonUtil.OBJECT_MAPPER.valueToTree(asset); + TbMsg tbMsg = TbMsg.newMsg(TbMsgType.ENTITY_CREATED, assetId, asset.getCustomerId(), + getActionTbMsgMetaData(edge, asset.getCustomerId()), TbMsgDataType.JSON, JacksonUtil.OBJECT_MAPPER.writeValueAsString(entityNode)); + tbClusterService.pushMsgToRuleEngine(tenantId, assetId, tbMsg, new TbQueueCallback() { + @Override + public void onSuccess(TbQueueMsgMetadata metadata) { + log.debug("Successfully send ENTITY_CREATED EVENT to rule engine [{}]", asset); + } + + @Override + public void onFailure(Throwable t) { + log.warn("Failed to send ENTITY_CREATED EVENT to rule engine [{}]", asset, t); + } + }); + } catch (JsonProcessingException | IllegalArgumentException e) { + log.warn("[{}] Failed to push asset action to rule engine: {}", assetId, DataConstants.ENTITY_CREATED, e); + } + } public DownlinkMsg convertAssetEventToDownlink(EdgeEvent edgeEvent) { AssetId assetId = new AssetId(edgeEvent.getEntityId()); @@ -43,7 +133,7 @@ public class AssetEdgeProcessor extends BaseEdgeProcessor { case ASSIGNED_TO_CUSTOMER: case UNASSIGNED_FROM_CUSTOMER: Asset asset = assetService.findAssetById(edgeEvent.getTenantId(), assetId); - if (asset != null) { + if (asset != null && !BaseAssetService.TB_SERVICE_QUEUE.equals(asset.getType())) { UpdateMsgType msgType = getUpdateMsgType(edgeEvent.getAction()); AssetUpdateMsg assetUpdateMsg = assetMsgConstructor.constructAssetUpdatedMsg(msgType, asset); diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/asset/AssetProfileEdgeProcessor.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/asset/AssetProfileEdgeProcessor.java index ec0e0b9761..3cae9c5123 100644 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/asset/AssetProfileEdgeProcessor.java +++ b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/asset/AssetProfileEdgeProcessor.java @@ -15,22 +15,91 @@ */ package org.thingsboard.server.service.edge.rpc.processor.asset; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.node.ObjectNode; +import com.google.common.util.concurrent.Futures; +import com.google.common.util.concurrent.ListenableFuture; import lombok.extern.slf4j.Slf4j; import org.springframework.stereotype.Component; +import org.thingsboard.common.util.JacksonUtil; +import org.thingsboard.server.common.data.DataConstants; import org.thingsboard.server.common.data.EdgeUtils; import org.thingsboard.server.common.data.asset.AssetProfile; +import org.thingsboard.server.common.data.edge.Edge; import org.thingsboard.server.common.data.edge.EdgeEvent; import org.thingsboard.server.common.data.id.AssetProfileId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.msg.TbMsgType; +import org.thingsboard.server.common.msg.TbMsg; +import org.thingsboard.server.common.msg.TbMsgDataType; +import org.thingsboard.server.dao.exception.DataValidationException; import org.thingsboard.server.gen.edge.v1.AssetProfileUpdateMsg; import org.thingsboard.server.gen.edge.v1.DownlinkMsg; import org.thingsboard.server.gen.edge.v1.UpdateMsgType; +import org.thingsboard.server.queue.TbQueueCallback; +import org.thingsboard.server.queue.TbQueueMsgMetadata; import org.thingsboard.server.queue.util.TbCoreComponent; -import org.thingsboard.server.service.edge.rpc.processor.BaseEdgeProcessor; + +import java.util.UUID; @Component @Slf4j @TbCoreComponent -public class AssetProfileEdgeProcessor extends BaseEdgeProcessor { +public class AssetProfileEdgeProcessor extends BaseAssetProfileProcessor { + + public ListenableFuture processAssetProfileMsgFromEdge(TenantId tenantId, Edge edge, AssetProfileUpdateMsg assetProfileUpdateMsg) { + log.trace("[{}] executing processAssetProfileMsgFromEdge [{}] from edge [{}]", tenantId, assetProfileUpdateMsg, edge.getName()); + AssetProfileId assetProfileId = new AssetProfileId(new UUID(assetProfileUpdateMsg.getIdMSB(), assetProfileUpdateMsg.getIdLSB())); + try { + edgeSynchronizationManager.getSync().set(true); + + switch (assetProfileUpdateMsg.getMsgType()) { + case ENTITY_CREATED_RPC_MESSAGE: + case ENTITY_UPDATED_RPC_MESSAGE: + saveOrUpdateAssetProfile(tenantId, assetProfileId, assetProfileUpdateMsg, edge); + return Futures.immediateFuture(null); + case ENTITY_DELETED_RPC_MESSAGE: + case UNRECOGNIZED: + default: + return handleUnsupportedMsgType(assetProfileUpdateMsg.getMsgType()); + } + } catch (DataValidationException e) { + log.warn("Failed to process AssetProfileUpdateMsg from Edge [{}]", assetProfileUpdateMsg, e); + return Futures.immediateFailedFuture(e); + } finally { + edgeSynchronizationManager.getSync().remove(); + } + } + + private void saveOrUpdateAssetProfile(TenantId tenantId, AssetProfileId assetProfileId, AssetProfileUpdateMsg assetProfileUpdateMsg, Edge edge) { + boolean created = super.saveOrUpdateAssetProfile(tenantId, assetProfileId, assetProfileUpdateMsg); + if (created) { + createRelationFromEdge(tenantId, edge.getId(), assetProfileId); + pushAssetProfileCreatedEventToRuleEngine(tenantId, edge, assetProfileId); + } + } + + private void pushAssetProfileCreatedEventToRuleEngine(TenantId tenantId, Edge edge, AssetProfileId assetProfileId) { + try { + AssetProfile assetProfile = assetProfileService.findAssetProfileById(tenantId, assetProfileId); + ObjectNode entityNode = JacksonUtil.OBJECT_MAPPER.valueToTree(assetProfile); + TbMsg tbMsg = TbMsg.newMsg(TbMsgType.ENTITY_CREATED, assetProfileId, getTbMsgMetaData(edge), + TbMsgDataType.JSON, JacksonUtil.OBJECT_MAPPER.writeValueAsString(entityNode)); + tbClusterService.pushMsgToRuleEngine(tenantId, assetProfileId, tbMsg, new TbQueueCallback() { + @Override + public void onSuccess(TbQueueMsgMetadata metadata) { + log.debug("Successfully send ENTITY_CREATED EVENT to rule engine [{}]", assetProfile); + } + + @Override + public void onFailure(Throwable t) { + log.warn("Failed to send ENTITY_CREATED EVENT to rule engine [{}]", assetProfile, t); + } + }); + } catch (JsonProcessingException | IllegalArgumentException e) { + log.warn("[{}] Failed to push asset profile action to rule engine: {}", assetProfileId, DataConstants.ENTITY_CREATED, e); + } + } public DownlinkMsg convertAssetProfileEventToDownlink(EdgeEvent edgeEvent) { AssetProfileId assetProfileId = new AssetProfileId(edgeEvent.getEntityId()); diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/asset/BaseAssetProcessor.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/asset/BaseAssetProcessor.java new file mode 100644 index 0000000000..5d728cb4e8 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/asset/BaseAssetProcessor.java @@ -0,0 +1,77 @@ +/** + * Copyright © 2016-2023 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.edge.rpc.processor.asset; + +import com.datastax.oss.driver.api.core.uuid.Uuids; +import lombok.extern.slf4j.Slf4j; +import org.springframework.data.util.Pair; +import org.thingsboard.common.util.JacksonUtil; +import org.thingsboard.server.common.data.StringUtils; +import org.thingsboard.server.common.data.asset.Asset; +import org.thingsboard.server.common.data.id.AssetId; +import org.thingsboard.server.common.data.id.AssetProfileId; +import org.thingsboard.server.common.data.id.CustomerId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.gen.edge.v1.AssetUpdateMsg; +import org.thingsboard.server.service.edge.rpc.processor.BaseEdgeProcessor; + +import java.util.UUID; + +@Slf4j +public abstract class BaseAssetProcessor extends BaseEdgeProcessor { + + protected Pair saveOrUpdateAsset(TenantId tenantId, AssetId assetId, AssetUpdateMsg assetUpdateMsg, CustomerId customerId) { + boolean created = false; + boolean assetNameUpdated = false; + assetCreationLock.lock(); + try { + Asset asset = assetService.findAssetById(tenantId, assetId); + String assetName = assetUpdateMsg.getName(); + if (asset == null) { + created = true; + asset = new Asset(); + asset.setTenantId(tenantId); + asset.setCreatedTime(Uuids.unixTimestamp(assetId.getId())); + } + Asset assetByName = assetService.findAssetByTenantIdAndName(tenantId, assetName); + if (assetByName != null && !assetByName.getId().equals(assetId)) { + assetName = assetName + "_" + StringUtils.randomAlphanumeric(15); + log.warn("Asset with name {} already exists. Renaming asset name to {}", + assetUpdateMsg.getName(), assetName); + assetNameUpdated = true; + } + asset.setName(assetName); + asset.setType(assetUpdateMsg.getType()); + asset.setLabel(assetUpdateMsg.hasLabel() ? assetUpdateMsg.getLabel() : null); + asset.setAdditionalInfo(assetUpdateMsg.hasAdditionalInfo() + ? JacksonUtil.toJsonNode(assetUpdateMsg.getAdditionalInfo()) : null); + + UUID assetProfileUUID = safeGetUUID(assetUpdateMsg.getAssetProfileIdMSB(), assetUpdateMsg.getAssetProfileIdLSB()); + asset.setAssetProfileId(assetProfileUUID != null ? new AssetProfileId(assetProfileUUID) : null); + + asset.setCustomerId(customerId); + + assetValidator.validate(asset, Asset::getTenantId); + if (created) { + asset.setId(assetId); + } + assetService.saveAsset(asset, false); + } finally { + assetCreationLock.unlock(); + } + return Pair.of(created, assetNameUpdated); + } +} diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/asset/BaseAssetProfileProcessor.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/asset/BaseAssetProfileProcessor.java new file mode 100644 index 0000000000..4fe778402f --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/asset/BaseAssetProfileProcessor.java @@ -0,0 +1,69 @@ +/** + * Copyright © 2016-2023 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.edge.rpc.processor.asset; + +import com.datastax.oss.driver.api.core.uuid.Uuids; +import lombok.extern.slf4j.Slf4j; +import org.thingsboard.server.common.data.asset.AssetProfile; +import org.thingsboard.server.common.data.id.AssetProfileId; +import org.thingsboard.server.common.data.id.DashboardId; +import org.thingsboard.server.common.data.id.RuleChainId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.gen.edge.v1.AssetProfileUpdateMsg; +import org.thingsboard.server.service.edge.rpc.processor.BaseEdgeProcessor; + +import java.nio.charset.StandardCharsets; +import java.util.UUID; + +@Slf4j +public class BaseAssetProfileProcessor extends BaseEdgeProcessor { + + protected boolean saveOrUpdateAssetProfile(TenantId tenantId, AssetProfileId assetProfileId, AssetProfileUpdateMsg assetProfileUpdateMsg) { + boolean created = false; + assetCreationLock.lock(); + try { + AssetProfile assetProfile = assetProfileService.findAssetProfileById(tenantId, assetProfileId); + String assetProfileName = assetProfileUpdateMsg.getName(); + if (assetProfile == null) { + created = true; + assetProfile = new AssetProfile(); + assetProfile.setTenantId(tenantId); + assetProfile.setCreatedTime(Uuids.unixTimestamp(assetProfileId.getId())); + } + assetProfile.setName(assetProfileName); + assetProfile.setDefault(assetProfileUpdateMsg.getDefault()); + assetProfile.setDefaultQueueName(assetProfileUpdateMsg.hasDefaultQueueName() ? assetProfileUpdateMsg.getDefaultQueueName() : null); + assetProfile.setDescription(assetProfileUpdateMsg.hasDescription() ? assetProfileUpdateMsg.getDescription() : null); + assetProfile.setImage(assetProfileUpdateMsg.hasImage() + ? new String(assetProfileUpdateMsg.getImage().toByteArray(), StandardCharsets.UTF_8) : null); + + UUID defaultRuleChainUUID = safeGetUUID(assetProfileUpdateMsg.getDefaultRuleChainIdMSB(), assetProfileUpdateMsg.getDefaultRuleChainIdLSB()); + assetProfile.setDefaultRuleChainId(defaultRuleChainUUID != null ? new RuleChainId(defaultRuleChainUUID) : null); + + UUID defaultDashboardUUID = safeGetUUID(assetProfileUpdateMsg.getDefaultDashboardIdMSB(), assetProfileUpdateMsg.getDefaultDashboardIdLSB()); + assetProfile.setDefaultDashboardId(defaultDashboardUUID != null ? new DashboardId(defaultDashboardUUID) : null); + + assetProfileValidator.validate(assetProfile, AssetProfile::getTenantId); + if (created) { + assetProfile.setId(assetProfileId); + } + assetProfileService.saveAssetProfile(assetProfile, false); + } finally { + assetCreationLock.unlock(); + } + return created; + } +} diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/dashboard/BaseDashboardProcessor.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/dashboard/BaseDashboardProcessor.java new file mode 100644 index 0000000000..b055c03cf7 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/dashboard/BaseDashboardProcessor.java @@ -0,0 +1,77 @@ +/** + * Copyright © 2016-2023 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.edge.rpc.processor.dashboard; + +import com.datastax.oss.driver.api.core.uuid.Uuids; +import com.fasterxml.jackson.core.type.TypeReference; +import lombok.extern.slf4j.Slf4j; +import org.thingsboard.common.util.JacksonUtil; +import org.thingsboard.server.common.data.Dashboard; +import org.thingsboard.server.common.data.ShortCustomerInfo; +import org.thingsboard.server.common.data.id.CustomerId; +import org.thingsboard.server.common.data.id.DashboardId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.gen.edge.v1.DashboardUpdateMsg; +import org.thingsboard.server.service.edge.rpc.processor.BaseEdgeProcessor; + +import java.util.Set; + +@Slf4j +public abstract class BaseDashboardProcessor extends BaseEdgeProcessor { + + protected boolean saveOrUpdateDashboard(TenantId tenantId, DashboardId dashboardId, DashboardUpdateMsg dashboardUpdateMsg, CustomerId customerId) { + boolean created = false; + Dashboard dashboard = dashboardService.findDashboardById(tenantId, dashboardId); + if (dashboard == null) { + created = true; + dashboard = new Dashboard(); + dashboard.setTenantId(tenantId); + dashboard.setCreatedTime(Uuids.unixTimestamp(dashboardId.getId())); + } + dashboard.setTitle(dashboardUpdateMsg.getTitle()); + dashboard.setConfiguration(JacksonUtil.toJsonNode(dashboardUpdateMsg.getConfiguration())); + Set assignedCustomers = null; + if (dashboardUpdateMsg.hasAssignedCustomers()) { + assignedCustomers = JacksonUtil.fromString(dashboardUpdateMsg.getAssignedCustomers(), new TypeReference<>() { + }); + dashboard.setAssignedCustomers(assignedCustomers); + } + + dashboardValidator.validate(dashboard, Dashboard::getTenantId); + if (created) { + dashboard.setId(dashboardId); + } + Dashboard savedDashboard = dashboardService.saveDashboard(dashboard, false); + if (assignedCustomers != null && !assignedCustomers.isEmpty()) { + for (ShortCustomerInfo assignedCustomer : assignedCustomers) { + if (assignedCustomer.getCustomerId().equals(customerId)) { + dashboardService.assignDashboardToCustomer(tenantId, dashboardId, assignedCustomer.getCustomerId()); + } + } + } else { + unassignCustomersFromDashboard(tenantId, savedDashboard); + } + return created; + } + + private void unassignCustomersFromDashboard(TenantId tenantId, Dashboard dashboard) { + if (dashboard.getAssignedCustomers() != null && !dashboard.getAssignedCustomers().isEmpty()) { + for (ShortCustomerInfo assignedCustomer : dashboard.getAssignedCustomers()) { + dashboardService.unassignDashboardFromCustomer(tenantId, dashboard.getId(), assignedCustomer.getCustomerId()); + } + } + } +} diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/dashboard/DashboardEdgeProcessor.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/dashboard/DashboardEdgeProcessor.java index 14f566db0a..59f12ec64b 100644 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/dashboard/DashboardEdgeProcessor.java +++ b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/dashboard/DashboardEdgeProcessor.java @@ -15,22 +15,103 @@ */ package org.thingsboard.server.service.edge.rpc.processor.dashboard; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.node.ObjectNode; +import com.google.common.util.concurrent.Futures; +import com.google.common.util.concurrent.ListenableFuture; import lombok.extern.slf4j.Slf4j; import org.springframework.stereotype.Component; +import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.server.common.data.Dashboard; +import org.thingsboard.server.common.data.DataConstants; import org.thingsboard.server.common.data.EdgeUtils; +import org.thingsboard.server.common.data.edge.Edge; import org.thingsboard.server.common.data.edge.EdgeEvent; +import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.DashboardId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.msg.TbMsgType; +import org.thingsboard.server.common.msg.TbMsg; +import org.thingsboard.server.common.msg.TbMsgDataType; +import org.thingsboard.server.dao.exception.DataValidationException; import org.thingsboard.server.gen.edge.v1.DashboardUpdateMsg; import org.thingsboard.server.gen.edge.v1.DownlinkMsg; import org.thingsboard.server.gen.edge.v1.UpdateMsgType; +import org.thingsboard.server.queue.TbQueueCallback; +import org.thingsboard.server.queue.TbQueueMsgMetadata; import org.thingsboard.server.queue.util.TbCoreComponent; -import org.thingsboard.server.service.edge.rpc.processor.BaseEdgeProcessor; + +import java.util.UUID; @Component @Slf4j @TbCoreComponent -public class DashboardEdgeProcessor extends BaseEdgeProcessor { +public class DashboardEdgeProcessor extends BaseDashboardProcessor { + + public ListenableFuture processDashboardMsgFromEdge(TenantId tenantId, Edge edge, DashboardUpdateMsg dashboardUpdateMsg) { + log.trace("[{}] executing processDashboardMsgFromEdge [{}] from edge [{}]", tenantId, dashboardUpdateMsg, edge.getName()); + DashboardId dashboardId = new DashboardId(new UUID(dashboardUpdateMsg.getIdMSB(), dashboardUpdateMsg.getIdLSB())); + try { + edgeSynchronizationManager.getSync().set(true); + + switch (dashboardUpdateMsg.getMsgType()) { + case ENTITY_CREATED_RPC_MESSAGE: + case ENTITY_UPDATED_RPC_MESSAGE: + saveOrUpdateDashboard(tenantId, dashboardId, dashboardUpdateMsg, edge); + return Futures.immediateFuture(null); + case ENTITY_DELETED_RPC_MESSAGE: + Dashboard dashboardToDelete = dashboardService.findDashboardById(tenantId, dashboardId); + if (dashboardToDelete != null) { + dashboardService.unassignDashboardFromEdge(tenantId, dashboardId, edge.getId()); + } + return Futures.immediateFuture(null); + case UNRECOGNIZED: + default: + return handleUnsupportedMsgType(dashboardUpdateMsg.getMsgType()); + } + } catch (DataValidationException e) { + if (e.getMessage().contains("limit reached")) { + log.warn("[{}] Number of allowed dashboard violated {}", tenantId, dashboardUpdateMsg, e); + return Futures.immediateFuture(null); + } else { + return Futures.immediateFailedFuture(e); + } + } finally { + edgeSynchronizationManager.getSync().remove(); + } + } + + private void saveOrUpdateDashboard(TenantId tenantId, DashboardId dashboardId, DashboardUpdateMsg dashboardUpdateMsg, Edge edge) { + CustomerId customerId = safeGetCustomerId(dashboardUpdateMsg.getCustomerIdMSB(), dashboardUpdateMsg.getCustomerIdLSB()); + boolean created = super.saveOrUpdateDashboard(tenantId, dashboardId, dashboardUpdateMsg, customerId); + if (created) { + createRelationFromEdge(tenantId, edge.getId(), dashboardId); + pushDashboardCreatedEventToRuleEngine(tenantId, edge, dashboardId); + dashboardService.assignDashboardToEdge(tenantId, dashboardId, edge.getId()); + } + } + + private void pushDashboardCreatedEventToRuleEngine(TenantId tenantId, Edge edge, DashboardId dashboardId) { + try { + Dashboard dashboard = dashboardService.findDashboardById(tenantId, dashboardId); + ObjectNode entityNode = JacksonUtil.OBJECT_MAPPER.valueToTree(dashboard); + TbMsg tbMsg = TbMsg.newMsg(TbMsgType.ENTITY_CREATED, dashboardId, null, + getActionTbMsgMetaData(edge, null), TbMsgDataType.JSON, JacksonUtil.OBJECT_MAPPER.writeValueAsString(entityNode)); + tbClusterService.pushMsgToRuleEngine(tenantId, dashboardId, tbMsg, new TbQueueCallback() { + @Override + public void onSuccess(TbQueueMsgMetadata metadata) { + log.debug("Successfully send ENTITY_CREATED EVENT to rule engine [{}]", dashboard); + } + + @Override + public void onFailure(Throwable t) { + log.warn("Failed to send ENTITY_CREATED EVENT to rule engine [{}]", dashboard, t); + } + }); + } catch (JsonProcessingException | IllegalArgumentException e) { + log.warn("[{}] Failed to push dashboard action to rule engine: {}", dashboardId, DataConstants.ENTITY_CREATED, e); + } + } public DownlinkMsg convertDashboardEventToDownlink(EdgeEvent edgeEvent) { DashboardId dashboardId = new DashboardId(edgeEvent.getEntityId()); diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/device/BaseDeviceProcessor.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/device/BaseDeviceProcessor.java index 1421cf32c0..580f752ac1 100644 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/device/BaseDeviceProcessor.java +++ b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/device/BaseDeviceProcessor.java @@ -57,13 +57,13 @@ public abstract class BaseDeviceProcessor extends BaseEdgeProcessor { device = new Device(); device.setTenantId(tenantId); device.setCreatedTime(Uuids.unixTimestamp(deviceId.getId())); - Device deviceByName = deviceService.findDeviceByTenantIdAndName(tenantId, deviceName); - if (deviceByName != null) { - deviceName = deviceName + "_" + StringUtils.randomAlphabetic(15); - log.warn("Device with name {} already exists. Renaming device name to {}", - deviceUpdateMsg.getName(), deviceName); - deviceNameUpdated = true; - } + } + Device deviceByName = deviceService.findDeviceByTenantIdAndName(tenantId, deviceName); + if (deviceByName != null && !deviceByName.getId().equals(deviceId)) { + deviceName = deviceName + "_" + StringUtils.randomAlphabetic(15); + log.warn("Device with name {} already exists. Renaming device name to {}", + deviceUpdateMsg.getName(), deviceName); + deviceNameUpdated = true; } device.setName(deviceName); device.setType(deviceUpdateMsg.getType()); diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/device/BaseDeviceProfileProcessor.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/device/BaseDeviceProfileProcessor.java new file mode 100644 index 0000000000..662a497528 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/device/BaseDeviceProfileProcessor.java @@ -0,0 +1,102 @@ +/** + * Copyright © 2016-2023 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.edge.rpc.processor.device; + +import com.datastax.oss.driver.api.core.uuid.Uuids; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Autowired; +import org.thingsboard.server.common.data.DeviceProfile; +import org.thingsboard.server.common.data.DeviceProfileProvisionType; +import org.thingsboard.server.common.data.DeviceProfileType; +import org.thingsboard.server.common.data.DeviceTransportType; +import org.thingsboard.server.common.data.StringUtils; +import org.thingsboard.server.common.data.device.profile.DeviceProfileData; +import org.thingsboard.server.common.data.id.DashboardId; +import org.thingsboard.server.common.data.id.DeviceProfileId; +import org.thingsboard.server.common.data.id.OtaPackageId; +import org.thingsboard.server.common.data.id.RuleChainId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.gen.edge.v1.DeviceProfileUpdateMsg; +import org.thingsboard.server.queue.util.DataDecodingEncodingService; +import org.thingsboard.server.service.edge.rpc.processor.BaseEdgeProcessor; + +import java.nio.charset.StandardCharsets; +import java.util.Optional; +import java.util.UUID; + +@Slf4j +public class BaseDeviceProfileProcessor extends BaseEdgeProcessor { + + @Autowired + private DataDecodingEncodingService dataDecodingEncodingService; + + protected boolean saveOrUpdateDeviceProfile(TenantId tenantId, DeviceProfileId deviceProfileId, DeviceProfileUpdateMsg deviceProfileUpdateMsg) { + boolean created = false; + deviceCreationLock.lock(); + try { + DeviceProfile deviceProfile = deviceProfileService.findDeviceProfileById(tenantId, deviceProfileId); + if (deviceProfile == null) { + created = true; + deviceProfile = new DeviceProfile(); + deviceProfile.setTenantId(tenantId); + deviceProfile.setCreatedTime(Uuids.unixTimestamp(deviceProfileId.getId())); + } + deviceProfile.setName(deviceProfileUpdateMsg.getName()); + deviceProfile.setDescription(deviceProfileUpdateMsg.hasDescription() ? deviceProfileUpdateMsg.getDescription() : null); + deviceProfile.setDefault(deviceProfileUpdateMsg.getDefault()); + deviceProfile.setType(DeviceProfileType.valueOf(deviceProfileUpdateMsg.getType())); + deviceProfile.setTransportType(deviceProfileUpdateMsg.hasTransportType() + ? DeviceTransportType.valueOf(deviceProfileUpdateMsg.getTransportType()) : DeviceTransportType.DEFAULT); + deviceProfile.setImage(deviceProfileUpdateMsg.hasImage() + ? new String(deviceProfileUpdateMsg.getImage().toByteArray(), StandardCharsets.UTF_8) : null); + deviceProfile.setProvisionType(deviceProfileUpdateMsg.hasProvisionType() + ? DeviceProfileProvisionType.valueOf(deviceProfileUpdateMsg.getProvisionType()) : DeviceProfileProvisionType.DISABLED); + deviceProfile.setProvisionDeviceKey(deviceProfileUpdateMsg.hasProvisionDeviceKey() + ? deviceProfileUpdateMsg.getProvisionDeviceKey() : null); + deviceProfile.setDefaultQueueName(deviceProfileUpdateMsg.getDefaultQueueName()); + + Optional profileDataOpt = + dataDecodingEncodingService.decode(deviceProfileUpdateMsg.getProfileDataBytes().toByteArray()); + deviceProfile.setProfileData(profileDataOpt.orElse(null)); + + UUID defaultRuleChainUUID = safeGetUUID(deviceProfileUpdateMsg.getDefaultRuleChainIdMSB(), deviceProfileUpdateMsg.getDefaultRuleChainIdLSB()); + deviceProfile.setDefaultRuleChainId(defaultRuleChainUUID != null ? new RuleChainId(defaultRuleChainUUID) : null); + + UUID defaultDashboardUUID = safeGetUUID(deviceProfileUpdateMsg.getDefaultDashboardIdMSB(), deviceProfileUpdateMsg.getDefaultDashboardIdLSB()); + deviceProfile.setDefaultDashboardId(defaultDashboardUUID != null ? new DashboardId(defaultDashboardUUID) : null); + + String defaultQueueName = StringUtils.isNotBlank(deviceProfileUpdateMsg.getDefaultQueueName()) + ? deviceProfileUpdateMsg.getDefaultQueueName() : null; + deviceProfile.setDefaultQueueName(defaultQueueName); + + UUID firmwareUUID = safeGetUUID(deviceProfileUpdateMsg.getFirmwareIdMSB(), deviceProfileUpdateMsg.getFirmwareIdLSB()); + deviceProfile.setFirmwareId(firmwareUUID != null ? new OtaPackageId(firmwareUUID) : null); + + UUID softwareUUID = safeGetUUID(deviceProfileUpdateMsg.getSoftwareIdMSB(), deviceProfileUpdateMsg.getSoftwareIdLSB()); + deviceProfile.setSoftwareId(softwareUUID != null ? new OtaPackageId(softwareUUID) : null); + + + deviceProfileValidator.validate(deviceProfile, DeviceProfile::getTenantId); + if (created) { + deviceProfile.setId(deviceProfileId); + } + deviceProfileService.saveDeviceProfile(deviceProfile, false); + } finally { + deviceCreationLock.unlock(); + } + return created; + } +} diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/device/DeviceEdgeProcessor.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/device/DeviceEdgeProcessor.java index d083c5b16f..dfa5705840 100644 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/device/DeviceEdgeProcessor.java +++ b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/device/DeviceEdgeProcessor.java @@ -35,12 +35,8 @@ import org.thingsboard.server.common.data.edge.EdgeEventActionType; import org.thingsboard.server.common.data.edge.EdgeEventType; import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.DeviceId; -import org.thingsboard.server.common.data.id.EdgeId; -import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.msg.TbMsgType; -import org.thingsboard.server.common.data.relation.EntityRelation; -import org.thingsboard.server.common.data.relation.RelationTypeGroup; import org.thingsboard.server.common.data.rpc.RpcError; import org.thingsboard.server.common.data.security.DeviceCredentials; import org.thingsboard.server.common.msg.TbMsg; @@ -114,15 +110,6 @@ public class DeviceEdgeProcessor extends BaseDeviceProcessor { } } - private void createRelationFromEdge(TenantId tenantId, EdgeId edgeId, EntityId entityId) { - EntityRelation relation = new EntityRelation(); - relation.setFrom(edgeId); - relation.setTo(entityId); - relation.setTypeGroup(RelationTypeGroup.COMMON); - relation.setType(EntityRelation.EDGE_TYPE); - relationService.saveRelation(tenantId, relation); - } - private void pushDeviceCreatedEventToRuleEngine(TenantId tenantId, Edge edge, DeviceId deviceId) { try { Device device = deviceService.findDeviceById(tenantId, deviceId); @@ -137,7 +124,7 @@ public class DeviceEdgeProcessor extends BaseDeviceProcessor { @Override public void onFailure(Throwable t) { - log.debug("Failed to send ENTITY_CREATED EVENT to rule engine [{}]", device, t); + log.warn("Failed to send ENTITY_CREATED EVENT to rule engine [{}]", device, t); } }); } catch (JsonProcessingException | IllegalArgumentException e) { @@ -145,21 +132,6 @@ public class DeviceEdgeProcessor extends BaseDeviceProcessor { } } - private TbMsgMetaData getActionTbMsgMetaData(Edge edge, CustomerId customerId) { - TbMsgMetaData metaData = getTbMsgMetaData(edge); - if (customerId != null && !customerId.isNullUid()) { - metaData.putValue("customerId", customerId.toString()); - } - return metaData; - } - - private TbMsgMetaData getTbMsgMetaData(Edge edge) { - TbMsgMetaData metaData = new TbMsgMetaData(); - metaData.putValue("edgeId", edge.getId().toString()); - metaData.putValue("edgeName", edge.getName()); - return metaData; - } - public ListenableFuture processDeviceRpcCallFromEdge(TenantId tenantId, Edge edge, DeviceRpcCallMsg deviceRpcCallMsg) { log.trace("[{}] processDeviceRpcCallFromEdge [{}]", tenantId, deviceRpcCallMsg); if (deviceRpcCallMsg.hasResponseMsg()) { diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/device/DeviceProfileEdgeProcessor.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/device/DeviceProfileEdgeProcessor.java index c888ec2925..edaffd6f74 100644 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/device/DeviceProfileEdgeProcessor.java +++ b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/device/DeviceProfileEdgeProcessor.java @@ -15,22 +15,92 @@ */ package org.thingsboard.server.service.edge.rpc.processor.device; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.node.ObjectNode; +import com.google.common.util.concurrent.Futures; +import com.google.common.util.concurrent.ListenableFuture; import lombok.extern.slf4j.Slf4j; import org.springframework.stereotype.Component; +import org.thingsboard.common.util.JacksonUtil; +import org.thingsboard.server.common.data.DataConstants; import org.thingsboard.server.common.data.DeviceProfile; import org.thingsboard.server.common.data.EdgeUtils; +import org.thingsboard.server.common.data.edge.Edge; import org.thingsboard.server.common.data.edge.EdgeEvent; import org.thingsboard.server.common.data.id.DeviceProfileId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.msg.TbMsgType; +import org.thingsboard.server.common.msg.TbMsg; +import org.thingsboard.server.common.msg.TbMsgDataType; +import org.thingsboard.server.dao.exception.DataValidationException; import org.thingsboard.server.gen.edge.v1.DeviceProfileUpdateMsg; import org.thingsboard.server.gen.edge.v1.DownlinkMsg; import org.thingsboard.server.gen.edge.v1.UpdateMsgType; +import org.thingsboard.server.queue.TbQueueCallback; +import org.thingsboard.server.queue.TbQueueMsgMetadata; import org.thingsboard.server.queue.util.TbCoreComponent; -import org.thingsboard.server.service.edge.rpc.processor.BaseEdgeProcessor; + +import java.util.UUID; @Component @Slf4j @TbCoreComponent -public class DeviceProfileEdgeProcessor extends BaseEdgeProcessor { +public class DeviceProfileEdgeProcessor extends BaseDeviceProfileProcessor { + + + public ListenableFuture processDeviceProfileMsgFromEdge(TenantId tenantId, Edge edge, DeviceProfileUpdateMsg deviceProfileUpdateMsg) { + log.trace("[{}] executing processDeviceProfileMsgFromEdge [{}] from edge [{}]", tenantId, deviceProfileUpdateMsg, edge.getName()); + DeviceProfileId deviceProfileId = new DeviceProfileId(new UUID(deviceProfileUpdateMsg.getIdMSB(), deviceProfileUpdateMsg.getIdLSB())); + try { + edgeSynchronizationManager.getSync().set(true); + + switch (deviceProfileUpdateMsg.getMsgType()) { + case ENTITY_CREATED_RPC_MESSAGE: + case ENTITY_UPDATED_RPC_MESSAGE: + saveOrUpdateDeviceProfile(tenantId, deviceProfileId, deviceProfileUpdateMsg, edge); + return Futures.immediateFuture(null); + case ENTITY_DELETED_RPC_MESSAGE: + case UNRECOGNIZED: + default: + return handleUnsupportedMsgType(deviceProfileUpdateMsg.getMsgType()); + } + } catch (DataValidationException e) { + log.warn("Failed to process DeviceProfileUpdateMsg from Edge [{}]", deviceProfileUpdateMsg, e); + return Futures.immediateFailedFuture(e); + } finally { + edgeSynchronizationManager.getSync().remove(); + } + } + + private void saveOrUpdateDeviceProfile(TenantId tenantId, DeviceProfileId deviceProfileId, DeviceProfileUpdateMsg deviceProfileUpdateMsg, Edge edge) { + boolean created = super.saveOrUpdateDeviceProfile(tenantId, deviceProfileId, deviceProfileUpdateMsg); + if (created) { + createRelationFromEdge(tenantId, edge.getId(), deviceProfileId); + pushDeviceProfileCreatedEventToRuleEngine(tenantId, edge, deviceProfileId); + } + } + + private void pushDeviceProfileCreatedEventToRuleEngine(TenantId tenantId, Edge edge, DeviceProfileId deviceProfileId) { + try { + DeviceProfile deviceProfile = deviceProfileService.findDeviceProfileById(tenantId, deviceProfileId); + ObjectNode entityNode = JacksonUtil.OBJECT_MAPPER.valueToTree(deviceProfile); + TbMsg tbMsg = TbMsg.newMsg(TbMsgType.ENTITY_CREATED, deviceProfileId, getTbMsgMetaData(edge), + TbMsgDataType.JSON, JacksonUtil.OBJECT_MAPPER.writeValueAsString(entityNode)); + tbClusterService.pushMsgToRuleEngine(tenantId, deviceProfileId, tbMsg, new TbQueueCallback() { + @Override + public void onSuccess(TbQueueMsgMetadata metadata) { + log.debug("Successfully send ENTITY_CREATED EVENT to rule engine [{}]", deviceProfile); + } + + @Override + public void onFailure(Throwable t) { + log.warn("Failed to send ENTITY_CREATED EVENT to rule engine [{}]", deviceProfile, t); + } + }); + } catch (JsonProcessingException | IllegalArgumentException e) { + log.warn("[{}] Failed to push device profile action to rule engine: {}", deviceProfileId, DataConstants.ENTITY_CREATED, e); + } + } public DownlinkMsg convertDeviceProfileEventToDownlink(EdgeEvent edgeEvent) { DeviceProfileId deviceProfileId = new DeviceProfileId(edgeEvent.getEntityId()); diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/entityview/BaseEntityViewProcessor.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/entityview/BaseEntityViewProcessor.java new file mode 100644 index 0000000000..509b2e5196 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/entityview/BaseEntityViewProcessor.java @@ -0,0 +1,76 @@ +/** + * Copyright © 2016-2023 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.edge.rpc.processor.entityview; + +import com.datastax.oss.driver.api.core.uuid.Uuids; +import lombok.extern.slf4j.Slf4j; +import org.springframework.data.util.Pair; +import org.thingsboard.common.util.JacksonUtil; +import org.thingsboard.server.common.data.EntityView; +import org.thingsboard.server.common.data.StringUtils; +import org.thingsboard.server.common.data.id.AssetId; +import org.thingsboard.server.common.data.id.CustomerId; +import org.thingsboard.server.common.data.id.DeviceId; +import org.thingsboard.server.common.data.id.EntityViewId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.gen.edge.v1.EdgeEntityType; +import org.thingsboard.server.gen.edge.v1.EntityViewUpdateMsg; +import org.thingsboard.server.service.edge.rpc.processor.BaseEdgeProcessor; + +import java.util.UUID; + +@Slf4j +public abstract class BaseEntityViewProcessor extends BaseEdgeProcessor { + + protected Pair saveOrUpdateEntityView(TenantId tenantId, EntityViewId entityViewId, EntityViewUpdateMsg entityViewUpdateMsg, CustomerId customerId) { + boolean created = false; + boolean entityViewNameUpdated = false; + EntityView entityView = entityViewService.findEntityViewById(tenantId, entityViewId); + String entityViewName = entityViewUpdateMsg.getName(); + if (entityView == null) { + created = true; + entityView = new EntityView(); + entityView.setTenantId(tenantId); + entityView.setCreatedTime(Uuids.unixTimestamp(entityViewId.getId())); + } + EntityView entityViewByName = entityViewService.findEntityViewByTenantIdAndName(tenantId, entityViewName); + if (entityViewByName != null && !entityViewByName.getId().equals(entityViewId)) { + entityViewName = entityViewName + "_" + StringUtils.randomAlphanumeric(15); + log.warn("Entity view with name {} already exists. Renaming entity view name to {}", + entityViewUpdateMsg.getName(), entityViewName); + entityViewNameUpdated = true; + } + entityView.setName(entityViewName); + entityView.setType(entityViewUpdateMsg.getType()); + entityView.setCustomerId(customerId); + entityView.setAdditionalInfo(entityViewUpdateMsg.hasAdditionalInfo() ? + JacksonUtil.toJsonNode(entityViewUpdateMsg.getAdditionalInfo()) : null); + + UUID entityIdUUID = safeGetUUID(entityViewUpdateMsg.getEntityIdMSB(), entityViewUpdateMsg.getEntityIdLSB()); + if (EdgeEntityType.DEVICE.equals(entityViewUpdateMsg.getEntityType())) { + entityView.setEntityId(entityIdUUID != null ? new DeviceId(entityIdUUID) : null); + } else if (EdgeEntityType.ASSET.equals(entityViewUpdateMsg.getEntityType())) { + entityView.setEntityId(entityIdUUID != null ? new AssetId(entityIdUUID) : null); + } + + entityViewValidator.validate(entityView, EntityView::getTenantId); + if (created) { + entityView.setId(entityViewId); + } + entityViewService.saveEntityView(entityView, false); + return Pair.of(created, entityViewNameUpdated); + } +} diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/entityview/EntityViewEdgeProcessor.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/entityview/EntityViewEdgeProcessor.java index 0964a434ba..e78b6b4ff5 100644 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/entityview/EntityViewEdgeProcessor.java +++ b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/entityview/EntityViewEdgeProcessor.java @@ -15,22 +15,111 @@ */ package org.thingsboard.server.service.edge.rpc.processor.entityview; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.node.ObjectNode; +import com.google.common.util.concurrent.Futures; +import com.google.common.util.concurrent.ListenableFuture; import lombok.extern.slf4j.Slf4j; +import org.springframework.data.util.Pair; import org.springframework.stereotype.Component; +import org.thingsboard.common.util.JacksonUtil; +import org.thingsboard.server.common.data.DataConstants; import org.thingsboard.server.common.data.EdgeUtils; import org.thingsboard.server.common.data.EntityView; +import org.thingsboard.server.common.data.edge.Edge; import org.thingsboard.server.common.data.edge.EdgeEvent; +import org.thingsboard.server.common.data.edge.EdgeEventActionType; +import org.thingsboard.server.common.data.edge.EdgeEventType; +import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.EntityViewId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.msg.TbMsgType; +import org.thingsboard.server.common.msg.TbMsg; +import org.thingsboard.server.common.msg.TbMsgDataType; +import org.thingsboard.server.dao.exception.DataValidationException; import org.thingsboard.server.gen.edge.v1.DownlinkMsg; import org.thingsboard.server.gen.edge.v1.EntityViewUpdateMsg; import org.thingsboard.server.gen.edge.v1.UpdateMsgType; +import org.thingsboard.server.queue.TbQueueCallback; +import org.thingsboard.server.queue.TbQueueMsgMetadata; import org.thingsboard.server.queue.util.TbCoreComponent; -import org.thingsboard.server.service.edge.rpc.processor.BaseEdgeProcessor; + +import java.util.UUID; @Component @Slf4j @TbCoreComponent -public class EntityViewEdgeProcessor extends BaseEdgeProcessor { +public class EntityViewEdgeProcessor extends BaseEntityViewProcessor { + + public ListenableFuture processEntityViewMsgFromEdge(TenantId tenantId, Edge edge, EntityViewUpdateMsg entityViewUpdateMsg) { + log.trace("[{}] executing processEntityViewMsgFromEdge [{}] from edge [{}]", tenantId, entityViewUpdateMsg, edge.getName()); + EntityViewId entityViewId = new EntityViewId(new UUID(entityViewUpdateMsg.getIdMSB(), entityViewUpdateMsg.getIdLSB())); + try { + edgeSynchronizationManager.getSync().set(true); + + switch (entityViewUpdateMsg.getMsgType()) { + case ENTITY_CREATED_RPC_MESSAGE: + case ENTITY_UPDATED_RPC_MESSAGE: + saveOrUpdateEntityView(tenantId, entityViewId, entityViewUpdateMsg, edge); + return Futures.immediateFuture(null); + case ENTITY_DELETED_RPC_MESSAGE: + EntityView entityViewToDelete = entityViewService.findEntityViewById(tenantId, entityViewId); + if (entityViewToDelete != null) { + entityViewService.unassignEntityViewFromEdge(tenantId, entityViewId, edge.getId()); + } + return Futures.immediateFuture(null); + case UNRECOGNIZED: + default: + return handleUnsupportedMsgType(entityViewUpdateMsg.getMsgType()); + } + } catch (DataValidationException e) { + if (e.getMessage().contains("limit reached")) { + log.warn("[{}] Number of allowed entity views violated {}", tenantId, entityViewUpdateMsg, e); + return Futures.immediateFuture(null); + } else { + return Futures.immediateFailedFuture(e); + } + } finally { + edgeSynchronizationManager.getSync().remove(); + } + } + + private void saveOrUpdateEntityView(TenantId tenantId, EntityViewId entityViewId, EntityViewUpdateMsg entityViewUpdateMsg, Edge edge) { + CustomerId customerId = safeGetCustomerId(entityViewUpdateMsg.getCustomerIdMSB(), entityViewUpdateMsg.getCustomerIdLSB()); + Pair resultPair = super.saveOrUpdateEntityView(tenantId, entityViewId, entityViewUpdateMsg, customerId); + Boolean created = resultPair.getFirst(); + if (created) { + createRelationFromEdge(tenantId, edge.getId(), entityViewId); + pushAssetCreatedEventToRuleEngine(tenantId, edge, entityViewId); + entityViewService.assignEntityViewToEdge(tenantId, entityViewId, edge.getId()); + } + Boolean assetNameUpdated = resultPair.getSecond(); + if (assetNameUpdated) { + saveEdgeEvent(tenantId, edge.getId(), EdgeEventType.ENTITY_VIEW, EdgeEventActionType.UPDATED, entityViewId, null); + } + } + + private void pushAssetCreatedEventToRuleEngine(TenantId tenantId, Edge edge, EntityViewId entityViewId) { + try { + EntityView entityView = entityViewService.findEntityViewById(tenantId, entityViewId); + ObjectNode entityNode = JacksonUtil.OBJECT_MAPPER.valueToTree(entityView); + TbMsg tbMsg = TbMsg.newMsg(TbMsgType.ENTITY_CREATED, entityViewId, entityView.getCustomerId(), + getActionTbMsgMetaData(edge, entityView.getCustomerId()), TbMsgDataType.JSON, JacksonUtil.OBJECT_MAPPER.writeValueAsString(entityNode)); + tbClusterService.pushMsgToRuleEngine(tenantId, entityViewId, tbMsg, new TbQueueCallback() { + @Override + public void onSuccess(TbQueueMsgMetadata metadata) { + log.debug("Successfully send ENTITY_CREATED EVENT to rule engine [{}]", entityView); + } + + @Override + public void onFailure(Throwable t) { + log.warn("Failed to send ENTITY_CREATED EVENT to rule engine [{}]", entityView, t); + } + }); + } catch (JsonProcessingException | IllegalArgumentException e) { + log.warn("[{}] Failed to push entity view action to rule engine: {}", entityViewId, DataConstants.ENTITY_CREATED, e); + } + } public DownlinkMsg convertEntityViewEventToDownlink(EdgeEvent edgeEvent) { EntityViewId entityViewId = new EntityViewId(edgeEvent.getEntityId()); diff --git a/application/src/main/java/org/thingsboard/server/service/entitiy/alarm/DefaultTbAlarmService.java b/application/src/main/java/org/thingsboard/server/service/entitiy/alarm/DefaultTbAlarmService.java index e776c040c0..2dc246d9a7 100644 --- a/application/src/main/java/org/thingsboard/server/service/entitiy/alarm/DefaultTbAlarmService.java +++ b/application/src/main/java/org/thingsboard/server/service/entitiy/alarm/DefaultTbAlarmService.java @@ -34,7 +34,6 @@ import org.thingsboard.server.common.data.alarm.AlarmUpdateRequest; import org.thingsboard.server.common.data.audit.ActionType; import org.thingsboard.server.common.data.exception.ThingsboardErrorCode; import org.thingsboard.server.common.data.exception.ThingsboardException; -import org.thingsboard.server.common.data.id.EdgeId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.id.UserId; import org.thingsboard.server.common.data.page.TimePageLink; diff --git a/application/src/main/java/org/thingsboard/server/service/entitiy/queue/DefaultTbQueueService.java b/application/src/main/java/org/thingsboard/server/service/entitiy/queue/DefaultTbQueueService.java index 40d294d238..b0ef8f4cde 100644 --- a/application/src/main/java/org/thingsboard/server/service/entitiy/queue/DefaultTbQueueService.java +++ b/application/src/main/java/org/thingsboard/server/service/entitiy/queue/DefaultTbQueueService.java @@ -15,7 +15,7 @@ */ package org.thingsboard.server.service.entitiy.queue; -import lombok.AllArgsConstructor; +import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.springframework.stereotype.Service; import org.thingsboard.server.cluster.TbClusterService; @@ -27,7 +27,6 @@ import org.thingsboard.server.common.data.tenant.profile.TenantProfileQueueConfi import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; import org.thingsboard.server.dao.queue.QueueService; import org.thingsboard.server.queue.TbQueueAdmin; -import org.thingsboard.server.queue.scheduler.SchedulerComponent; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.entitiy.AbstractTbEntityService; @@ -35,20 +34,17 @@ import java.util.ArrayList; import java.util.Collections; import java.util.List; import java.util.Map; -import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; @Slf4j @Service @TbCoreComponent -@AllArgsConstructor +@RequiredArgsConstructor public class DefaultTbQueueService extends AbstractTbEntityService implements TbQueueService { - private static final long DELETE_DELAY = 30; private final QueueService queueService; private final TbClusterService tbClusterService; private final TbQueueAdmin tbQueueAdmin; - private final SchedulerComponent scheduler; @Override public Queue saveQueue(Queue queue) { @@ -90,7 +86,9 @@ public class DefaultTbQueueService extends AbstractTbEntityService implements Tb private void onQueueCreated(Queue queue) { for (int i = 0; i < queue.getPartitions(); i++) { tbQueueAdmin.createTopicIfNotExists( - new TopicPartitionInfo(queue.getTopic(), queue.getTenantId(), i, false).getFullTopicName()); + new TopicPartitionInfo(queue.getTopic(), queue.getTenantId(), i, false).getFullTopicName(), + queue.getCustomProperties() + ); } tbClusterService.onQueueChange(queue); @@ -105,21 +103,15 @@ public class DefaultTbQueueService extends AbstractTbEntityService implements Tb log.info("Added [{}] new partitions to [{}] queue", currentPartitions - oldPartitions, queue.getName()); for (int i = oldPartitions; i < currentPartitions; i++) { tbQueueAdmin.createTopicIfNotExists( - new TopicPartitionInfo(queue.getTopic(), queue.getTenantId(), i, false).getFullTopicName()); + new TopicPartitionInfo(queue.getTopic(), queue.getTenantId(), i, false).getFullTopicName(), + queue.getCustomProperties() + ); } tbClusterService.onQueueChange(queue); } else { log.info("Removed [{}] partitions from [{}] queue", oldPartitions - currentPartitions, queue.getName()); tbClusterService.onQueueChange(queue); - - scheduler.schedule(() -> { - for (int i = currentPartitions; i < oldPartitions; i++) { - String fullTopicName = new TopicPartitionInfo(queue.getTopic(), queue.getTenantId(), i, false).getFullTopicName(); - log.info("Removed partition [{}]", fullTopicName); - tbQueueAdmin.deleteTopic( - fullTopicName); - } - }, DELETE_DELAY, TimeUnit.SECONDS); + // TODO: move all the messages left in old partitions and delete topics } } else if (!oldQueue.equals(queue)) { tbClusterService.onQueueChange(queue); @@ -128,20 +120,7 @@ public class DefaultTbQueueService extends AbstractTbEntityService implements Tb private void onQueueDeleted(Queue queue) { tbClusterService.onQueueDelete(queue); - // queueStatsService.deleteQueueStatsByQueueId(tenantId, queueId); - - scheduler.schedule(() -> { - for (int i = 0; i < queue.getPartitions(); i++) { - String fullTopicName = new TopicPartitionInfo(queue.getTopic(), queue.getTenantId(), i, false).getFullTopicName(); - log.info("Deleting queue [{}]", fullTopicName); - try { - tbQueueAdmin.deleteTopic(fullTopicName); - } catch (Exception e) { - log.error("Failed to delete queue [{}]", fullTopicName); - } - } - }, DELETE_DELAY, TimeUnit.SECONDS); } @Override @@ -193,6 +172,10 @@ public class DefaultTbQueueService extends AbstractTbEntityService implements Tb } } + if (log.isDebugEnabled()) { + log.debug("[{}] Handling profile queue config update: creating queues {}, updating {}, deleting {}. Affected tenants: {}", + newTenantProfile.getUuidId(), toCreate, toUpdate, toRemove, tenantIds); + } tenantIds.forEach(tenantId -> { toCreate.forEach(key -> saveQueue(new Queue(tenantId, newQueues.get(key)))); diff --git a/application/src/main/java/org/thingsboard/server/service/entitiy/tenant/profile/DefaultTbTenantProfileService.java b/application/src/main/java/org/thingsboard/server/service/entitiy/tenant/profile/DefaultTbTenantProfileService.java index 1219a5b929..15bcc2e040 100644 --- a/application/src/main/java/org/thingsboard/server/service/entitiy/tenant/profile/DefaultTbTenantProfileService.java +++ b/application/src/main/java/org/thingsboard/server/service/entitiy/tenant/profile/DefaultTbTenantProfileService.java @@ -44,16 +44,14 @@ public class DefaultTbTenantProfileService extends AbstractTbEntityService imple @Override public TenantProfile save(TenantId tenantId, TenantProfile tenantProfile, TenantProfile oldTenantProfile) throws ThingsboardException { TenantProfile savedTenantProfile = checkNotNull(tenantProfileService.saveTenantProfile(tenantId, tenantProfile)); - if (oldTenantProfile != null && savedTenantProfile.isIsolatedTbRuleEngine()) { - List tenantIds = tenantService.findTenantIdsByTenantProfileId(savedTenantProfile.getId()); - tbQueueService.updateQueuesByTenants(tenantIds, savedTenantProfile, oldTenantProfile); - } - tenantProfileCache.put(savedTenantProfile); tbClusterService.onTenantProfileChange(savedTenantProfile, null); tbClusterService.broadcastEntityStateChangeEvent(TenantId.SYS_TENANT_ID, savedTenantProfile.getId(), tenantProfile.getId() == null ? ComponentLifecycleEvent.CREATED : ComponentLifecycleEvent.UPDATED); + List tenantIds = tenantService.findTenantIdsByTenantProfileId(savedTenantProfile.getId()); + tbQueueService.updateQueuesByTenants(tenantIds, savedTenantProfile, oldTenantProfile); + return savedTenantProfile; } diff --git a/application/src/main/java/org/thingsboard/server/service/notification/DefaultNotificationCenter.java b/application/src/main/java/org/thingsboard/server/service/notification/DefaultNotificationCenter.java index cb77cc0948..042d5cbf00 100644 --- a/application/src/main/java/org/thingsboard/server/service/notification/DefaultNotificationCenter.java +++ b/application/src/main/java/org/thingsboard/server/service/notification/DefaultNotificationCenter.java @@ -39,6 +39,7 @@ import org.thingsboard.server.common.data.notification.NotificationRequestStatus import org.thingsboard.server.common.data.notification.NotificationStatus; import org.thingsboard.server.common.data.notification.info.RuleOriginatedNotificationInfo; import org.thingsboard.server.common.data.notification.settings.NotificationSettings; +import org.thingsboard.server.common.data.notification.settings.UserNotificationSettings; import org.thingsboard.server.common.data.notification.targets.NotificationRecipient; import org.thingsboard.server.common.data.notification.targets.NotificationTarget; import org.thingsboard.server.common.data.notification.targets.platform.PlatformUsersNotificationTargetConfig; @@ -237,7 +238,17 @@ public class DefaultNotificationCenter extends AbstractSubscriptionService imple private void processForRecipient(NotificationDeliveryMethod deliveryMethod, NotificationRecipient recipient, NotificationProcessingContext ctx) throws Exception { if (ctx.getStats().contains(deliveryMethod, recipient.getId())) { throw new AlreadySentException(); + } else { + ctx.getStats().reportProcessed(deliveryMethod, recipient.getId()); } + + if (recipient instanceof User) { + UserNotificationSettings settings = notificationSettingsService.getUserNotificationSettings(ctx.getTenantId(), ((User) recipient).getId(), false); + if (!settings.isEnabled(ctx.getNotificationType(), deliveryMethod)) { + throw new RuntimeException("User disabled " + deliveryMethod.getName() + " notifications of this type"); + } + } + NotificationChannel notificationChannel = channels.get(deliveryMethod); DeliveryMethodNotificationTemplate processedTemplate = ctx.getProcessedTemplate(deliveryMethod, recipient); @@ -251,7 +262,7 @@ public class DefaultNotificationCenter extends AbstractSubscriptionService imple Notification notification = Notification.builder() .requestId(request.getId()) .recipientId(recipient.getId()) - .type(ctx.getNotificationTemplate().getNotificationType()) + .type(ctx.getNotificationType()) .subject(processedTemplate.getSubject()) .text(processedTemplate.getBody()) .additionalConfig(processedTemplate.getAdditionalConfig()) diff --git a/application/src/main/java/org/thingsboard/server/service/notification/NotificationProcessingContext.java b/application/src/main/java/org/thingsboard/server/service/notification/NotificationProcessingContext.java index 27a9cabe43..c4d8895266 100644 --- a/application/src/main/java/org/thingsboard/server/service/notification/NotificationProcessingContext.java +++ b/application/src/main/java/org/thingsboard/server/service/notification/NotificationProcessingContext.java @@ -23,6 +23,7 @@ import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.notification.NotificationDeliveryMethod; import org.thingsboard.server.common.data.notification.NotificationRequest; import org.thingsboard.server.common.data.notification.NotificationRequestStats; +import org.thingsboard.server.common.data.notification.NotificationType; import org.thingsboard.server.common.data.notification.settings.NotificationDeliveryMethodConfig; import org.thingsboard.server.common.data.notification.settings.NotificationSettings; import org.thingsboard.server.common.data.notification.targets.NotificationRecipient; @@ -52,6 +53,8 @@ public class NotificationProcessingContext { private final Set deliveryMethods; @Getter private final NotificationTemplate notificationTemplate; + @Getter + private final NotificationType notificationType; private final Map templates; @Getter @@ -65,6 +68,7 @@ public class NotificationProcessingContext { this.deliveryMethods = deliveryMethods; this.settings = settings; this.notificationTemplate = template; + this.notificationType = template.getNotificationType(); this.templates = new EnumMap<>(NotificationDeliveryMethod.class); this.stats = new NotificationRequestStats(); init(); diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbRuleEngineConsumerService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbRuleEngineConsumerService.java index ac7d796e33..670dbcaa6d 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbRuleEngineConsumerService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbRuleEngineConsumerService.java @@ -23,11 +23,14 @@ import org.springframework.scheduling.annotation.Scheduled; import org.springframework.stereotype.Service; import org.thingsboard.common.util.ThingsBoardThreadFactory; import org.thingsboard.server.actors.ActorSystemContext; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.EntityIdFactory; import org.thingsboard.server.common.data.id.QueueId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.queue.Queue; import org.thingsboard.server.common.data.rpc.RpcError; import org.thingsboard.server.common.msg.TbMsg; +import org.thingsboard.server.common.msg.gen.MsgProtos; import org.thingsboard.server.common.msg.queue.QueueToRuleEngineMsg; import org.thingsboard.server.common.msg.queue.RuleEngineException; import org.thingsboard.server.common.msg.queue.RuleNodeInfo; @@ -42,12 +45,14 @@ import org.thingsboard.server.dao.tenant.TbTenantProfileCache; import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineNotificationMsg; +import org.thingsboard.server.queue.TbQueueAdmin; import org.thingsboard.server.queue.TbQueueConsumer; import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.queue.discovery.PartitionService; import org.thingsboard.server.queue.discovery.QueueKey; import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; import org.thingsboard.server.queue.discovery.event.PartitionChangeEvent; +import org.thingsboard.server.queue.provider.TbQueueProducerProvider; import org.thingsboard.server.queue.provider.TbRuleEngineQueueFactory; import org.thingsboard.server.queue.util.DataDecodingEncodingService; import org.thingsboard.server.queue.util.TbRuleEngineComponent; @@ -98,6 +103,8 @@ public class DefaultTbRuleEngineConsumerService extends AbstractConsumerService< private boolean statsEnabled; @Value("${queue.rule-engine.prometheus-stats.enabled:false}") boolean prometheusStatsEnabled; + @Value("${queue.rule-engine.topic-deletion-delay:30}") + private int topicDeletionDelayInSec; private final StatsFactory statsFactory; private final TbRuleEngineSubmitStrategyFactory submitStrategyFactory; @@ -107,7 +114,8 @@ public class DefaultTbRuleEngineConsumerService extends AbstractConsumerService< private final TbRuleEngineDeviceRpcService tbDeviceRpcService; private final TbServiceInfoProvider serviceInfoProvider; private final QueueService queueService; - // private final TenantId tenantId; + private final TbQueueProducerProvider producerProvider; + private final TbQueueAdmin queueAdmin; private final ConcurrentMap>> consumers = new ConcurrentHashMap<>(); private final ConcurrentMap consumerConfigurations = new ConcurrentHashMap<>(); private final ConcurrentMap consumerStats = new ConcurrentHashMap<>(); @@ -128,7 +136,8 @@ public class DefaultTbRuleEngineConsumerService extends AbstractConsumerService< TbTenantProfileCache tenantProfileCache, TbApiUsageStateService apiUsageStateService, PartitionService partitionService, ApplicationEventPublisher eventPublisher, - TbServiceInfoProvider serviceInfoProvider, QueueService queueService) { + TbServiceInfoProvider serviceInfoProvider, QueueService queueService, + TbQueueProducerProvider producerProvider, TbQueueAdmin queueAdmin) { super(actorContext, encodingService, tenantProfileCache, deviceProfileCache, assetProfileCache, apiUsageStateService, partitionService, eventPublisher, tbRuleEngineQueueFactory.createToRuleEngineNotificationsMsgConsumer(), Optional.empty()); this.statisticsService = statisticsService; this.tbRuleEngineQueueFactory = tbRuleEngineQueueFactory; @@ -138,6 +147,8 @@ public class DefaultTbRuleEngineConsumerService extends AbstractConsumerService< this.statsFactory = statsFactory; this.serviceInfoProvider = serviceInfoProvider; this.queueService = queueService; + this.producerProvider = producerProvider; + this.queueAdmin = queueAdmin; } @PostConstruct @@ -145,14 +156,16 @@ public class DefaultTbRuleEngineConsumerService extends AbstractConsumerService< super.init("tb-rule-engine-consumer", "tb-rule-engine-notifications-consumer"); List queues = queueService.findAllQueues(); for (Queue configuration : queues) { - initConsumer(configuration); + if (partitionService.isManagedByCurrentService(configuration.getTenantId())) { + initConsumer(configuration); + } } } private void initConsumer(Queue configuration) { QueueKey queueKey = new QueueKey(ServiceType.TB_RULE_ENGINE, configuration); consumerConfigurations.putIfAbsent(queueKey, configuration); - consumerStats.putIfAbsent(queueKey, new TbRuleEngineConsumerStats(configuration.getName(), statsFactory)); + consumerStats.putIfAbsent(queueKey, new TbRuleEngineConsumerStats(configuration, statsFactory)); if (!configuration.isConsumerPerPartition()) { consumers.computeIfAbsent(queueKey, queueName -> tbRuleEngineQueueFactory.createToRuleEngineMsgConsumer(configuration)); } else { @@ -172,7 +185,12 @@ public class DefaultTbRuleEngineConsumerService extends AbstractConsumerService< if (event.getServiceType().equals(getServiceType())) { String serviceQueue = event.getQueueKey().getQueueName(); log.info("[{}] Subscribing to partitions: {}", serviceQueue, event.getPartitions()); - if (!consumerConfigurations.get(event.getQueueKey()).isConsumerPerPartition()) { + Queue configuration = consumerConfigurations.get(event.getQueueKey()); + if (configuration == null) { + log.warn("Received invalid partition change event for {} that is not managed by this service", event.getQueueKey()); + return; + } + if (!configuration.isConsumerPerPartition()) { consumers.get(event.getQueueKey()).subscribe(event.getPartitions()); } else { log.info("[{}] Subscribing consumer per partition: {}", serviceQueue, event.getPartitions()); @@ -230,7 +248,6 @@ public class DefaultTbRuleEngineConsumerService extends AbstractConsumerService< launchConsumer(consumer, consumerConfigurations.get(queueKey), consumerStats.get(queueKey), "" + queueKey + "-" + tpi.getPartition().orElse(-999999)); consumer.subscribe(Collections.singleton(tpi)); }); - } finally { tbTopicWithConsumerPerPartition.getLock().unlock(); } @@ -278,9 +295,9 @@ public class DefaultTbRuleEngineConsumerService extends AbstractConsumerService< void consumerLoop(TbQueueConsumer> consumer, org.thingsboard.server.common.data.queue.Queue configuration, TbRuleEngineConsumerStats stats, String threadSuffix) { updateCurrentThreadName(threadSuffix); - while (!stopped && !consumer.isStopped()) { + while (!stopped && !consumer.isStopped() && !consumer.isQueueDeleted()) { try { - List> msgs = consumer.poll(pollDuration); + List> msgs = consumer.poll(configuration.getPollInterval()); if (msgs.isEmpty()) { continue; } @@ -328,6 +345,10 @@ public class DefaultTbRuleEngineConsumerService extends AbstractConsumerService< } } } + + if (consumer.isQueueDeleted()) { + processQueueDeletion(configuration, consumer); + } log.info("TB Rule Engine Consumer stopped."); } @@ -425,32 +446,34 @@ public class DefaultTbRuleEngineConsumerService extends AbstractConsumerService< private void updateQueue(TransportProtos.QueueUpdateMsg queueUpdateMsg) { log.info("Received queue update msg: [{}]", queueUpdateMsg); - String queueName = queueUpdateMsg.getQueueName(); TenantId tenantId = new TenantId(new UUID(queueUpdateMsg.getTenantIdMSB(), queueUpdateMsg.getTenantIdLSB())); - QueueId queueId = new QueueId(new UUID(queueUpdateMsg.getQueueIdMSB(), queueUpdateMsg.getQueueIdLSB())); - QueueKey queueKey = new QueueKey(ServiceType.TB_RULE_ENGINE, queueUpdateMsg.getQueueName(), tenantId); - Queue queue = queueService.findQueueById(tenantId, queueId); - Queue oldQueue = consumerConfigurations.remove(queueKey); - if (oldQueue != null) { - if (oldQueue.isConsumerPerPartition()) { - TbTopicWithConsumerPerPartition consumerPerPartition = topicsConsumerPerPartition.remove(queueKey); - ReentrantLock lock = consumerPerPartition.getLock(); - try { - lock.lock(); - consumerPerPartition.getConsumers().values().forEach(TbQueueConsumer::unsubscribe); - } finally { - lock.unlock(); + if (partitionService.isManagedByCurrentService(tenantId)) { + QueueId queueId = new QueueId(new UUID(queueUpdateMsg.getQueueIdMSB(), queueUpdateMsg.getQueueIdLSB())); + String queueName = queueUpdateMsg.getQueueName(); + QueueKey queueKey = new QueueKey(ServiceType.TB_RULE_ENGINE, queueName, tenantId); + Queue queue = queueService.findQueueById(tenantId, queueId); + Queue oldQueue = consumerConfigurations.remove(queueKey); + if (oldQueue != null) { + if (oldQueue.isConsumerPerPartition()) { + TbTopicWithConsumerPerPartition consumerPerPartition = topicsConsumerPerPartition.remove(queueKey); + ReentrantLock lock = consumerPerPartition.getLock(); + try { + lock.lock(); + consumerPerPartition.getConsumers().values().forEach(TbQueueConsumer::unsubscribe); + } finally { + lock.unlock(); + } + } else { + TbQueueConsumer> consumer = consumers.remove(queueKey); + consumer.unsubscribe(); } - } else { - TbQueueConsumer> consumer = consumers.remove(queueKey); - consumer.unsubscribe(); } - } - initConsumer(queue); + initConsumer(queue); - if (!queue.isConsumerPerPartition()) { - launchConsumer(consumers.get(queueKey), consumerConfigurations.get(queueKey), consumerStats.get(queueKey), queueName); + if (!queue.isConsumerPerPartition()) { + launchConsumer(consumers.get(queueKey), consumerConfigurations.get(queueKey), consumerStats.get(queueKey), queueName); + } } partitionService.updateQueue(queueUpdateMsg); @@ -462,22 +485,22 @@ public class DefaultTbRuleEngineConsumerService extends AbstractConsumerService< TenantId tenantId = new TenantId(new UUID(queueDeleteMsg.getTenantIdMSB(), queueDeleteMsg.getTenantIdLSB())); QueueKey queueKey = new QueueKey(ServiceType.TB_RULE_ENGINE, queueDeleteMsg.getQueueName(), tenantId); + partitionService.removeQueue(queueDeleteMsg); Queue queue = consumerConfigurations.remove(queueKey); if (queue != null) { if (queue.isConsumerPerPartition()) { TbTopicWithConsumerPerPartition tbTopicWithConsumerPerPartition = topicsConsumerPerPartition.remove(queueKey); if (tbTopicWithConsumerPerPartition != null) { - tbTopicWithConsumerPerPartition.getConsumers().values().forEach(TbQueueConsumer::unsubscribe); + tbTopicWithConsumerPerPartition.getConsumers().values().forEach(TbQueueConsumer::onQueueDelete); tbTopicWithConsumerPerPartition.getConsumers().clear(); } } else { TbQueueConsumer> consumer = consumers.remove(queueKey); if (consumer != null) { - consumer.unsubscribe(); + consumer.onQueueDelete(); } } } - partitionService.removeQueue(queueDeleteMsg); } private void forwardToRuleEngineActor(String queueName, TenantId tenantId, ToRuleEngineMsg toRuleEngineMsg, TbMsgCallback callback) { @@ -496,6 +519,47 @@ public class DefaultTbRuleEngineConsumerService extends AbstractConsumerService< actorContext.tell(msg); } + private void processQueueDeletion(Queue queue, TbQueueConsumer> consumer) { + long finishTs = System.currentTimeMillis() + TimeUnit.SECONDS.toMillis(topicDeletionDelayInSec); + try { + int n = 0; + while (System.currentTimeMillis() <= finishTs) { + List> msgs = consumer.poll(queue.getPollInterval()); + if (msgs.isEmpty()) { + continue; + } + for (TbProtoQueueMsg msg : msgs) { + try { + MsgProtos.TbMsgProto tbMsgProto = MsgProtos.TbMsgProto.parseFrom(msg.getValue().getTbMsg().toByteArray()); + EntityId originator = EntityIdFactory.getByTypeAndUuid(tbMsgProto.getEntityType(), new UUID(tbMsgProto.getEntityIdMSB(), tbMsgProto.getEntityIdLSB())); + + TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_RULE_ENGINE, queue.getName(), TenantId.SYS_TENANT_ID, originator); + producerProvider.getRuleEngineMsgProducer().send(tpi, msg, null); + n++; + } catch (Throwable e) { + log.debug("Failed to move message to system {}: {}", consumer.getTopic(), msg, e); + } + } + consumer.commit(); + } + if (n > 0) { + log.info("Moved {} messages from {} to system {}", n, consumer.getFullTopicNames(), consumer.getTopic()); + } + + consumer.unsubscribe(); + for (String topic : consumer.getFullTopicNames()) { + try { + queueAdmin.deleteTopic(topic); + log.info("Deleted topic {}", topic); + } catch (Exception e) { + log.error("Failed to delete topic {} after unsubscribing", topic, e); + } + } + } catch (Exception e) { + log.error("Failed to process deletion of {} ({})", consumer.getTopic(), queue.getTenantId(), e); + } + } + @Scheduled(fixedDelayString = "${queue.rule-engine.stats.print-interval-ms}") public void printStats() { if (statsEnabled) { diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTenantRoutingInfoService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTenantRoutingInfoService.java index 400586235e..ea90364ef7 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTenantRoutingInfoService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTenantRoutingInfoService.java @@ -22,7 +22,6 @@ import org.thingsboard.server.common.data.TenantProfile; import org.thingsboard.server.common.data.exception.TenantNotFoundException; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.dao.tenant.TbTenantProfileCache; -import org.thingsboard.server.dao.tenant.TenantService; import org.thingsboard.server.queue.discovery.TenantRoutingInfo; import org.thingsboard.server.queue.discovery.TenantRoutingInfoService; @@ -31,12 +30,9 @@ import org.thingsboard.server.queue.discovery.TenantRoutingInfoService; @ConditionalOnExpression("'${service.type:null}'=='monolith' || '${service.type:null}'=='tb-core' || '${service.type:null}'=='tb-rule-engine'") public class DefaultTenantRoutingInfoService implements TenantRoutingInfoService { - private final TenantService tenantService; - private final TbTenantProfileCache tenantProfileCache; - public DefaultTenantRoutingInfoService(TenantService tenantService, TbTenantProfileCache tenantProfileCache) { - this.tenantService = tenantService; + public DefaultTenantRoutingInfoService(TbTenantProfileCache tenantProfileCache) { this.tenantProfileCache = tenantProfileCache; } @@ -44,7 +40,7 @@ public class DefaultTenantRoutingInfoService implements TenantRoutingInfoService public TenantRoutingInfo getRoutingInfo(TenantId tenantId) { TenantProfile tenantProfile = tenantProfileCache.get(tenantId); if (tenantProfile != null) { - return new TenantRoutingInfo(tenantId, tenantProfile.isIsolatedTbRuleEngine()); + return new TenantRoutingInfo(tenantId, tenantProfile.getId(), tenantProfile.isIsolatedTbRuleEngine()); } else { throw new TenantNotFoundException(tenantId); } diff --git a/application/src/main/java/org/thingsboard/server/service/queue/TbRuleEngineConsumerStats.java b/application/src/main/java/org/thingsboard/server/service/queue/TbRuleEngineConsumerStats.java index 77e805eb62..2904c299ce 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/TbRuleEngineConsumerStats.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/TbRuleEngineConsumerStats.java @@ -18,6 +18,7 @@ package org.thingsboard.server.service.queue; import io.micrometer.core.instrument.Timer; import lombok.extern.slf4j.Slf4j; import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.queue.Queue; import org.thingsboard.server.common.msg.queue.RuleEngineException; import org.thingsboard.server.common.stats.StatsCounter; import org.thingsboard.server.common.stats.StatsFactory; @@ -63,9 +64,11 @@ public class TbRuleEngineConsumerStats { private final ConcurrentMap tenantExceptions = new ConcurrentHashMap<>(); private final String queueName; + private final TenantId tenantId; - public TbRuleEngineConsumerStats(String queueName, StatsFactory statsFactory) { - this.queueName = queueName; + public TbRuleEngineConsumerStats(Queue queue, StatsFactory statsFactory) { + this.queueName = queue.getName(); + this.tenantId = queue.getTenantId(); this.statsFactory = statsFactory; String statsKey = StatsType.RULE_ENGINE.getName() + "." + queueName; @@ -156,7 +159,11 @@ public class TbRuleEngineConsumerStats { counters.forEach(counter -> { stats.append(counter.getName()).append(" = [").append(counter.get()).append("] "); }); - log.info("[{}] Stats: {}", queueName, stats); + if (tenantId.isSysTenantId()) { + log.info("[{}] Stats: {}", queueName, stats); + } else { + log.info("[{}][{}] Stats: {}", queueName, tenantId, stats); + } } } diff --git a/application/src/main/resources/thingsboard.yml b/application/src/main/resources/thingsboard.yml index e874e41cd7..d5aa55f1be 100644 --- a/application/src/main/resources/thingsboard.yml +++ b/application/src/main/resources/thingsboard.yml @@ -1091,6 +1091,7 @@ queue: fetch_max_bytes: "${TB_QUEUE_KAFKA_FETCH_MAX_BYTES:134217728}" request.timeout.ms: "${TB_QUEUE_KAFKA_REQUEST_TIMEOUT_MS:30000}" # (30 seconds) # refer to https://docs.confluent.io/platform/current/installation/configuration/producer-configs.html#producerconfigs_request.timeout.ms session.timeout.ms: "${TB_QUEUE_KAFKA_SESSION_TIMEOUT_MS:10000}" # (10 seconds) # refer to https://docs.confluent.io/platform/current/installation/configuration/consumer-configs.html#consumerconfigs_session.timeout.ms + auto_offset_reset: "${TB_QUEUE_KAFKA_AUTO_OFFSET_RESET:earliest}" # earliest, latest or none use_confluent_cloud: "${TB_QUEUE_KAFKA_USE_CONFLUENT_CLOUD:false}" confluent: ssl.algorithm: "${TB_QUEUE_KAFKA_CONFLUENT_SSL_ALGORITHM:https}" @@ -1287,6 +1288,8 @@ queue: failure-percentage: "${TB_QUEUE_RE_SQ_PROCESSING_STRATEGY_FAILURE_PERCENTAGE:0}" # Skip retry if failures or timeouts are less then X percentage of messages; pause-between-retries: "${TB_QUEUE_RE_SQ_PROCESSING_STRATEGY_RETRY_PAUSE:5}" # Time in seconds to wait in consumer thread before retries; max-pause-between-retries: "${TB_QUEUE_RE_SQ_PROCESSING_STRATEGY_MAX_RETRY_PAUSE:5}" # Max allowed time in seconds for pause between retries. + # After a queue is deleted (or profile's isolation option was disabled), Rule Engine will continue reading related topics during this period, before deleting the actual topics + topic-deletion-delay: "${TB_QUEUE_RULE_ENGINE_TOPIC_DELETION_DELAY_SEC:30}" transport: # For high priority notifications that require minimum latency and processing time notifications_topic: "${TB_QUEUE_TRANSPORT_NOTIFICATIONS_TOPIC:tb_transport.notifications}" @@ -1300,6 +1303,10 @@ service: type: "${TB_SERVICE_TYPE:monolith}" # monolith or tb-core or tb-rule-engine # Unique id for this service (autogenerated if empty) id: "${TB_SERVICE_ID:}" + rule_engine: + # Comma-separated list of tenant profiles ids assigned to this Rule Engine. + # This Rule Engine will only be responsible for tenants with these profiles (in case 'isolation' option is enabled in profile). + assigned_tenant_profiles: "${TB_RULE_ENGINE_ASSIGNED_TENANT_PROFILES:}" metrics: # Enable/disable actuator metrics. diff --git a/application/src/test/java/org/thingsboard/server/controller/AbstractWebTest.java b/application/src/test/java/org/thingsboard/server/controller/AbstractWebTest.java index c32b4cf8d7..888c9de367 100644 --- a/application/src/test/java/org/thingsboard/server/controller/AbstractWebTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/AbstractWebTest.java @@ -1029,13 +1029,20 @@ public abstract class AbstractWebTest extends AbstractInMemoryStorageTest { return (DeviceActorMessageProcessor) ReflectionTestUtils.getField(actor, "processor"); } - protected void updateDefaultTenantProfile(Consumer updater) throws ThingsboardException { - TenantProfile tenantProfile = tenantProfileService.findDefaultTenantProfile(TenantId.SYS_TENANT_ID); - TenantProfileData profileData = tenantProfile.getProfileData(); - DefaultTenantProfileConfiguration profileConfiguration = (DefaultTenantProfileConfiguration) profileData.getConfiguration(); - updater.accept(profileConfiguration); - tenantProfile.setProfileData(profileData); - tbTenantProfileService.save(TenantId.SYS_TENANT_ID, tenantProfile, null); + protected void updateDefaultTenantProfileConfig(Consumer updater) throws ThingsboardException { + updateDefaultTenantProfile(tenantProfile -> { + TenantProfileData profileData = tenantProfile.getProfileData(); + DefaultTenantProfileConfiguration profileConfiguration = (DefaultTenantProfileConfiguration) profileData.getConfiguration(); + updater.accept(profileConfiguration); + tenantProfile.setProfileData(profileData); + }); + } + + protected void updateDefaultTenantProfile(Consumer updater) throws ThingsboardException { + TenantProfile oldTenantProfile = tenantProfileService.findDefaultTenantProfile(TenantId.SYS_TENANT_ID); + TenantProfile tenantProfile = JacksonUtil.clone(oldTenantProfile); + updater.accept(tenantProfile); + tbTenantProfileService.save(TenantId.SYS_TENANT_ID, tenantProfile, oldTenantProfile); } } diff --git a/application/src/test/java/org/thingsboard/server/controller/TenantControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/TenantControllerTest.java index a5b86fff85..4b594d55f9 100644 --- a/application/src/test/java/org/thingsboard/server/controller/TenantControllerTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/TenantControllerTest.java @@ -27,15 +27,20 @@ import org.junit.Before; import org.junit.Test; import org.mockito.ArgumentMatcher; import org.mockito.Mockito; +import org.springframework.boot.test.mock.mockito.SpyBean; import org.springframework.test.context.TestPropertySource; import org.springframework.test.web.servlet.ResultActions; import org.thingsboard.common.util.ThingsBoardExecutors; +import org.thingsboard.server.actors.ActorSystemContext; import org.thingsboard.server.common.data.DataConstants; +import org.thingsboard.server.common.data.Device; +import org.thingsboard.server.common.data.DeviceProfile; import org.thingsboard.server.common.data.StringUtils; import org.thingsboard.server.common.data.Tenant; import org.thingsboard.server.common.data.TenantInfo; import org.thingsboard.server.common.data.TenantProfile; import org.thingsboard.server.common.data.User; +import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; @@ -50,24 +55,47 @@ import org.thingsboard.server.common.data.tenant.profile.DefaultTenantProfileCon import org.thingsboard.server.common.data.tenant.profile.TenantProfileData; import org.thingsboard.server.common.data.tenant.profile.TenantProfileQueueConfiguration; import org.thingsboard.server.dao.service.DaoSqlTest; +import org.thingsboard.server.common.msg.TbMsg; +import org.thingsboard.server.common.msg.TbMsgMetaData; +import org.thingsboard.server.common.msg.queue.QueueToRuleEngineMsg; +import org.thingsboard.server.common.msg.queue.ServiceType; +import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; +import org.thingsboard.server.gen.transport.TransportProtos; +import org.thingsboard.server.queue.TbQueueAdmin; +import org.thingsboard.server.queue.discovery.PartitionService; import java.util.ArrayList; +import java.util.Collections; import java.util.Comparator; +import java.util.Deque; import java.util.HashMap; +import java.util.LinkedList; import java.util.List; import java.util.Map; import java.util.Random; +import java.util.UUID; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicReference; +import java.util.function.Consumer; +import java.util.function.Predicate; import java.util.stream.Collectors; import static org.assertj.core.api.Assertions.assertThat; +import static org.awaitility.Awaitility.await; import static org.hamcrest.Matchers.containsString; +import static org.mockito.ArgumentMatchers.argThat; +import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.never; +import static org.mockito.Mockito.timeout; import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; +import static org.thingsboard.server.common.data.DataConstants.MAIN_QUEUE_NAME; +import static org.thingsboard.server.common.data.DataConstants.MAIN_QUEUE_TOPIC; @TestPropertySource(properties = { "js.evaluator=mock", + "queue.rule-engine.topic-deletion-delay=10" }) @Slf4j @DaoSqlTest @@ -80,6 +108,13 @@ public class TenantControllerTest extends AbstractControllerTest { ListeningExecutorService executor; + @SpyBean + private PartitionService partitionService; + @SpyBean + private ActorSystemContext actorContext; + @SpyBean + private TbQueueAdmin queueAdmin; + @Before public void setUp() throws Exception { executor = MoreExecutors.listeningDecorator(ThingsBoardExecutors.newWorkStealingPool(8, getClass())); @@ -87,6 +122,12 @@ public class TenantControllerTest extends AbstractControllerTest { @After public void tearDown() throws Exception { + loginSysAdmin(); + for (Queue queue : doGetTypedWithPageLink("/api/queues?serviceType=TB_RULE_ENGINE&", new TypeReference>() {}, new PageLink(100)).getData()) { + if (!queue.getName().equals(MAIN_QUEUE_NAME)) { + doDelete("/api/queues/" + queue.getId()).andExpect(status().isOk()); + } + } executor.shutdownNow(); } @@ -430,7 +471,7 @@ public class TenantControllerTest extends AbstractControllerTest { tenantProfileData.setConfiguration(new DefaultTenantProfileConfiguration()); tenantProfile.setProfileData(tenantProfileData); tenantProfile.setIsolatedTbRuleEngine(true); - addQueueConfig(tenantProfile, DataConstants.MAIN_QUEUE_NAME); + addQueueConfig(tenantProfile, MAIN_QUEUE_NAME); addQueueConfig(tenantProfile, "Test"); tenantProfile = doPost("/api/tenantProfile", tenantProfile, TenantProfile.class); @@ -459,7 +500,7 @@ public class TenantControllerTest extends AbstractControllerTest { tenantProfileData2.setConfiguration(new DefaultTenantProfileConfiguration()); tenantProfile2.setProfileData(tenantProfileData2); tenantProfile2.setIsolatedTbRuleEngine(true); - addQueueConfig(tenantProfile2, DataConstants.MAIN_QUEUE_NAME); + addQueueConfig(tenantProfile2, MAIN_QUEUE_NAME); addQueueConfig(tenantProfile2, "Test"); addQueueConfig(tenantProfile2, "Test2"); tenantProfile2 = doPost("/api/tenantProfile", tenantProfile2, TenantProfile.class); @@ -520,10 +561,201 @@ public class TenantControllerTest extends AbstractControllerTest { doDelete("/api/tenant/" + tenant.getId().getId().toString()).andExpect(status().isOk()); } + @Test + public void testUpdateTenantProfileToIsolated() throws Exception { + loginSysAdmin(); + doPost("/api/queues?serviceType=TB_RULE_ENGINE", new Queue(TenantId.SYS_TENANT_ID, getQueueConfig(DataConstants.HP_QUEUE_NAME, DataConstants.HP_QUEUE_TOPIC))).andExpect(status().isOk()); + TenantProfile tenantProfile = new TenantProfile(); + tenantProfile.setName("Test profile"); + TenantProfileData tenantProfileData = new TenantProfileData(); + tenantProfileData.setConfiguration(new DefaultTenantProfileConfiguration()); + tenantProfile.setProfileData(tenantProfileData); + tenantProfile.setIsolatedTbRuleEngine(false); + tenantProfile = doPost("/api/tenantProfile", tenantProfile, TenantProfile.class); + createDifferentTenant(); + loginSysAdmin(); + savedDifferentTenant.setTenantProfileId(tenantProfile.getId()); + savedDifferentTenant = doPost("/api/tenant", savedDifferentTenant, Tenant.class); + TenantId tenantId = differentTenantId; + + loginDifferentTenant(); + DeviceProfile hpQueueProfile = createDeviceProfile("HighPriority profile"); + hpQueueProfile.setDefaultQueueName(DataConstants.HP_QUEUE_NAME); + hpQueueProfile = doPost("/api/deviceProfile", hpQueueProfile, DeviceProfile.class); + Device hpQueueDevice = createDevice("HP", hpQueueProfile.getName(), "HP"); + + DeviceProfile mainQueueProfile = createDeviceProfile("Main profile"); + mainQueueProfile.setDefaultQueueName(MAIN_QUEUE_NAME); + mainQueueProfile = doPost("/api/deviceProfile", mainQueueProfile, DeviceProfile.class); + Device mainQueueDevice = createDevice("Main", mainQueueProfile.getName(), "Main"); + + verifyUsedQueueAndMessage(DataConstants.HP_QUEUE_NAME, tenantId, hpQueueDevice.getId(), DataConstants.ATTRIBUTES_UPDATED, () -> { + doPost("/api/plugins/telemetry/DEVICE/" + hpQueueDevice.getId() + "/attributes/SERVER_SCOPE", "{\"test\":123}", String.class); + }, usedTpi -> { + assertThat(usedTpi.getTopic()).isEqualTo(DataConstants.HP_QUEUE_TOPIC); + assertThat(usedTpi.getTenantId()).get().isEqualTo(TenantId.SYS_TENANT_ID); + }); + verifyUsedQueueAndMessage(MAIN_QUEUE_NAME, tenantId, mainQueueDevice.getId(), DataConstants.ATTRIBUTES_UPDATED, () -> { + doPost("/api/plugins/telemetry/DEVICE/" + mainQueueDevice.getId() + "/attributes/SERVER_SCOPE", "{\"test\":123}", String.class); + }, usedTpi -> { + assertThat(usedTpi.getTopic()).isEqualTo(MAIN_QUEUE_TOPIC); + assertThat(usedTpi.getTenantId()).get().isEqualTo(TenantId.SYS_TENANT_ID); + }); + + loginSysAdmin(); + tenantProfile.setIsolatedTbRuleEngine(true); + tenantProfile.getProfileData().setQueueConfiguration(List.of( + getQueueConfig(MAIN_QUEUE_NAME, MAIN_QUEUE_TOPIC) + )); + tenantProfile = doPost("/api/tenantProfile", tenantProfile, TenantProfile.class); + + loginDifferentTenant(); + verifyUsedQueueAndMessage(MAIN_QUEUE_NAME, tenantId, mainQueueDevice.getId(), DataConstants.ATTRIBUTES_UPDATED, () -> { + doPost("/api/plugins/telemetry/DEVICE/" + mainQueueDevice.getId() + "/attributes/SERVER_SCOPE", "{\"test\":123}", String.class); + }, usedTpi -> { + assertThat(usedTpi.getTopic()).isEqualTo(MAIN_QUEUE_TOPIC); + assertThat(usedTpi.getTenantId()).get().isEqualTo(tenantId); + }); + verifyUsedQueueAndMessage(DataConstants.HP_QUEUE_NAME, tenantId, hpQueueDevice.getId(), DataConstants.ATTRIBUTES_UPDATED, () -> { + doPost("/api/plugins/telemetry/DEVICE/" + hpQueueDevice.getId() + "/attributes/SERVER_SCOPE", "{\"test\":123}", String.class); + }, usedTpi -> { + assertThat(usedTpi.getTopic()).isEqualTo(DataConstants.HP_QUEUE_TOPIC); + assertThat(usedTpi.getTenantId()).get().isEqualTo(TenantId.SYS_TENANT_ID); + }); + + loginSysAdmin(); + tenantProfile.setIsolatedTbRuleEngine(true); + tenantProfile.getProfileData().setQueueConfiguration(List.of( + getQueueConfig(MAIN_QUEUE_NAME, MAIN_QUEUE_TOPIC), + getQueueConfig(DataConstants.HP_QUEUE_NAME, DataConstants.HP_QUEUE_TOPIC) + )); + tenantProfile = doPost("/api/tenantProfile", tenantProfile, TenantProfile.class); + + loginDifferentTenant(); + verifyUsedQueueAndMessage(DataConstants.HP_QUEUE_NAME, tenantId, hpQueueDevice.getId(), DataConstants.ATTRIBUTES_UPDATED, () -> { + doPost("/api/plugins/telemetry/DEVICE/" + hpQueueDevice.getId() + "/attributes/SERVER_SCOPE", "{\"test\":123}", String.class); + }, usedTpi -> { + assertThat(usedTpi.getTopic()).isEqualTo(DataConstants.HP_QUEUE_TOPIC); + assertThat(usedTpi.getTenantId()).get().isEqualTo(tenantId); + }); + verifyUsedQueueAndMessage(MAIN_QUEUE_NAME, tenantId, mainQueueDevice.getId(), DataConstants.ATTRIBUTES_UPDATED, () -> { + doPost("/api/plugins/telemetry/DEVICE/" + mainQueueDevice.getId() + "/attributes/SERVER_SCOPE", "{\"test\":123}", String.class); + }, usedTpi -> { + assertThat(usedTpi.getTopic()).isEqualTo(MAIN_QUEUE_TOPIC); + assertThat(usedTpi.getTenantId()).get().isEqualTo(tenantId); + }); + } + + @Test + public void testIsolatedQueueDeletion() throws Exception { + loginSysAdmin(); + TenantProfile tenantProfile = new TenantProfile(); + tenantProfile.setName("Test profile"); + TenantProfileData tenantProfileData = new TenantProfileData(); + tenantProfileData.setConfiguration(new DefaultTenantProfileConfiguration()); + tenantProfile.setProfileData(tenantProfileData); + tenantProfile.setIsolatedTbRuleEngine(true); + addQueueConfig(tenantProfile, MAIN_QUEUE_NAME); + tenantProfile = doPost("/api/tenantProfile", tenantProfile, TenantProfile.class); + createDifferentTenant(); + loginSysAdmin(); + savedDifferentTenant.setTenantProfileId(tenantProfile.getId()); + savedDifferentTenant = doPost("/api/tenant", savedDifferentTenant, Tenant.class); + TenantId tenantId = differentTenantId; + await().atMost(10, TimeUnit.SECONDS) + .until(() -> { + TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_RULE_ENGINE, MAIN_QUEUE_NAME, tenantId, tenantId); + return !tpi.getTenantId().get().isSysTenantId(); + }); + TopicPartitionInfo tpi = new TopicPartitionInfo(MAIN_QUEUE_TOPIC, tenantId, 0, false); + String isolatedTopic = tpi.getFullTopicName(); + TbMsg expectedMsg = publishTbMsg(tenantId, tpi); + awaitTbMsg(tbMsg -> tbMsg.getId().equals(expectedMsg.getId()), 10000); // to wait for consumer start + + loginSysAdmin(); + tenantProfile.setIsolatedTbRuleEngine(false); + tenantProfile.getProfileData().setQueueConfiguration(Collections.emptyList()); + tenantProfile = doPost("/api/tenantProfile", tenantProfile, TenantProfile.class); + await().atMost(10, TimeUnit.SECONDS) + .until(() -> partitionService.resolve(ServiceType.TB_RULE_ENGINE, MAIN_QUEUE_NAME, tenantId, tenantId) + .getTenantId().get().isSysTenantId()); + + Deque submittedMsgs = new LinkedList<>(); + await().atLeast(8, TimeUnit.SECONDS) // due to topic-deletion-delay + .atMost(20, TimeUnit.SECONDS) + .pollInterval(1, TimeUnit.SECONDS) + .untilAsserted(() -> { + TbMsg tbMsg = publishTbMsg(tenantId, tpi); + submittedMsgs.add(tbMsg.getId()); + + verify(queueAdmin, times(1)).deleteTopic(eq(isolatedTopic)); + }); + submittedMsgs.removeLast(); + for (UUID msgId : submittedMsgs) { + verify(actorContext, timeout(2000)).tell(argThat(msg -> { + return msg instanceof QueueToRuleEngineMsg && ((QueueToRuleEngineMsg) msg).getMsg().getId().equals(msgId); + })); + } + } + + private TbMsg publishTbMsg(TenantId tenantId, TopicPartitionInfo tpi) { + TbMsg tbMsg = TbMsg.newMsg("POST_TELEMETRY_REQUEST", tenantId, TbMsgMetaData.EMPTY, "{\"test\":1}"); + TransportProtos.ToRuleEngineMsg msg = TransportProtos.ToRuleEngineMsg.newBuilder() + .setTenantIdMSB(tenantId.getId().getMostSignificantBits()) + .setTenantIdLSB(tenantId.getId().getLeastSignificantBits()) + .setTbMsg(TbMsg.toByteString(tbMsg)).build(); + tbClusterService.pushMsgToRuleEngine(tpi, tbMsg.getId(), msg, null); + return tbMsg; + } + + private void verifyUsedQueueAndMessage(String queue, TenantId tenantId, EntityId entityId, String msgType, Runnable action, Consumer tpiAssert) { + await().atMost(15, TimeUnit.SECONDS) + .untilAsserted(() -> { + TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_RULE_ENGINE, queue, tenantId, entityId); + tpiAssert.accept(tpi); + }); + action.run(); + TbMsg tbMsg = awaitTbMsg(msg -> msg.getOriginator().equals(entityId) + && msg.getType().equals(msgType), 10000); + assertThat(tbMsg.getQueueName()).isEqualTo(queue); + + TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_RULE_ENGINE, queue, tenantId, entityId); + tpiAssert.accept(tpi); + } + + protected TbMsg awaitTbMsg(Predicate predicate, int timeoutMillis) { + AtomicReference tbMsgCaptor = new AtomicReference<>(); + verify(actorContext, timeout(timeoutMillis).atLeastOnce()).tell(argThat(actorMsg -> { + if (!(actorMsg instanceof QueueToRuleEngineMsg)) { + return false; + } + TbMsg tbMsg = ((QueueToRuleEngineMsg) actorMsg).getMsg(); + if (predicate.test(tbMsg)) { + tbMsgCaptor.set(tbMsg); + return true; + } + return false; + })); + return tbMsgCaptor.get(); + } + private void addQueueConfig(TenantProfile tenantProfile, String queueName) { + TenantProfileQueueConfiguration queueConfiguration = getQueueConfig(queueName, "tb_rule_engine." + queueName.toLowerCase()); + TenantProfileData profileData = tenantProfile.getProfileData(); + + List configs = profileData.getQueueConfiguration(); + if (configs == null) { + configs = new ArrayList<>(); + } + configs.add(queueConfiguration); + profileData.setQueueConfiguration(configs); + tenantProfile.setProfileData(profileData); + } + + private TenantProfileQueueConfiguration getQueueConfig(String queueName, String topic) { TenantProfileQueueConfiguration queueConfiguration = new TenantProfileQueueConfiguration(); queueConfiguration.setName(queueName); - queueConfiguration.setTopic("tb_rule_engine." + queueName.toLowerCase()); + queueConfiguration.setTopic(topic); queueConfiguration.setPollInterval(25); queueConfiguration.setPartitions(1 + new Random().nextInt(99)); queueConfiguration.setConsumerPerPartition(true); @@ -539,15 +771,7 @@ public class TenantControllerTest extends AbstractControllerTest { processingStrategy.setPauseBetweenRetries(3); processingStrategy.setMaxPauseBetweenRetries(3); queueConfiguration.setProcessingStrategy(processingStrategy); - TenantProfileData profileData = tenantProfile.getProfileData(); - - List configs = profileData.getQueueConfiguration(); - if (configs == null) { - configs = new ArrayList<>(); - } - configs.add(queueConfiguration); - profileData.setQueueConfiguration(configs); - tenantProfile.setProfileData(profileData); + return queueConfiguration; } private List getQueuesFromConfig(List queueConfiguration, List queues) { diff --git a/application/src/test/java/org/thingsboard/server/controller/TenantProfileControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/TenantProfileControllerTest.java index e6b37143ad..eba9fb8b12 100644 --- a/application/src/test/java/org/thingsboard/server/controller/TenantProfileControllerTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/TenantProfileControllerTest.java @@ -167,23 +167,6 @@ public class TenantProfileControllerTest extends AbstractControllerTest { testBroadcastEntityStateChangeEventNeverTenantProfile(); } - @Test - public void testSaveSameTenantProfileWithDifferentIsolatedTbRuleEngine() throws Exception { - loginSysAdmin(); - TenantProfile tenantProfile = this.createTenantProfile("Tenant Profile"); - TenantProfile savedTenantProfile = doPost("/api/tenantProfile", tenantProfile, TenantProfile.class); - savedTenantProfile.setIsolatedTbRuleEngine(true); - addMainQueueConfig(savedTenantProfile); - - Mockito.reset(tbClusterService); - - doPost("/api/tenantProfile", savedTenantProfile) - .andExpect(status().isBadRequest()) - .andExpect(statusReason(containsString("Can't update isolatedTbRuleEngine property"))); - - testBroadcastEntityStateChangeEventNeverTenantProfile(); - } - @Test public void testDeleteTenantProfileWithExistingTenant() throws Exception { loginSysAdmin(); diff --git a/application/src/test/java/org/thingsboard/server/edge/AssetEdgeTest.java b/application/src/test/java/org/thingsboard/server/edge/AssetEdgeTest.java index e8639e0db7..faf0896b81 100644 --- a/application/src/test/java/org/thingsboard/server/edge/AssetEdgeTest.java +++ b/application/src/test/java/org/thingsboard/server/edge/AssetEdgeTest.java @@ -15,19 +15,29 @@ */ package org.thingsboard.server.edge; +import com.datastax.oss.driver.api.core.uuid.Uuids; +import com.fasterxml.jackson.core.type.TypeReference; import com.google.protobuf.AbstractMessage; import org.junit.Assert; import org.junit.Test; import org.thingsboard.server.common.data.Customer; +import org.thingsboard.server.common.data.Device; +import org.thingsboard.server.common.data.StringUtils; import org.thingsboard.server.common.data.asset.Asset; +import org.thingsboard.server.common.data.asset.AssetInfo; import org.thingsboard.server.common.data.edge.Edge; import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.page.PageData; +import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.dao.service.DaoSqlTest; import org.thingsboard.server.gen.edge.v1.AssetProfileUpdateMsg; import org.thingsboard.server.gen.edge.v1.AssetUpdateMsg; import org.thingsboard.server.gen.edge.v1.UpdateMsgType; +import org.thingsboard.server.gen.edge.v1.UplinkMsg; +import org.thingsboard.server.gen.edge.v1.UplinkResponseMsg; +import java.util.List; import java.util.Optional; import java.util.UUID; @@ -154,5 +164,115 @@ public class AssetEdgeTest extends AbstractEdgeTest { Assert.assertEquals(savedAsset.getUuidId().getLeastSignificantBits(), assetUpdateMsg.getIdLSB()); } + @Test + public void testSendAssetToCloud() throws Exception { + UUID uuid = Uuids.timeBased(); + + UplinkMsg.Builder uplinkMsgBuilder = UplinkMsg.newBuilder(); + AssetUpdateMsg.Builder assetUpdateMsgBuilder = AssetUpdateMsg.newBuilder(); + assetUpdateMsgBuilder.setIdMSB(uuid.getMostSignificantBits()); + assetUpdateMsgBuilder.setIdLSB(uuid.getLeastSignificantBits()); + assetUpdateMsgBuilder.setName("Asset Edge 2"); + assetUpdateMsgBuilder.setType("test"); + assetUpdateMsgBuilder.setMsgType(UpdateMsgType.ENTITY_CREATED_RPC_MESSAGE); + testAutoGeneratedCodeByProtobuf(assetUpdateMsgBuilder); + uplinkMsgBuilder.addAssetUpdateMsg(assetUpdateMsgBuilder.build()); + + testAutoGeneratedCodeByProtobuf(uplinkMsgBuilder); + + edgeImitator.expectResponsesAmount(1); + edgeImitator.sendUplinkMsg(uplinkMsgBuilder.build()); + + Assert.assertTrue(edgeImitator.waitForResponses()); + + UplinkResponseMsg latestResponseMsg = edgeImitator.getLatestResponseMsg(); + Assert.assertTrue(latestResponseMsg.getSuccess()); + + Asset asset = doGet("/api/asset/" + uuid, Asset.class); + Assert.assertNotNull(asset); + Assert.assertEquals("Asset Edge 2", asset.getName()); + } + + @Test + public void testSendAssetToCloudWithNameThatAlreadyExistsOnCloud() throws Exception { + String assetOnCloudName = StringUtils.randomAlphanumeric(15); + Asset assetOnCloud = saveAsset(assetOnCloudName); + + UUID uuid = Uuids.timeBased(); + + UplinkMsg.Builder uplinkMsgBuilder = UplinkMsg.newBuilder(); + AssetUpdateMsg.Builder assetUpdateMsgBuilder = AssetUpdateMsg.newBuilder(); + assetUpdateMsgBuilder.setIdMSB(uuid.getMostSignificantBits()); + assetUpdateMsgBuilder.setIdLSB(uuid.getLeastSignificantBits()); + assetUpdateMsgBuilder.setName(assetOnCloudName); + assetUpdateMsgBuilder.setType("test"); + assetUpdateMsgBuilder.setMsgType(UpdateMsgType.ENTITY_CREATED_RPC_MESSAGE); + testAutoGeneratedCodeByProtobuf(assetUpdateMsgBuilder); + uplinkMsgBuilder.addAssetUpdateMsg(assetUpdateMsgBuilder.build()); + + testAutoGeneratedCodeByProtobuf(uplinkMsgBuilder); + + edgeImitator.expectResponsesAmount(1); + edgeImitator.expectMessageAmount(1); + + edgeImitator.sendUplinkMsg(uplinkMsgBuilder.build()); + + Assert.assertTrue(edgeImitator.waitForResponses()); + Assert.assertTrue(edgeImitator.waitForMessages()); + + Optional assetUpdateMsgOpt = edgeImitator.findMessageByType(AssetUpdateMsg.class); + Assert.assertTrue(assetUpdateMsgOpt.isPresent()); + AssetUpdateMsg latestAssetUpdateMsg = assetUpdateMsgOpt.get(); + Assert.assertNotEquals(assetOnCloudName, latestAssetUpdateMsg.getName()); + + UUID newAssetId = new UUID(latestAssetUpdateMsg.getIdMSB(), latestAssetUpdateMsg.getIdLSB()); + + Assert.assertNotEquals(assetOnCloud.getUuidId(), newAssetId); + + Asset asset = doGet("/api/asset/" + newAssetId, Asset.class); + Assert.assertNotNull(asset); + Assert.assertNotEquals(assetOnCloudName, asset.getName()); + } + + @Test + public void testSendDeleteAssetOnEdgeToCloud() throws Exception { + Asset savedAsset = saveAssetOnCloudAndVerifyDeliveryToEdge(); + UplinkMsg.Builder upLinkMsgBuilder = UplinkMsg.newBuilder(); + AssetUpdateMsg.Builder assetDeleteMsgBuilder = AssetUpdateMsg.newBuilder(); + assetDeleteMsgBuilder.setMsgType(UpdateMsgType.ENTITY_DELETED_RPC_MESSAGE); + assetDeleteMsgBuilder.setIdMSB(savedAsset.getUuidId().getMostSignificantBits()); + assetDeleteMsgBuilder.setIdLSB(savedAsset.getUuidId().getLeastSignificantBits()); + testAutoGeneratedCodeByProtobuf(assetDeleteMsgBuilder); + + upLinkMsgBuilder.addAssetUpdateMsg(assetDeleteMsgBuilder.build()); + + testAutoGeneratedCodeByProtobuf(upLinkMsgBuilder); + + edgeImitator.expectResponsesAmount(1); + edgeImitator.sendUplinkMsg(upLinkMsgBuilder.build()); + Assert.assertTrue(edgeImitator.waitForResponses()); + AssetInfo assetInfo = doGet("/api/asset/info/" + savedAsset.getUuidId(), AssetInfo.class); + Assert.assertNotNull(assetInfo); + List edgeAssets = doGetTypedWithPageLink("/api/edge/" + edge.getUuidId() + "/assets?", + new TypeReference>() { + }, new PageLink(100)).getData(); + Assert.assertFalse(edgeAssets.contains(assetInfo)); + } + + private Asset saveAssetOnCloudAndVerifyDeliveryToEdge() throws Exception { + // create asset and assign to edge + Asset savedAsset = saveAsset(StringUtils.randomAlphanumeric(15)); + edgeImitator.expectMessageAmount(1); // asset message + doPost("/api/edge/" + edge.getUuidId() + + "/asset/" + savedAsset.getUuidId(), Device.class); + Assert.assertTrue(edgeImitator.waitForMessages()); + Optional assetUpdateMsgOpt = edgeImitator.findMessageByType(AssetUpdateMsg.class); + Assert.assertTrue(assetUpdateMsgOpt.isPresent()); + AssetUpdateMsg assetUpdateMsg = assetUpdateMsgOpt.get(); + Assert.assertEquals(UpdateMsgType.ENTITY_CREATED_RPC_MESSAGE, assetUpdateMsg.getMsgType()); + Assert.assertEquals(savedAsset.getUuidId().getMostSignificantBits(), assetUpdateMsg.getIdMSB()); + Assert.assertEquals(savedAsset.getUuidId().getLeastSignificantBits(), assetUpdateMsg.getIdLSB()); + return savedAsset; + } } diff --git a/application/src/test/java/org/thingsboard/server/edge/AssetProfileEdgeTest.java b/application/src/test/java/org/thingsboard/server/edge/AssetProfileEdgeTest.java index fd36c11db8..f77975f0a9 100644 --- a/application/src/test/java/org/thingsboard/server/edge/AssetProfileEdgeTest.java +++ b/application/src/test/java/org/thingsboard/server/edge/AssetProfileEdgeTest.java @@ -15,17 +15,22 @@ */ package org.thingsboard.server.edge; +import com.datastax.oss.driver.api.core.uuid.Uuids; import com.google.protobuf.AbstractMessage; import com.google.protobuf.ByteString; import org.junit.Assert; import org.junit.Test; import org.thingsboard.server.common.data.asset.AssetProfile; +import org.thingsboard.server.common.data.id.DashboardId; import org.thingsboard.server.common.data.id.RuleChainId; import org.thingsboard.server.dao.service.DaoSqlTest; import org.thingsboard.server.gen.edge.v1.AssetProfileUpdateMsg; import org.thingsboard.server.gen.edge.v1.UpdateMsgType; +import org.thingsboard.server.gen.edge.v1.UplinkMsg; +import org.thingsboard.server.gen.edge.v1.UplinkResponseMsg; import java.nio.charset.StandardCharsets; +import java.util.UUID; import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; @@ -77,4 +82,56 @@ public class AssetProfileEdgeTest extends AbstractEdgeTest { unAssignFromEdgeAndDeleteRuleChain(buildingsRuleChainId); } + + @Test + public void testSendAssetProfileToCloud() throws Exception { + RuleChainId ruleChainId = createEdgeRuleChainAndAssignToEdge("Asset Profile Rule Chain"); + DashboardId dashboardId = createDashboardAndAssignToEdge("Asset Profile Dashboard"); + + UUID uuid = Uuids.timeBased(); + + UplinkMsg.Builder uplinkMsgBuilder = UplinkMsg.newBuilder(); + AssetProfileUpdateMsg.Builder assetProfileUpdateMsgBuilder = AssetProfileUpdateMsg.newBuilder(); + assetProfileUpdateMsgBuilder.setIdMSB(uuid.getMostSignificantBits()); + assetProfileUpdateMsgBuilder.setIdLSB(uuid.getLeastSignificantBits()); + assetProfileUpdateMsgBuilder.setName("Asset Profile On Edge"); + assetProfileUpdateMsgBuilder.setDefault(false); + assetProfileUpdateMsgBuilder.setDefaultRuleChainIdMSB(ruleChainId.getId().getMostSignificantBits()); + assetProfileUpdateMsgBuilder.setDefaultRuleChainIdLSB(ruleChainId.getId().getLeastSignificantBits()); + assetProfileUpdateMsgBuilder.setDefaultDashboardIdMSB(dashboardId.getId().getMostSignificantBits()); + assetProfileUpdateMsgBuilder.setDefaultDashboardIdLSB(dashboardId.getId().getLeastSignificantBits()); + assetProfileUpdateMsgBuilder.setMsgType(UpdateMsgType.ENTITY_CREATED_RPC_MESSAGE); + testAutoGeneratedCodeByProtobuf(assetProfileUpdateMsgBuilder); + uplinkMsgBuilder.addAssetProfileUpdateMsg(assetProfileUpdateMsgBuilder.build()); + + testAutoGeneratedCodeByProtobuf(uplinkMsgBuilder); + + edgeImitator.expectResponsesAmount(1); + edgeImitator.sendUplinkMsg(uplinkMsgBuilder.build()); + + Assert.assertTrue(edgeImitator.waitForResponses()); + + UplinkResponseMsg latestResponseMsg = edgeImitator.getLatestResponseMsg(); + Assert.assertTrue(latestResponseMsg.getSuccess()); + + AssetProfile assetProfile = doGet("/api/assetProfile/" + uuid, AssetProfile.class); + Assert.assertNotNull(assetProfile); + Assert.assertEquals("Asset Profile On Edge", assetProfile.getName()); + + // delete profile + edgeImitator.expectMessageAmount(1); + doDelete("/api/assetProfile/" + assetProfile.getUuidId()) + .andExpect(status().isOk()); + Assert.assertTrue(edgeImitator.waitForMessages()); + AbstractMessage latestMessage = edgeImitator.getLatestMessage(); + Assert.assertTrue(latestMessage instanceof AssetProfileUpdateMsg); + AssetProfileUpdateMsg assetProfileUpdateMsg = (AssetProfileUpdateMsg) latestMessage; + Assert.assertEquals(UpdateMsgType.ENTITY_DELETED_RPC_MESSAGE, assetProfileUpdateMsg.getMsgType()); + Assert.assertEquals(assetProfile.getUuidId().getMostSignificantBits(), assetProfileUpdateMsg.getIdMSB()); + Assert.assertEquals(assetProfile.getUuidId().getLeastSignificantBits(), assetProfileUpdateMsg.getIdLSB()); + + // cleanup + unAssignFromEdgeAndDeleteDashboard(dashboardId); + unAssignFromEdgeAndDeleteRuleChain(ruleChainId); + } } diff --git a/application/src/test/java/org/thingsboard/server/edge/DashboardEdgeTest.java b/application/src/test/java/org/thingsboard/server/edge/DashboardEdgeTest.java index 94743c4a6b..fd7dfbaafa 100644 --- a/application/src/test/java/org/thingsboard/server/edge/DashboardEdgeTest.java +++ b/application/src/test/java/org/thingsboard/server/edge/DashboardEdgeTest.java @@ -15,6 +15,7 @@ */ package org.thingsboard.server.edge; +import com.datastax.oss.driver.api.core.uuid.Uuids; import com.fasterxml.jackson.core.type.TypeReference; import com.google.protobuf.AbstractMessage; import org.junit.Assert; @@ -22,13 +23,22 @@ import org.junit.Test; import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.server.common.data.Customer; import org.thingsboard.server.common.data.Dashboard; +import org.thingsboard.server.common.data.DashboardInfo; import org.thingsboard.server.common.data.ShortCustomerInfo; +import org.thingsboard.server.common.data.StringUtils; import org.thingsboard.server.common.data.edge.Edge; +import org.thingsboard.server.common.data.page.PageData; +import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.dao.service.DaoSqlTest; import org.thingsboard.server.gen.edge.v1.DashboardUpdateMsg; import org.thingsboard.server.gen.edge.v1.UpdateMsgType; +import org.thingsboard.server.gen.edge.v1.UplinkMsg; +import org.thingsboard.server.gen.edge.v1.UplinkResponseMsg; +import java.util.List; +import java.util.Optional; import java.util.Set; +import java.util.UUID; import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; @@ -149,4 +159,74 @@ public class DashboardEdgeTest extends AbstractEdgeTest { Assert.assertEquals(savedDashboard.getUuidId().getLeastSignificantBits(), dashboardUpdateMsg.getIdLSB()); } + @Test + public void testSendDashboardToCloud() throws Exception { + UUID uuid = Uuids.timeBased(); + + UplinkMsg.Builder uplinkMsgBuilder = UplinkMsg.newBuilder(); + DashboardUpdateMsg.Builder dashboardUpdateMsgBuilder = DashboardUpdateMsg.newBuilder(); + dashboardUpdateMsgBuilder.setIdMSB(uuid.getMostSignificantBits()); + dashboardUpdateMsgBuilder.setIdLSB(uuid.getLeastSignificantBits()); + dashboardUpdateMsgBuilder.setTitle("Edge Test Dashboard"); + dashboardUpdateMsgBuilder.setConfiguration(""); + dashboardUpdateMsgBuilder.setMsgType(UpdateMsgType.ENTITY_CREATED_RPC_MESSAGE); + testAutoGeneratedCodeByProtobuf(dashboardUpdateMsgBuilder); + uplinkMsgBuilder.addDashboardUpdateMsg(dashboardUpdateMsgBuilder.build()); + + testAutoGeneratedCodeByProtobuf(uplinkMsgBuilder); + + edgeImitator.expectResponsesAmount(1); + edgeImitator.sendUplinkMsg(uplinkMsgBuilder.build()); + + Assert.assertTrue(edgeImitator.waitForResponses()); + + Dashboard dashboard = doGet("/api/dashboard/" + uuid, Dashboard.class); + Assert.assertNotNull(dashboard); + Assert.assertEquals("Edge Test Dashboard", dashboard.getName()); + } + + @Test + public void testSendDeleteEntityViewOnEdgeToCloud() throws Exception { + Dashboard savedDashboard = saveDashboardOnCloudAndVerifyDeliveryToEdge(); + + UplinkMsg.Builder upLinkMsgBuilder = UplinkMsg.newBuilder(); + DashboardUpdateMsg.Builder dashboardDeleteMsgBuilder = DashboardUpdateMsg.newBuilder(); + dashboardDeleteMsgBuilder.setMsgType(UpdateMsgType.ENTITY_DELETED_RPC_MESSAGE); + dashboardDeleteMsgBuilder.setIdMSB(savedDashboard.getUuidId().getMostSignificantBits()); + dashboardDeleteMsgBuilder.setIdLSB(savedDashboard.getUuidId().getLeastSignificantBits()); + testAutoGeneratedCodeByProtobuf(dashboardDeleteMsgBuilder); + + upLinkMsgBuilder.addDashboardUpdateMsg(dashboardDeleteMsgBuilder.build()); + + testAutoGeneratedCodeByProtobuf(upLinkMsgBuilder); + + edgeImitator.expectResponsesAmount(1); + edgeImitator.sendUplinkMsg(upLinkMsgBuilder.build()); + Assert.assertTrue(edgeImitator.waitForResponses()); + DashboardInfo dashboardInfo = doGet("/api/dashboard/info/" + savedDashboard.getUuidId(), DashboardInfo.class); + Assert.assertNotNull(dashboardInfo); + List edgeAssets = doGetTypedWithPageLink("/api/edge/" + edge.getUuidId() + "/dashboards?", + new TypeReference>() { + }, new PageLink(100)).getData(); + Assert.assertFalse(edgeAssets.contains(dashboardInfo)); + } + + private Dashboard saveDashboardOnCloudAndVerifyDeliveryToEdge() throws Exception { + // create dashboard and assign to edge + Dashboard dashboard = new Dashboard(); + dashboard.setTitle(StringUtils.randomAlphanumeric(15)); + Dashboard savedDashboard = doPost("/api/dashboard", dashboard, Dashboard.class); + edgeImitator.expectMessageAmount(1); // dashboard message + doPost("/api/edge/" + edge.getUuidId() + + "/dashboard/" + savedDashboard.getUuidId(), Dashboard.class); + Assert.assertTrue(edgeImitator.waitForMessages()); + Optional dashboardUpdateMsgOpt = edgeImitator.findMessageByType(DashboardUpdateMsg.class); + Assert.assertTrue(dashboardUpdateMsgOpt.isPresent()); + DashboardUpdateMsg entityViewUpdateMsg = dashboardUpdateMsgOpt.get(); + Assert.assertEquals(UpdateMsgType.ENTITY_CREATED_RPC_MESSAGE, entityViewUpdateMsg.getMsgType()); + Assert.assertEquals(savedDashboard.getUuidId().getMostSignificantBits(), entityViewUpdateMsg.getIdMSB()); + Assert.assertEquals(savedDashboard.getUuidId().getLeastSignificantBits(), entityViewUpdateMsg.getIdLSB()); + return savedDashboard; + } + } diff --git a/application/src/test/java/org/thingsboard/server/edge/DeviceProfileEdgeTest.java b/application/src/test/java/org/thingsboard/server/edge/DeviceProfileEdgeTest.java index 9d4cc0ca15..4e8664aa97 100644 --- a/application/src/test/java/org/thingsboard/server/edge/DeviceProfileEdgeTest.java +++ b/application/src/test/java/org/thingsboard/server/edge/DeviceProfileEdgeTest.java @@ -15,19 +15,26 @@ */ package org.thingsboard.server.edge; +import com.datastax.oss.driver.api.core.uuid.Uuids; import com.google.protobuf.AbstractMessage; +import com.google.protobuf.ByteString; import org.junit.Assert; import org.junit.Test; import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.server.common.data.DeviceProfile; +import org.thingsboard.server.common.data.DeviceProfileType; import org.thingsboard.server.common.data.DeviceTransportType; import org.thingsboard.server.common.data.OtaPackageInfo; +import org.thingsboard.server.common.data.asset.AssetProfile; import org.thingsboard.server.common.data.device.data.PowerMode; import org.thingsboard.server.common.data.device.data.PowerSavingConfiguration; import org.thingsboard.server.common.data.device.profile.CoapDeviceProfileTransportConfiguration; import org.thingsboard.server.common.data.device.profile.DefaultCoapDeviceTypeConfiguration; +import org.thingsboard.server.common.data.device.profile.DefaultDeviceProfileConfiguration; +import org.thingsboard.server.common.data.device.profile.DefaultDeviceProfileTransportConfiguration; import org.thingsboard.server.common.data.device.profile.DeviceProfileData; import org.thingsboard.server.common.data.device.profile.DeviceProfileTransportConfiguration; +import org.thingsboard.server.common.data.device.profile.DisabledDeviceProfileProvisionConfiguration; import org.thingsboard.server.common.data.device.profile.Lwm2mDeviceProfileTransportConfiguration; import org.thingsboard.server.common.data.device.profile.ProtoTransportPayloadConfiguration; import org.thingsboard.server.common.data.device.profile.SnmpDeviceProfileTransportConfiguration; @@ -46,12 +53,15 @@ import org.thingsboard.server.common.data.transport.snmp.config.impl.TelemetryQu import org.thingsboard.server.dao.service.DaoSqlTest; import org.thingsboard.server.gen.edge.v1.DeviceProfileUpdateMsg; import org.thingsboard.server.gen.edge.v1.UpdateMsgType; +import org.thingsboard.server.gen.edge.v1.UplinkMsg; +import org.thingsboard.server.gen.edge.v1.UplinkResponseMsg; import org.thingsboard.server.transport.AbstractTransportIntegrationTest; import org.thingsboard.server.transport.lwm2m.AbstractLwM2MIntegrationTest; import java.util.ArrayList; import java.util.List; import java.util.Optional; +import java.util.UUID; import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; @@ -260,6 +270,67 @@ public class DeviceProfileEdgeTest extends AbstractEdgeTest { removeDeviceProfileAndDoBasicAssert(deviceProfile); } + @Test + public void testSendDeviceProfileToCloud() throws Exception { + RuleChainId ruleChainId = createEdgeRuleChainAndAssignToEdge("Device Profile Rule Chain"); + DashboardId dashboardId = createDashboardAndAssignToEdge("Device Profile Dashboard"); + + UUID uuid = Uuids.timeBased(); + + UplinkMsg.Builder uplinkMsgBuilder = UplinkMsg.newBuilder(); + DeviceProfileUpdateMsg.Builder deviceProfileUpdateMsgBuilder = DeviceProfileUpdateMsg.newBuilder(); + deviceProfileUpdateMsgBuilder.setIdMSB(uuid.getMostSignificantBits()); + deviceProfileUpdateMsgBuilder.setIdLSB(uuid.getLeastSignificantBits()); + deviceProfileUpdateMsgBuilder.setName("Device Profile On Edge"); + deviceProfileUpdateMsgBuilder.setDefault(false); + deviceProfileUpdateMsgBuilder.setType(DeviceProfileType.DEFAULT.name()); + deviceProfileUpdateMsgBuilder.setProfileDataBytes(ByteString.copyFrom(dataDecodingEncodingService.encode(createProfileData()))); + deviceProfileUpdateMsgBuilder.setDefaultRuleChainIdMSB(ruleChainId.getId().getMostSignificantBits()); + deviceProfileUpdateMsgBuilder.setDefaultRuleChainIdLSB(ruleChainId.getId().getLeastSignificantBits()); + deviceProfileUpdateMsgBuilder.setDefaultDashboardIdMSB(dashboardId.getId().getMostSignificantBits()); + deviceProfileUpdateMsgBuilder.setDefaultDashboardIdLSB(dashboardId.getId().getLeastSignificantBits()); + deviceProfileUpdateMsgBuilder.setMsgType(UpdateMsgType.ENTITY_CREATED_RPC_MESSAGE); + testAutoGeneratedCodeByProtobuf(deviceProfileUpdateMsgBuilder); + uplinkMsgBuilder.addDeviceProfileUpdateMsg(deviceProfileUpdateMsgBuilder.build()); + + testAutoGeneratedCodeByProtobuf(uplinkMsgBuilder); + + edgeImitator.expectResponsesAmount(1); + edgeImitator.sendUplinkMsg(uplinkMsgBuilder.build()); + + Assert.assertTrue(edgeImitator.waitForResponses()); + + UplinkResponseMsg latestResponseMsg = edgeImitator.getLatestResponseMsg(); + Assert.assertTrue(latestResponseMsg.getSuccess()); + + AssetProfile assetProfile = doGet("/api/deviceProfile/" + uuid, AssetProfile.class); + Assert.assertNotNull(assetProfile); + Assert.assertEquals("Device Profile On Edge", assetProfile.getName()); + + // delete profile + edgeImitator.expectMessageAmount(1); + doDelete("/api/deviceProfile/" + assetProfile.getUuidId()) + .andExpect(status().isOk()); + Assert.assertTrue(edgeImitator.waitForMessages()); + AbstractMessage latestMessage = edgeImitator.getLatestMessage(); + Assert.assertTrue(latestMessage instanceof DeviceProfileUpdateMsg); + DeviceProfileUpdateMsg deviceProfileUpdateMsg = (DeviceProfileUpdateMsg) latestMessage; + Assert.assertEquals(UpdateMsgType.ENTITY_DELETED_RPC_MESSAGE, deviceProfileUpdateMsg.getMsgType()); + Assert.assertEquals(assetProfile.getUuidId().getMostSignificantBits(), deviceProfileUpdateMsg.getIdMSB()); + Assert.assertEquals(assetProfile.getUuidId().getLeastSignificantBits(), deviceProfileUpdateMsg.getIdLSB()); + + // cleanup + unAssignFromEdgeAndDeleteDashboard(dashboardId); + unAssignFromEdgeAndDeleteRuleChain(ruleChainId); + } + + private DeviceProfileData createProfileData() { + DeviceProfileData deviceProfileData = new DeviceProfileData(); + deviceProfileData.setConfiguration(new DefaultDeviceProfileConfiguration()); + deviceProfileData.setTransportConfiguration(new DefaultDeviceProfileTransportConfiguration()); + deviceProfileData.setProvisionConfiguration(new DisabledDeviceProfileProvisionConfiguration("Device Secret")); + return deviceProfileData; + } private DeviceProfile createDeviceProfileAndDoBasicAssert(String deviceProfileName, DeviceProfileTransportConfiguration deviceProfileTransportConfiguration) throws Exception { DeviceProfile deviceProfile = this.createDeviceProfile(deviceProfileName, deviceProfileTransportConfiguration); diff --git a/application/src/test/java/org/thingsboard/server/edge/EntityViewEdgeTest.java b/application/src/test/java/org/thingsboard/server/edge/EntityViewEdgeTest.java index 75386dbb17..470fd8e576 100644 --- a/application/src/test/java/org/thingsboard/server/edge/EntityViewEdgeTest.java +++ b/application/src/test/java/org/thingsboard/server/edge/EntityViewEdgeTest.java @@ -15,6 +15,8 @@ */ package org.thingsboard.server.edge; +import com.datastax.oss.driver.api.core.uuid.Uuids; +import com.fasterxml.jackson.core.type.TypeReference; import com.google.protobuf.AbstractMessage; import com.google.protobuf.InvalidProtocolBufferException; import org.junit.Assert; @@ -22,15 +24,24 @@ import org.junit.Test; import org.thingsboard.server.common.data.Customer; import org.thingsboard.server.common.data.Device; import org.thingsboard.server.common.data.EntityView; +import org.thingsboard.server.common.data.EntityViewInfo; +import org.thingsboard.server.common.data.StringUtils; import org.thingsboard.server.common.data.edge.Edge; import org.thingsboard.server.common.data.id.CustomerId; +import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.page.PageData; +import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.dao.service.DaoSqlTest; +import org.thingsboard.server.gen.edge.v1.EdgeEntityType; import org.thingsboard.server.gen.edge.v1.EntityViewUpdateMsg; import org.thingsboard.server.gen.edge.v1.EntityViewsRequestMsg; import org.thingsboard.server.gen.edge.v1.UpdateMsgType; import org.thingsboard.server.gen.edge.v1.UplinkMsg; +import org.thingsboard.server.gen.edge.v1.UplinkResponseMsg; +import java.util.List; +import java.util.Optional; import java.util.UUID; import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; @@ -43,11 +54,7 @@ public class EntityViewEdgeTest extends AbstractEdgeTest { // create entity view and assign to edge edgeImitator.expectMessageAmount(1); Device device = findDeviceByName("Edge Device 1"); - EntityView entityView = new EntityView(); - entityView.setName("Edge EntityView 1"); - entityView.setType("test"); - entityView.setEntityId(device.getId()); - EntityView savedEntityView = doPost("/api/entityView", entityView, EntityView.class); + EntityView savedEntityView = saveEntityView("Edge EntityView 1", device.getId()); doPost("/api/edge/" + edge.getUuidId() + "/entityView/" + savedEntityView.getUuidId(), EntityView.class); Assert.assertTrue(edgeImitator.waitForMessages()); @@ -102,11 +109,7 @@ public class EntityViewEdgeTest extends AbstractEdgeTest { // create entity view #2 and assign to edge edgeImitator.expectMessageAmount(1); - entityView = new EntityView(); - entityView.setName("Edge EntityView 2"); - entityView.setType("test"); - entityView.setEntityId(device.getId()); - savedEntityView = doPost("/api/entityView", entityView, EntityView.class); + savedEntityView = saveEntityView("Edge EntityView 2", device.getId()); doPost("/api/edge/" + edge.getUuidId() + "/entityView/" + savedEntityView.getUuidId(), EntityView.class); Assert.assertTrue(edgeImitator.waitForMessages()); @@ -158,6 +161,115 @@ public class EntityViewEdgeTest extends AbstractEdgeTest { } + @Test + public void testSendEntityViewToCloud() throws Exception { + Device device = findDeviceByName("Edge Device 1"); + + UUID uuid = Uuids.timeBased(); + + UplinkMsg.Builder uplinkMsgBuilder = UplinkMsg.newBuilder(); + EntityViewUpdateMsg.Builder entityViewUpdateMsgBuilder = EntityViewUpdateMsg.newBuilder(); + entityViewUpdateMsgBuilder.setIdMSB(uuid.getMostSignificantBits()); + entityViewUpdateMsgBuilder.setIdLSB(uuid.getLeastSignificantBits()); + entityViewUpdateMsgBuilder.setName("Edge EntityView 2"); + entityViewUpdateMsgBuilder.setType("test"); + entityViewUpdateMsgBuilder.setEntityType(EdgeEntityType.DEVICE); + entityViewUpdateMsgBuilder.setEntityIdMSB(device.getUuidId().getMostSignificantBits()); + entityViewUpdateMsgBuilder.setEntityIdLSB(device.getUuidId().getLeastSignificantBits()); + entityViewUpdateMsgBuilder.setMsgType(UpdateMsgType.ENTITY_CREATED_RPC_MESSAGE); + testAutoGeneratedCodeByProtobuf(entityViewUpdateMsgBuilder); + uplinkMsgBuilder.addEntityViewUpdateMsg(entityViewUpdateMsgBuilder.build()); + + testAutoGeneratedCodeByProtobuf(uplinkMsgBuilder); + + edgeImitator.expectResponsesAmount(1); + edgeImitator.expectMessageAmount(1); + + edgeImitator.sendUplinkMsg(uplinkMsgBuilder.build()); + + Assert.assertTrue(edgeImitator.waitForResponses()); + + UplinkResponseMsg latestResponseMsg = edgeImitator.getLatestResponseMsg(); + Assert.assertTrue(latestResponseMsg.getSuccess()); + + EntityView entityView = doGet("/api/entityView/" + uuid, EntityView.class); + Assert.assertNotNull(entityView); + Assert.assertEquals("Edge EntityView 2", entityView.getName()); + } + + @Test + public void testSendEntityViewToCloudWithNameThatAlreadyExistsOnCloud() throws Exception { + Device device = findDeviceByName("Edge Device 1"); + + String entityViewOnCloudName = StringUtils.randomAlphanumeric(15); + EntityView entityViewOnCloud = saveEntityView(entityViewOnCloudName, device.getId()); + + UUID uuid = Uuids.timeBased(); + + UplinkMsg.Builder uplinkMsgBuilder = UplinkMsg.newBuilder(); + EntityViewUpdateMsg.Builder entityViewUpdateMsgBuilder = EntityViewUpdateMsg.newBuilder(); + entityViewUpdateMsgBuilder.setIdMSB(uuid.getMostSignificantBits()); + entityViewUpdateMsgBuilder.setIdLSB(uuid.getLeastSignificantBits()); + entityViewUpdateMsgBuilder.setName(entityViewOnCloudName); + entityViewUpdateMsgBuilder.setType("test"); + entityViewUpdateMsgBuilder.setEntityType(EdgeEntityType.DEVICE); + entityViewUpdateMsgBuilder.setEntityIdMSB(device.getUuidId().getMostSignificantBits()); + entityViewUpdateMsgBuilder.setEntityIdLSB(device.getUuidId().getLeastSignificantBits()); + entityViewUpdateMsgBuilder.setMsgType(UpdateMsgType.ENTITY_CREATED_RPC_MESSAGE); + testAutoGeneratedCodeByProtobuf(entityViewUpdateMsgBuilder); + uplinkMsgBuilder.addEntityViewUpdateMsg(entityViewUpdateMsgBuilder.build()); + + testAutoGeneratedCodeByProtobuf(uplinkMsgBuilder); + + edgeImitator.expectResponsesAmount(1); + edgeImitator.expectMessageAmount(1); + + edgeImitator.sendUplinkMsg(uplinkMsgBuilder.build()); + + Assert.assertTrue(edgeImitator.waitForResponses()); + Assert.assertTrue(edgeImitator.waitForMessages()); + + Optional entityViewUpdateMsgOpt = edgeImitator.findMessageByType(EntityViewUpdateMsg.class); + Assert.assertTrue(entityViewUpdateMsgOpt.isPresent()); + EntityViewUpdateMsg latestEntityViewUpdateMsg = entityViewUpdateMsgOpt.get(); + Assert.assertNotEquals(entityViewOnCloudName, latestEntityViewUpdateMsg.getName()); + + UUID newEntityViewId = new UUID(latestEntityViewUpdateMsg.getIdMSB(), latestEntityViewUpdateMsg.getIdLSB()); + + Assert.assertNotEquals(entityViewOnCloud.getId().getId(), newEntityViewId); + + EntityView entityView = doGet("/api/entityView/" + newEntityViewId, EntityView.class); + Assert.assertNotNull(entityView); + Assert.assertNotEquals(entityViewOnCloudName, entityView.getName()); + } + + @Test + public void testSendDeleteEntityViewOnEdgeToCloud() throws Exception { + Device device = findDeviceByName("Edge Device 1"); + EntityView savedEntityView = saveEntityViewOnCloudAndVerifyDeliveryToEdge(device); + + UplinkMsg.Builder upLinkMsgBuilder = UplinkMsg.newBuilder(); + EntityViewUpdateMsg.Builder entityViewDeleteMsgBuilder = EntityViewUpdateMsg.newBuilder(); + entityViewDeleteMsgBuilder.setMsgType(UpdateMsgType.ENTITY_DELETED_RPC_MESSAGE); + entityViewDeleteMsgBuilder.setIdMSB(savedEntityView.getUuidId().getMostSignificantBits()); + entityViewDeleteMsgBuilder.setIdLSB(savedEntityView.getUuidId().getLeastSignificantBits()); + testAutoGeneratedCodeByProtobuf(entityViewDeleteMsgBuilder); + + upLinkMsgBuilder.addEntityViewUpdateMsg(entityViewDeleteMsgBuilder.build()); + + testAutoGeneratedCodeByProtobuf(upLinkMsgBuilder); + + edgeImitator.expectResponsesAmount(1); + edgeImitator.sendUplinkMsg(upLinkMsgBuilder.build()); + Assert.assertTrue(edgeImitator.waitForResponses()); + EntityViewInfo entityViewInfo = doGet("/api/entityView/info/" + savedEntityView.getUuidId(), EntityViewInfo.class); + Assert.assertNotNull(entityViewInfo); + List edgeAssets = doGetTypedWithPageLink("/api/edge/" + edge.getUuidId() + "/entityViews?", + new TypeReference>() { + }, new PageLink(100)).getData(); + Assert.assertFalse(edgeAssets.contains(entityViewInfo)); + } + private void verifyEntityViewUpdateMsg(EntityView entityView, Device device) throws InvalidProtocolBufferException { AbstractMessage latestMessage = edgeImitator.getLatestMessage(); Assert.assertTrue(latestMessage instanceof EntityViewUpdateMsg); @@ -173,6 +285,28 @@ public class EntityViewEdgeTest extends AbstractEdgeTest { testAutoGeneratedCodeByProtobuf(entityViewUpdateMsg); } + private EntityView saveEntityViewOnCloudAndVerifyDeliveryToEdge(Device device) throws Exception { + // create entity view and assign to edge + EntityView savedEntityView = saveEntityView(StringUtils.randomAlphanumeric(15), device.getId()); + edgeImitator.expectMessageAmount(1); // entity view message + doPost("/api/edge/" + edge.getUuidId() + + "/entityView/" + savedEntityView.getUuidId(), EntityView.class); + Assert.assertTrue(edgeImitator.waitForMessages()); + Optional entityViewUpdateMsgOpt = edgeImitator.findMessageByType(EntityViewUpdateMsg.class); + Assert.assertTrue(entityViewUpdateMsgOpt.isPresent()); + EntityViewUpdateMsg entityViewUpdateMsg = entityViewUpdateMsgOpt.get(); + Assert.assertEquals(UpdateMsgType.ENTITY_CREATED_RPC_MESSAGE, entityViewUpdateMsg.getMsgType()); + Assert.assertEquals(savedEntityView.getUuidId().getMostSignificantBits(), entityViewUpdateMsg.getIdMSB()); + Assert.assertEquals(savedEntityView.getUuidId().getLeastSignificantBits(), entityViewUpdateMsg.getIdLSB()); + return savedEntityView; + } + private EntityView saveEntityView(String name, DeviceId deviceId) { + EntityView entityView = new EntityView(); + entityView.setName(name); + entityView.setType("test"); + entityView.setEntityId(deviceId); + return doPost("/api/entityView", entityView, EntityView.class); + } } diff --git a/application/src/test/java/org/thingsboard/server/queue/discovery/HashPartitionServiceTest.java b/application/src/test/java/org/thingsboard/server/queue/discovery/HashPartitionServiceTest.java index 25b06e7840..7bd9ec576f 100644 --- a/application/src/test/java/org/thingsboard/server/queue/discovery/HashPartitionServiceTest.java +++ b/application/src/test/java/org/thingsboard/server/queue/discovery/HashPartitionServiceTest.java @@ -18,6 +18,7 @@ package org.thingsboard.server.queue.discovery; import com.datastax.oss.driver.api.core.uuid.Uuids; import lombok.SneakyThrows; import lombok.extern.slf4j.Slf4j; +import org.apache.commons.collections4.ListUtils; import org.junit.Assert; import org.junit.Before; import org.junit.Test; @@ -25,24 +26,36 @@ import org.junit.runner.RunWith; import org.mockito.junit.MockitoJUnitRunner; import org.springframework.context.ApplicationEventPublisher; import org.springframework.test.util.ReflectionTestUtils; +import org.thingsboard.server.common.data.DataConstants; import org.thingsboard.server.common.data.id.DeviceId; +import org.thingsboard.server.common.data.id.QueueId; import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.id.TenantProfileId; +import org.thingsboard.server.common.data.id.UUIDBased; +import org.thingsboard.server.common.data.queue.Queue; import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; -import org.thingsboard.server.gen.transport.TransportProtos; +import org.thingsboard.server.gen.transport.TransportProtos.ServiceInfo; import java.text.SimpleDateFormat; import java.util.ArrayList; +import java.util.Collection; import java.util.Collections; import java.util.Comparator; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Random; +import java.util.Set; +import java.util.UUID; import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; +import java.util.stream.Stream; +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; @Slf4j @RunWith(MockitoJUnitRunner.class) @@ -57,7 +70,7 @@ public class HashPartitionServiceTest { private ApplicationEventPublisher applicationEventPublisher; private QueueRoutingInfoService queueRoutingInfoService; - private String hashFunctionName = "sha256"; + private String hashFunctionName = "murmur3_128"; @Before public void setup() throws Exception { @@ -74,15 +87,15 @@ public class HashPartitionServiceTest { ReflectionTestUtils.setField(clusterRoutingService, "vcTopic", "tb.vc"); ReflectionTestUtils.setField(clusterRoutingService, "vcPartitions", 10); ReflectionTestUtils.setField(clusterRoutingService, "hashFunctionName", hashFunctionName); - TransportProtos.ServiceInfo currentServer = TransportProtos.ServiceInfo.newBuilder() + ServiceInfo currentServer = ServiceInfo.newBuilder() .setServiceId("tb-core-0") .addAllServiceTypes(Collections.singletonList(ServiceType.TB_CORE.name())) .build(); // when(queueService.resolve(Mockito.any(), Mockito.anyString())).thenAnswer(i -> i.getArguments()[1]); // when(discoveryService.getServiceInfo()).thenReturn(currentServer); - List otherServers = new ArrayList<>(); + List otherServers = new ArrayList<>(); for (int i = 1; i < SERVER_COUNT; i++) { - otherServers.add(TransportProtos.ServiceInfo.newBuilder() + otherServers.add(ServiceInfo.newBuilder() .setServiceId("tb-rule-" + i) .addAllServiceTypes(Collections.singletonList(ServiceType.TB_CORE.name())) .build()); @@ -122,10 +135,10 @@ public class HashPartitionServiceTest { int queueCount = 3; int partitionCount = 3; - List services = new ArrayList<>(); + List services = new ArrayList<>(); for (int i = 0; i < serverCount; i++) { - services.add(TransportProtos.ServiceInfo.newBuilder().setServiceId("RE-" + i).build()); + services.add(ServiceInfo.newBuilder().setServiceId("RE-" + i).build()); } long start = System.currentTimeMillis(); @@ -140,7 +153,7 @@ public class HashPartitionServiceTest { for (int queueIndex = 0; queueIndex < queueCount; queueIndex++) { QueueKey queueKey = new QueueKey(ServiceType.TB_RULE_ENGINE, "queue" + queueIndex, tenantId); for (int partition = 0; partition < partitionCount; partition++) { - TransportProtos.ServiceInfo serviceInfo = clusterRoutingService.resolveByPartitionIdx(services, queueKey, partition); + ServiceInfo serviceInfo = clusterRoutingService.resolveByPartitionIdx(services, queueKey, partition); String serviceId = serviceInfo.getServiceId(); map.put(serviceId, map.get(serviceId) + 1); } @@ -163,4 +176,124 @@ public class HashPartitionServiceTest { Assert.assertTrue(diffPercent < maxDiffPercent); } + @Test + public void testPartitionsAssignmentWithDedicatedServers() { + int isolatedProfilesCount = 5; + int tenantsCountPerProfile = 100; + int dedicatedServerSetsCount = 3; + int serversCountPerSet = 3; + int profilesPerSet = (int) Math.ceil((double) isolatedProfilesCount / dedicatedServerSetsCount); + + List isolatedTenantProfiles = Stream.generate(() -> new TenantProfileId(UUID.randomUUID())) + .limit(isolatedProfilesCount).collect(Collectors.toList()); + Map tenants = new HashMap<>(); + for (TenantProfileId tenantProfileId : isolatedTenantProfiles) { + for (int i = 0; i < tenantsCountPerProfile; i++) { + tenants.put(new TenantId(UUID.randomUUID()), tenantProfileId); + } + } + + List queues = new ArrayList<>(); + Queue systemQueue = new Queue(); + systemQueue.setTenantId(TenantId.SYS_TENANT_ID); + systemQueue.setName("Main"); + systemQueue.setTopic(DataConstants.MAIN_QUEUE_TOPIC); + systemQueue.setPartitions(10); + systemQueue.setId(new QueueId(UUID.randomUUID())); + queues.add(systemQueue); + tenants.forEach((tenantId, profileId) -> { + Queue isolatedQueue = new Queue(); + isolatedQueue.setTenantId(tenantId); + isolatedQueue.setName("Main"); + isolatedQueue.setTopic(DataConstants.MAIN_QUEUE_TOPIC); + isolatedQueue.setPartitions(2); + isolatedQueue.setId(new QueueId(UUID.randomUUID())); + queues.add(isolatedQueue); + when(routingInfoService.getRoutingInfo(eq(tenantId))).thenReturn(new TenantRoutingInfo(tenantId, profileId, true)); + }); + when(queueRoutingInfoService.getAllQueuesRoutingInfo()).thenReturn(queues.stream() + .map(QueueRoutingInfo::new).collect(Collectors.toList())); + + List ruleEngines = new ArrayList<>(); + Map> dedicatedServers = new HashMap<>(); + int serviceId = 0; + for (int i = 0; i < serversCountPerSet; i++) { + ServiceInfo commonServer = ServiceInfo.newBuilder() + .setServiceId("tb-rule-engine-" + serviceId) + .addAllServiceTypes(List.of(ServiceType.TB_RULE_ENGINE.name())) + .build(); + ruleEngines.add(commonServer); + serviceId++; + } + for (int i = 0; i < dedicatedServerSetsCount; i++) { + List assignedProfiles = ListUtils.partition(isolatedTenantProfiles, profilesPerSet).get(i); + for (int j = 0; j < serversCountPerSet; j++) { + ServiceInfo dedicatedServer = ServiceInfo.newBuilder() + .setServiceId("tb-rule-engine-" + serviceId) + .addAllServiceTypes(List.of(ServiceType.TB_RULE_ENGINE.name())) + .addAllAssignedTenantProfiles(assignedProfiles.stream().map(UUIDBased::toString).collect(Collectors.toList())) + .build(); + ruleEngines.add(dedicatedServer); + serviceId++; + + for (TenantProfileId assignedProfileId : assignedProfiles) { + dedicatedServers.computeIfAbsent(assignedProfileId, p -> new ArrayList<>()).add(dedicatedServer); + } + } + } + + Map>> serversPartitions = new HashMap<>(); + clusterRoutingService.init(); + for (ServiceInfo ruleEngine : ruleEngines) { + List other = new ArrayList<>(ruleEngines); + other.removeIf(serviceInfo -> serviceInfo.getServiceId().equals(ruleEngine.getServiceId())); + + clusterRoutingService.recalculatePartitions(ruleEngine, other); + clusterRoutingService.myPartitions.forEach((queueKey, partitions) -> { + serversPartitions.computeIfAbsent(queueKey, k -> new HashMap<>()).put(ruleEngine, partitions); + }); + } + assertThat(serversPartitions.keySet()).containsAll(queues.stream().map(queue -> new QueueKey(ServiceType.TB_RULE_ENGINE, queue)).collect(Collectors.toList())); + + serversPartitions.forEach((queueKey, partitionsPerServer) -> { + if (queueKey.getTenantId().isSysTenantId()) { + partitionsPerServer.forEach((server, partitions) -> { + assertThat(server.getAssignedTenantProfilesCount()).as("system queues are not assigned to dedicated servers").isZero(); + }); + } else { + List responsibleServers = dedicatedServers.get(tenants.get(queueKey.getTenantId())); + partitionsPerServer.forEach((server, partitions) -> { + assertThat(server.getAssignedTenantProfilesCount()).as("isolated queues are only assigned to dedicated servers").isPositive(); + assertThat(responsibleServers).contains(server); + }); + } + + List allPartitions = partitionsPerServer.values().stream() + .flatMap(Collection::stream) + .collect(Collectors.toList()); + assertThat(allPartitions).doesNotHaveDuplicates(); + }); + } + + @Test + public void testIsManagedByCurrentServiceCheck() { + TenantProfileId isolatedProfileId = new TenantProfileId(UUID.randomUUID()); + when(discoveryService.getAssignedTenantProfiles()).thenReturn(Set.of(isolatedProfileId.getId())); // dedicated server + TenantProfileId regularProfileId = new TenantProfileId(UUID.randomUUID()); + + TenantId isolatedTenantId = new TenantId(UUID.randomUUID()); + when(routingInfoService.getRoutingInfo(eq(isolatedTenantId))).thenReturn(new TenantRoutingInfo(isolatedTenantId, isolatedProfileId, true)); + TenantId regularTenantId = new TenantId(UUID.randomUUID()); + when(routingInfoService.getRoutingInfo(eq(regularTenantId))).thenReturn(new TenantRoutingInfo(regularTenantId, regularProfileId, false)); + + assertThat(clusterRoutingService.isManagedByCurrentService(isolatedTenantId)).isTrue(); + assertThat(clusterRoutingService.isManagedByCurrentService(regularTenantId)).isFalse(); + + + when(discoveryService.getAssignedTenantProfiles()).thenReturn(Collections.emptySet()); // common server + + assertThat(clusterRoutingService.isManagedByCurrentService(isolatedTenantId)).isTrue(); + assertThat(clusterRoutingService.isManagedByCurrentService(regularTenantId)).isTrue(); + } + } diff --git a/application/src/test/java/org/thingsboard/server/service/notification/NotificationApiTest.java b/application/src/test/java/org/thingsboard/server/service/notification/NotificationApiTest.java index 851550eee7..e9c0000565 100644 --- a/application/src/test/java/org/thingsboard/server/service/notification/NotificationApiTest.java +++ b/application/src/test/java/org/thingsboard/server/service/notification/NotificationApiTest.java @@ -15,6 +15,7 @@ */ package org.thingsboard.server.service.notification; +import com.google.common.util.concurrent.SettableFuture; import lombok.extern.slf4j.Slf4j; import org.assertj.core.data.Offset; import org.java_websocket.client.WebSocketClient; @@ -23,6 +24,7 @@ import org.junit.Test; import org.springframework.beans.factory.annotation.Autowired; import org.thingsboard.rule.engine.api.NotificationCenter; import org.thingsboard.server.common.data.User; +import org.thingsboard.server.common.data.id.NotificationRuleId; import org.thingsboard.server.common.data.id.NotificationTargetId; import org.thingsboard.server.common.data.notification.Notification; import org.thingsboard.server.common.data.notification.NotificationDeliveryMethod; @@ -35,6 +37,7 @@ import org.thingsboard.server.common.data.notification.NotificationRequestStatus import org.thingsboard.server.common.data.notification.NotificationType; import org.thingsboard.server.common.data.notification.settings.NotificationSettings; import org.thingsboard.server.common.data.notification.settings.SlackNotificationDeliveryMethodConfig; +import org.thingsboard.server.common.data.notification.settings.UserNotificationSettings; import org.thingsboard.server.common.data.notification.targets.NotificationTarget; import org.thingsboard.server.common.data.notification.targets.platform.CustomerUsersFilter; import org.thingsboard.server.common.data.notification.targets.platform.PlatformUsersNotificationTargetConfig; @@ -59,6 +62,7 @@ import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.UUID; import java.util.concurrent.TimeUnit; import static org.assertj.core.api.Assertions.assertThat; @@ -470,6 +474,66 @@ public class NotificationApiTest extends AbstractNotificationApiTest { assertThat(stats.getSent().get(NotificationDeliveryMethod.WEB)).hasValue(1); } + @Test + public void testUserNotificationSettings() throws Exception { + var entityActionNotificationPref = new UserNotificationSettings.NotificationPref(); + entityActionNotificationPref.setEnabled(true); + entityActionNotificationPref.setEnabledDeliveryMethods(Map.of( + NotificationDeliveryMethod.WEB, true, + NotificationDeliveryMethod.SMS, false, + NotificationDeliveryMethod.EMAIL, false + )); + + var entitiesLimitNotificationPref = new UserNotificationSettings.NotificationPref(); + entitiesLimitNotificationPref.setEnabled(true); + entitiesLimitNotificationPref.setEnabledDeliveryMethods(Map.of( + NotificationDeliveryMethod.SMS, true, + NotificationDeliveryMethod.WEB, false, + NotificationDeliveryMethod.EMAIL, false + )); + + var apiUsageLimitNotificationPref = new UserNotificationSettings.NotificationPref(); + apiUsageLimitNotificationPref.setEnabled(false); + apiUsageLimitNotificationPref.setEnabledDeliveryMethods(Map.of( + NotificationDeliveryMethod.WEB, true, + NotificationDeliveryMethod.SMS, false, + NotificationDeliveryMethod.EMAIL, false + )); + + UserNotificationSettings settings = new UserNotificationSettings(Map.of( + NotificationType.ENTITY_ACTION, entityActionNotificationPref, + NotificationType.ENTITIES_LIMIT, entitiesLimitNotificationPref, + NotificationType.API_USAGE_LIMIT, apiUsageLimitNotificationPref + )); + doPost("/api/notification/settings/user", settings, UserNotificationSettings.class); + + var entityActionNotificationTemplate = createNotificationTemplate(NotificationType.ENTITY_ACTION, "Entity action", "Entity action", NotificationDeliveryMethod.WEB); + var entitiesLimitNotificationTemplate = createNotificationTemplate(NotificationType.ENTITIES_LIMIT, "Entities limit", "Entities limit", NotificationDeliveryMethod.WEB); + var apiUsageLimitNotificationTemplate = createNotificationTemplate(NotificationType.API_USAGE_LIMIT, "API usage limit", "API usage limit", NotificationDeliveryMethod.WEB); + NotificationTarget target = createNotificationTarget(tenantAdminUserId); + + NotificationRequest notificationRequest = NotificationRequest.builder() + .tenantId(tenantId) + .templateId(entityActionNotificationTemplate.getId()) + .originatorEntityId(tenantAdminUserId) + .targets(List.of(target.getUuidId())) + .ruleId(new NotificationRuleId(UUID.randomUUID())) // to trigger user settings check + .build(); + NotificationRequestStats stats = submitNotificationRequestAndWait(notificationRequest); + assertThat(stats.getErrors()).isEmpty(); + assertThat(stats.getSent().get(NotificationDeliveryMethod.WEB).get()).isOne(); + + notificationRequest.setTemplateId(entitiesLimitNotificationTemplate.getId()); + stats = submitNotificationRequestAndWait(notificationRequest); + assertThat(stats.getSent().get(NotificationDeliveryMethod.WEB)).matches(n -> n == null || n.get() == 0); + assertThat(stats.getErrors().get(NotificationDeliveryMethod.WEB).values()).first().asString().contains("disabled"); + + notificationRequest.setTemplateId(apiUsageLimitNotificationTemplate.getId()); + stats = submitNotificationRequestAndWait(notificationRequest); + assertThat(stats.getSent().get(NotificationDeliveryMethod.WEB)).matches(n -> n == null || n.get() == 0); + assertThat(stats.getErrors().get(NotificationDeliveryMethod.WEB).values()).first().asString().contains("disabled"); + } + @Test public void testSlackNotifications() throws Exception { NotificationSettings settings = new NotificationSettings(); @@ -524,6 +588,12 @@ public class NotificationApiTest extends AbstractNotificationApiTest { assertThat(stats.getErrors().get(NotificationDeliveryMethod.SLACK).values()).containsExactly(errorMessage); } + private NotificationRequestStats submitNotificationRequestAndWait(NotificationRequest notificationRequest) throws Exception { + SettableFuture future = SettableFuture.create(); + notificationCenter.processNotificationRequest(notificationRequest.getTenantId(), notificationRequest, future::set); + return future.get(30, TimeUnit.SECONDS); + } + private void checkFullNotificationsUpdate(UnreadNotificationsUpdate notificationsUpdate, String... expectedNotifications) { assertThat(notificationsUpdate.getNotifications()).extracting(Notification::getText).containsOnly(expectedNotifications); assertThat(notificationsUpdate.getNotifications()).extracting(Notification::getType).containsOnly(DEFAULT_NOTIFICATION_TYPE); diff --git a/application/src/test/java/org/thingsboard/server/service/notification/NotificationRuleApiTest.java b/application/src/test/java/org/thingsboard/server/service/notification/NotificationRuleApiTest.java index f89730b689..0a2854a28d 100644 --- a/application/src/test/java/org/thingsboard/server/service/notification/NotificationRuleApiTest.java +++ b/application/src/test/java/org/thingsboard/server/service/notification/NotificationRuleApiTest.java @@ -342,7 +342,7 @@ public class NotificationRuleApiTest extends AbstractNotificationApiTest { @Test public void testNotificationRuleProcessing_entitiesLimit() throws Exception { int limit = 5; - updateDefaultTenantProfile(profileConfiguration -> { + updateDefaultTenantProfileConfig(profileConfiguration -> { profileConfiguration.setMaxDevices(limit); profileConfiguration.setMaxAssets(limit); profileConfiguration.setMaxCustomers(limit); @@ -421,10 +421,10 @@ public class NotificationRuleApiTest extends AbstractNotificationApiTest { int n = 10; updateDefaultTenantProfile(profileConfiguration -> { - profileConfiguration.setTenantEntityExportRateLimit(n + ":600"); - profileConfiguration.setCustomerServerRestLimitsConfiguration(n + ":600"); - profileConfiguration.setTenantNotificationRequestsPerRuleRateLimit(n + ":600"); - profileConfiguration.setTransportDeviceTelemetryMsgRateLimit(n + ":600"); + profileConfiguration.getProfileConfiguration().get().setTenantEntityExportRateLimit(n + ":600"); + profileConfiguration.getProfileConfiguration().get().setCustomerServerRestLimitsConfiguration(n + ":600"); + profileConfiguration.getProfileConfiguration().get().setTenantNotificationRequestsPerRuleRateLimit(n + ":600"); + profileConfiguration.getProfileConfiguration().get().setTransportDeviceTelemetryMsgRateLimit(n + ":600"); }); loginTenantAdmin(); NotificationRule rule = createNotificationRule(AlarmCommentNotificationRuleTriggerConfig.builder() @@ -608,7 +608,7 @@ public class NotificationRuleApiTest extends AbstractNotificationApiTest { @Test public void testNotificationRequestsPerRuleRateLimits() throws Exception { int notificationRequestsLimit = 10; - updateDefaultTenantProfile(profileConfiguration -> { + updateDefaultTenantProfileConfig(profileConfiguration -> { profileConfiguration.setTenantNotificationRequestsPerRuleRateLimit(notificationRequestsLimit + ":300"); }); @@ -691,8 +691,8 @@ public class NotificationRuleApiTest extends AbstractNotificationApiTest { int n = 5; updateDefaultTenantProfile(profileConfiguration -> { - profileConfiguration.setTenantEntityExportRateLimit(n + ":600"); - profileConfiguration.setTransportDeviceTelemetryMsgRateLimit(n + ":800"); + profileConfiguration.getProfileConfiguration().get().setTenantEntityExportRateLimit(n + ":600"); + profileConfiguration.getProfileConfiguration().get().setTransportDeviceTelemetryMsgRateLimit(n + ":800"); }); RateLimitsTrigger expectedTrigger = RateLimitsTrigger.builder() diff --git a/application/src/test/java/org/thingsboard/server/service/notification/MockNotificationSettingsService.java b/application/src/test/java/org/thingsboard/server/service/notification/TestNotificationSettingsService.java similarity index 60% rename from application/src/test/java/org/thingsboard/server/service/notification/MockNotificationSettingsService.java rename to application/src/test/java/org/thingsboard/server/service/notification/TestNotificationSettingsService.java index a49f8dc8cb..9b4ced8567 100644 --- a/application/src/test/java/org/thingsboard/server/service/notification/MockNotificationSettingsService.java +++ b/application/src/test/java/org/thingsboard/server/service/notification/TestNotificationSettingsService.java @@ -19,14 +19,20 @@ import org.springframework.context.annotation.Primary; import org.springframework.stereotype.Service; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.dao.notification.DefaultNotificationSettingsService; +import org.thingsboard.server.dao.notification.NotificationTargetService; +import org.thingsboard.server.dao.notification.NotificationTemplateService; import org.thingsboard.server.dao.settings.AdminSettingsService; +import org.thingsboard.server.dao.user.UserSettingsService; @Service @Primary -public class MockNotificationSettingsService extends DefaultNotificationSettingsService { +public class TestNotificationSettingsService extends DefaultNotificationSettingsService { - public MockNotificationSettingsService(AdminSettingsService adminSettingsService) { - super(adminSettingsService, null, null, null); + public TestNotificationSettingsService(AdminSettingsService adminSettingsService, + NotificationTargetService notificationTargetService, + NotificationTemplateService notificationTemplateService, + UserSettingsService userSettingsService) { + super(adminSettingsService, notificationTargetService, notificationTemplateService, null, userSettingsService); } @Override diff --git a/application/src/test/java/org/thingsboard/server/service/ttl/AlarmsCleanUpServiceTest.java b/application/src/test/java/org/thingsboard/server/service/ttl/AlarmsCleanUpServiceTest.java index 6cabff788e..09e78df9ff 100644 --- a/application/src/test/java/org/thingsboard/server/service/ttl/AlarmsCleanUpServiceTest.java +++ b/application/src/test/java/org/thingsboard/server/service/ttl/AlarmsCleanUpServiceTest.java @@ -67,7 +67,7 @@ public class AlarmsCleanUpServiceTest extends AbstractControllerTest { @Test public void testAlarmsCleanUp() throws Exception { int ttlDays = 1; - updateDefaultTenantProfile(profileConfiguration -> { + updateDefaultTenantProfileConfig(profileConfiguration -> { profileConfiguration.setAlarmsTtlDays(ttlDays); }); diff --git a/application/src/test/resources/application-test.properties b/application/src/test/resources/application-test.properties index 99055e0e5f..d96af8a910 100644 --- a/application/src/test/resources/application-test.properties +++ b/application/src/test/resources/application-test.properties @@ -68,3 +68,5 @@ sql.ttl.audit_logs.ttl=2592000 sql.edge_events.partition_size=168 sql.ttl.edge_events.edge_event_ttl=2592000 + +server.log_controller_error_stack_trace=false diff --git a/common/actor/src/main/java/org/thingsboard/server/actors/TbActorCtx.java b/common/actor/src/main/java/org/thingsboard/server/actors/TbActorCtx.java index 2a8f641c4e..bbe19fb3dd 100644 --- a/common/actor/src/main/java/org/thingsboard/server/actors/TbActorCtx.java +++ b/common/actor/src/main/java/org/thingsboard/server/actors/TbActorCtx.java @@ -32,7 +32,7 @@ public interface TbActorCtx extends TbActorRef { void stop(TbActorId target); - TbActorRef getOrCreateChildActor(TbActorId actorId, Supplier dispatcher, Supplier creator); + TbActorRef getOrCreateChildActor(TbActorId actorId, Supplier dispatcher, Supplier creator, Supplier createCondition); void broadcastToChildren(TbActorMsg msg); diff --git a/common/actor/src/main/java/org/thingsboard/server/actors/TbActorMailbox.java b/common/actor/src/main/java/org/thingsboard/server/actors/TbActorMailbox.java index ad1604f7b0..34537c143c 100644 --- a/common/actor/src/main/java/org/thingsboard/server/actors/TbActorMailbox.java +++ b/common/actor/src/main/java/org/thingsboard/server/actors/TbActorMailbox.java @@ -15,7 +15,8 @@ */ package org.thingsboard.server.actors; -import lombok.Data; +import lombok.Getter; +import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.msg.MsgType; @@ -31,7 +32,8 @@ import java.util.function.Predicate; import java.util.function.Supplier; @Slf4j -@Data +@Getter +@RequiredArgsConstructor public final class TbActorMailbox implements TbActorCtx { private static final boolean HIGH_PRIORITY = true; private static final boolean NORMAL_PRIORITY = false; @@ -212,9 +214,9 @@ public final class TbActorMailbox implements TbActorCtx { } @Override - public TbActorRef getOrCreateChildActor(TbActorId actorId, Supplier dispatcher, Supplier creator) { + public TbActorRef getOrCreateChildActor(TbActorId actorId, Supplier dispatcher, Supplier creator, Supplier createCondition) { TbActorRef actorRef = system.getActor(actorId); - if (actorRef == null) { + if (actorRef == null && createCondition.get()) { return system.createChildActor(dispatcher.get(), creator.get(), selfId); } else { return actorRef; diff --git a/common/cluster-api/src/main/java/org/thingsboard/server/queue/TbQueueAdmin.java b/common/cluster-api/src/main/java/org/thingsboard/server/queue/TbQueueAdmin.java index 4b2bde733e..19aa0284ea 100644 --- a/common/cluster-api/src/main/java/org/thingsboard/server/queue/TbQueueAdmin.java +++ b/common/cluster-api/src/main/java/org/thingsboard/server/queue/TbQueueAdmin.java @@ -17,7 +17,11 @@ package org.thingsboard.server.queue; public interface TbQueueAdmin { - void createTopicIfNotExists(String topic); + default void createTopicIfNotExists(String topic) { + createTopicIfNotExists(topic, null); + } + + void createTopicIfNotExists(String topic, String properties); void destroy(); diff --git a/common/cluster-api/src/main/java/org/thingsboard/server/queue/TbQueueConsumer.java b/common/cluster-api/src/main/java/org/thingsboard/server/queue/TbQueueConsumer.java index 04439fc85d..9c41f9d342 100644 --- a/common/cluster-api/src/main/java/org/thingsboard/server/queue/TbQueueConsumer.java +++ b/common/cluster-api/src/main/java/org/thingsboard/server/queue/TbQueueConsumer.java @@ -36,4 +36,10 @@ public interface TbQueueConsumer { boolean isStopped(); + void onQueueDelete(); + + boolean isQueueDeleted(); + + List getFullTopicNames(); + } diff --git a/common/cluster-api/src/main/proto/queue.proto b/common/cluster-api/src/main/proto/queue.proto index 80f44e59be..78614911d4 100644 --- a/common/cluster-api/src/main/proto/queue.proto +++ b/common/cluster-api/src/main/proto/queue.proto @@ -28,6 +28,7 @@ message ServiceInfo { repeated string serviceTypes = 2; repeated string transports = 6; SystemInfoProto systemInfo = 10; + repeated string assignedTenantProfiles = 11; } message SystemInfoProto { diff --git a/common/dao-api/src/main/java/org/thingsboard/server/dao/asset/AssetProfileService.java b/common/dao-api/src/main/java/org/thingsboard/server/dao/asset/AssetProfileService.java index f89153feee..1ec6fb3fb5 100644 --- a/common/dao-api/src/main/java/org/thingsboard/server/dao/asset/AssetProfileService.java +++ b/common/dao-api/src/main/java/org/thingsboard/server/dao/asset/AssetProfileService.java @@ -31,6 +31,8 @@ public interface AssetProfileService extends EntityDaoService { AssetProfileInfo findAssetProfileInfoById(TenantId tenantId, AssetProfileId assetProfileId); + AssetProfile saveAssetProfile(AssetProfile assetProfile, boolean doValidate); + AssetProfile saveAssetProfile(AssetProfile assetProfile); void deleteAssetProfile(TenantId tenantId, AssetProfileId assetProfileId); diff --git a/common/dao-api/src/main/java/org/thingsboard/server/dao/asset/AssetService.java b/common/dao-api/src/main/java/org/thingsboard/server/dao/asset/AssetService.java index 8b825e0837..722eeff365 100644 --- a/common/dao-api/src/main/java/org/thingsboard/server/dao/asset/AssetService.java +++ b/common/dao-api/src/main/java/org/thingsboard/server/dao/asset/AssetService.java @@ -41,6 +41,8 @@ public interface AssetService extends EntityDaoService { Asset findAssetByTenantIdAndName(TenantId tenantId, String name); + Asset saveAsset(Asset asset, boolean doValidate); + Asset saveAsset(Asset asset); Asset assignAssetToCustomer(TenantId tenantId, AssetId assetId, CustomerId customerId); diff --git a/common/dao-api/src/main/java/org/thingsboard/server/dao/dashboard/DashboardService.java b/common/dao-api/src/main/java/org/thingsboard/server/dao/dashboard/DashboardService.java index c9974bba69..434088180c 100644 --- a/common/dao-api/src/main/java/org/thingsboard/server/dao/dashboard/DashboardService.java +++ b/common/dao-api/src/main/java/org/thingsboard/server/dao/dashboard/DashboardService.java @@ -40,6 +40,8 @@ public interface DashboardService extends EntityDaoService { ListenableFuture findDashboardInfoByIdAsync(TenantId tenantId, DashboardId dashboardId); + Dashboard saveDashboard(Dashboard dashboard, boolean doValidate); + Dashboard saveDashboard(Dashboard dashboard); Dashboard assignDashboardToCustomer(TenantId tenantId, DashboardId dashboardId, CustomerId customerId); diff --git a/common/dao-api/src/main/java/org/thingsboard/server/dao/device/DeviceProfileService.java b/common/dao-api/src/main/java/org/thingsboard/server/dao/device/DeviceProfileService.java index e765cc5030..f94b709c97 100644 --- a/common/dao-api/src/main/java/org/thingsboard/server/dao/device/DeviceProfileService.java +++ b/common/dao-api/src/main/java/org/thingsboard/server/dao/device/DeviceProfileService.java @@ -31,6 +31,8 @@ public interface DeviceProfileService extends EntityDaoService { DeviceProfileInfo findDeviceProfileInfoById(TenantId tenantId, DeviceProfileId deviceProfileId); + DeviceProfile saveDeviceProfile(DeviceProfile deviceProfile, boolean doValidate); + DeviceProfile saveDeviceProfile(DeviceProfile deviceProfile); void deleteDeviceProfile(TenantId tenantId, DeviceProfileId deviceProfileId); diff --git a/common/dao-api/src/main/java/org/thingsboard/server/dao/entityview/EntityViewService.java b/common/dao-api/src/main/java/org/thingsboard/server/dao/entityview/EntityViewService.java index 31b2004a38..ea9dc31b45 100644 --- a/common/dao-api/src/main/java/org/thingsboard/server/dao/entityview/EntityViewService.java +++ b/common/dao-api/src/main/java/org/thingsboard/server/dao/entityview/EntityViewService.java @@ -38,6 +38,8 @@ public interface EntityViewService extends EntityDaoService { EntityView saveEntityView(EntityView entityView); + EntityView saveEntityView(EntityView entityView, boolean doValidate); + EntityView assignEntityViewToCustomer(TenantId tenantId, EntityViewId entityViewId, CustomerId customerId); EntityView unassignEntityViewFromCustomer(TenantId tenantId, EntityViewId entityViewId); diff --git a/common/dao-api/src/main/java/org/thingsboard/server/dao/notification/NotificationSettingsService.java b/common/dao-api/src/main/java/org/thingsboard/server/dao/notification/NotificationSettingsService.java index a5433915b3..9fca174b3a 100644 --- a/common/dao-api/src/main/java/org/thingsboard/server/dao/notification/NotificationSettingsService.java +++ b/common/dao-api/src/main/java/org/thingsboard/server/dao/notification/NotificationSettingsService.java @@ -16,7 +16,9 @@ package org.thingsboard.server.dao.notification; import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.id.UserId; import org.thingsboard.server.common.data.notification.settings.NotificationSettings; +import org.thingsboard.server.common.data.notification.settings.UserNotificationSettings; public interface NotificationSettingsService { @@ -24,6 +26,10 @@ public interface NotificationSettingsService { NotificationSettings findNotificationSettings(TenantId tenantId); + UserNotificationSettings saveUserNotificationSettings(TenantId tenantId, UserId userId, UserNotificationSettings settings); + + UserNotificationSettings getUserNotificationSettings(TenantId tenantId, UserId userId, boolean format); + void createDefaultNotificationConfigs(TenantId tenantId); void updateDefaultNotificationConfigs(TenantId tenantId); diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/notification/NotificationRequestStats.java b/common/data/src/main/java/org/thingsboard/server/common/data/notification/NotificationRequestStats.java index 619e1ad38f..691ecf8bc1 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/notification/NotificationRequestStats.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/notification/NotificationRequestStats.java @@ -54,7 +54,6 @@ public class NotificationRequestStats { public void reportSent(NotificationDeliveryMethod deliveryMethod, NotificationRecipient recipient) { sent.computeIfAbsent(deliveryMethod, k -> new AtomicInteger()).incrementAndGet(); - processedRecipients.computeIfAbsent(deliveryMethod, k -> ConcurrentHashMap.newKeySet()).add(recipient.getId()); } public void reportError(NotificationDeliveryMethod deliveryMethod, Throwable error, NotificationRecipient recipient) { @@ -68,6 +67,10 @@ public class NotificationRequestStats { errors.computeIfAbsent(deliveryMethod, k -> new ConcurrentHashMap<>()).put(recipient.getTitle(), errorMessage); } + public void reportProcessed(NotificationDeliveryMethod deliveryMethod, Object recipientId) { + processedRecipients.computeIfAbsent(deliveryMethod, k -> ConcurrentHashMap.newKeySet()).add(recipientId); + } + public boolean contains(NotificationDeliveryMethod deliveryMethod, Object recipientId) { Set processedRecipients = this.processedRecipients.get(deliveryMethod); return processedRecipients != null && processedRecipients.contains(recipientId); diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/notification/settings/UserNotificationSettings.java b/common/data/src/main/java/org/thingsboard/server/common/data/notification/settings/UserNotificationSettings.java new file mode 100644 index 0000000000..0ad7cc6d78 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/notification/settings/UserNotificationSettings.java @@ -0,0 +1,82 @@ +/** + * Copyright © 2016-2023 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.notification.settings; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.fasterxml.jackson.annotation.JsonProperty; +import lombok.Data; +import org.thingsboard.server.common.data.notification.NotificationDeliveryMethod; +import org.thingsboard.server.common.data.notification.NotificationType; +import org.thingsboard.server.common.data.notification.targets.NotificationTargetType; + +import javax.validation.Valid; +import javax.validation.constraints.AssertTrue; +import javax.validation.constraints.NotNull; +import java.util.Collections; +import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; + +@Data +public class UserNotificationSettings { + + @NotNull + @Valid + private final Map prefs; + + public static final UserNotificationSettings DEFAULT = new UserNotificationSettings(Collections.emptyMap()); + + public static final Set deliveryMethods = NotificationTargetType.PLATFORM_USERS.getSupportedDeliveryMethods(); + + @JsonCreator + public UserNotificationSettings(@JsonProperty("prefs") Map prefs) { + this.prefs = prefs; + } + + public boolean isEnabled(NotificationType notificationType, NotificationDeliveryMethod deliveryMethod) { + NotificationPref pref = prefs.get(notificationType); + if (pref == null) { + return true; + } + if (!pref.isEnabled()) { + return false; + } + return pref.getEnabledDeliveryMethods().getOrDefault(deliveryMethod, true); + } + + @Data + public static class NotificationPref { + private boolean enabled; + @NotNull + private Map enabledDeliveryMethods; + + public static NotificationPref createDefault() { + NotificationPref pref = new NotificationPref(); + pref.setEnabled(true); + pref.setEnabledDeliveryMethods(deliveryMethods.stream().collect(Collectors.toMap(v -> v, v -> true))); + return pref; + } + + @JsonIgnore + @AssertTrue(message = "Only email, Web and SMS delivery methods are allowed") + public boolean isValid() { + return enabledDeliveryMethods.entrySet().stream() + .allMatch(entry -> deliveryMethods.contains(entry.getKey()) && entry.getValue() != null); + } + } + +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/page/SortOrder.java b/common/data/src/main/java/org/thingsboard/server/common/data/page/SortOrder.java index 5c540995aa..62a6377519 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/page/SortOrder.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/page/SortOrder.java @@ -36,4 +36,6 @@ public class SortOrder { ASC, DESC } + public static final SortOrder BY_CREATED_TIME_DESC = new SortOrder("createdTime", Direction.DESC); + } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/queue/Queue.java b/common/data/src/main/java/org/thingsboard/server/common/data/queue/Queue.java index 6839c60bcd..5b613a7c05 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/queue/Queue.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/queue/Queue.java @@ -15,6 +15,8 @@ */ package org.thingsboard.server.common.data.queue; +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.fasterxml.jackson.databind.JsonNode; import lombok.Data; import org.thingsboard.server.common.data.BaseDataWithAdditionalInfo; import org.thingsboard.server.common.data.HasName; @@ -25,6 +27,8 @@ import org.thingsboard.server.common.data.tenant.profile.TenantProfileQueueConfi import org.thingsboard.server.common.data.validation.Length; import org.thingsboard.server.common.data.validation.NoXss; +import java.util.Optional; + @Data public class Queue extends BaseDataWithAdditionalInfo implements HasName, HasTenantId { private TenantId tenantId; @@ -60,4 +64,13 @@ public class Queue extends BaseDataWithAdditionalInfo implements HasNam this.processingStrategy = queueConfiguration.getProcessingStrategy(); setAdditionalInfo(queueConfiguration.getAdditionalInfo()); } -} \ No newline at end of file + + + @JsonIgnore + public String getCustomProperties() { + return Optional.ofNullable(getAdditionalInfo()) + .map(info -> info.get("customProperties")) + .filter(JsonNode::isTextual).map(JsonNode::asText).orElse(null); + } + +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/settings/UserSettingsType.java b/common/data/src/main/java/org/thingsboard/server/common/data/settings/UserSettingsType.java index b19dbbbaee..cd627821ac 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/settings/UserSettingsType.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/settings/UserSettingsType.java @@ -19,7 +19,7 @@ import lombok.Getter; public enum UserSettingsType { - GENERAL, VISITED_DASHBOARDS(true), QUICK_LINKS, DOC_LINKS, DASHBOARDS, GETTING_STARTED; + GENERAL, VISITED_DASHBOARDS(true), QUICK_LINKS, DOC_LINKS, DASHBOARDS, GETTING_STARTED, NOTIFICATIONS; @Getter private final boolean reserved; diff --git a/common/edge-api/src/main/java/org/thingsboard/edge/rpc/EdgeGrpcClient.java b/common/edge-api/src/main/java/org/thingsboard/edge/rpc/EdgeGrpcClient.java index 4e8548aa33..dcd543a9a6 100644 --- a/common/edge-api/src/main/java/org/thingsboard/edge/rpc/EdgeGrpcClient.java +++ b/common/edge-api/src/main/java/org/thingsboard/edge/rpc/EdgeGrpcClient.java @@ -221,17 +221,11 @@ public class EdgeGrpcClient implements EdgeRpcClient { } @Override - public void sendSyncRequestMsg(boolean syncRequired) { - sendSyncRequestMsg(syncRequired, true); - } - - @Override - public void sendSyncRequestMsg(boolean syncRequired, boolean fullSync) { + public void sendSyncRequestMsg(boolean fullSyncRequired) { uplinkMsgLock.lock(); try { SyncRequestMsg syncRequestMsg = SyncRequestMsg.newBuilder() - .setSyncRequired(syncRequired) - .setFullSync(fullSync) + .setFullSync(fullSyncRequired) .build(); this.inputStream.onNext(RequestMsg.newBuilder() .setMsgType(RequestMsgType.SYNC_REQUEST_RPC_MESSAGE) diff --git a/common/edge-api/src/main/java/org/thingsboard/edge/rpc/EdgeRpcClient.java b/common/edge-api/src/main/java/org/thingsboard/edge/rpc/EdgeRpcClient.java index 44d00e22a8..9f2a303f5b 100644 --- a/common/edge-api/src/main/java/org/thingsboard/edge/rpc/EdgeRpcClient.java +++ b/common/edge-api/src/main/java/org/thingsboard/edge/rpc/EdgeRpcClient.java @@ -34,9 +34,7 @@ public interface EdgeRpcClient { void disconnect(boolean onError) throws InterruptedException; - void sendSyncRequestMsg(boolean syncRequired); - - void sendSyncRequestMsg(boolean syncRequired, boolean fullSync); + void sendSyncRequestMsg(boolean fullSyncRequired); void sendUplinkMsg(UplinkMsg uplinkMsg); diff --git a/common/edge-api/src/main/proto/edge.proto b/common/edge-api/src/main/proto/edge.proto index afcf4056c4..94418566c2 100644 --- a/common/edge-api/src/main/proto/edge.proto +++ b/common/edge-api/src/main/proto/edge.proto @@ -85,7 +85,7 @@ message ConnectResponseMsg { } message SyncRequestMsg { - bool syncRequired = 1; + bool syncRequired = 1; // deprecated optional bool fullSync = 2; } @@ -529,6 +529,11 @@ message UplinkMsg { repeated DeviceProfileDevicesRequestMsg deviceProfileDevicesRequestMsg = 13; // deprecated repeated WidgetBundleTypesRequestMsg widgetBundleTypesRequestMsg = 14; repeated EntityViewsRequestMsg entityViewsRequestMsg = 15; + repeated AssetUpdateMsg assetUpdateMsg = 16; + repeated DashboardUpdateMsg dashboardUpdateMsg = 17; + repeated EntityViewUpdateMsg entityViewUpdateMsg = 18; + repeated AssetProfileUpdateMsg assetProfileUpdateMsg = 19; + repeated DeviceProfileUpdateMsg deviceProfileUpdateMsg = 20; } message UplinkResponseMsg { diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/RuleEngineTbQueueAdminFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/RuleEngineTbQueueAdminFactory.java index fe29c3a04f..7a8764325f 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/RuleEngineTbQueueAdminFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/RuleEngineTbQueueAdminFactory.java @@ -99,7 +99,7 @@ public class RuleEngineTbQueueAdminFactory { return new TbQueueAdmin() { @Override - public void createTopicIfNotExists(String topic) { + public void createTopicIfNotExists(String topic, String properties) { } @Override diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/azure/servicebus/TbServiceBusAdmin.java b/common/queue/src/main/java/org/thingsboard/server/queue/azure/servicebus/TbServiceBusAdmin.java index e171bb7a31..d95d2064a4 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/azure/servicebus/TbServiceBusAdmin.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/azure/servicebus/TbServiceBusAdmin.java @@ -22,6 +22,7 @@ import com.microsoft.azure.servicebus.primitives.MessagingEntityAlreadyExistsExc import com.microsoft.azure.servicebus.primitives.ServiceBusException; import lombok.extern.slf4j.Slf4j; import org.thingsboard.server.queue.TbQueueAdmin; +import org.thingsboard.server.queue.util.PropertyUtils; import java.io.IOException; import java.time.Duration; @@ -60,7 +61,7 @@ public class TbServiceBusAdmin implements TbQueueAdmin { } @Override - public void createTopicIfNotExists(String topic) { + public void createTopicIfNotExists(String topic, String properties) { if (queues.contains(topic)) { return; } @@ -68,7 +69,7 @@ public class TbServiceBusAdmin implements TbQueueAdmin { try { QueueDescription queueDescription = new QueueDescription(topic); queueDescription.setRequiresDuplicateDetection(false); - setQueueConfigs(queueDescription); + setQueueConfigs(queueDescription, PropertyUtils.getProps(queueConfigs, properties)); client.createQueue(queueDescription); queues.add(topic); @@ -107,7 +108,7 @@ public class TbServiceBusAdmin implements TbQueueAdmin { } } - private void setQueueConfigs(QueueDescription queueDescription) { + private void setQueueConfigs(QueueDescription queueDescription, Map queueConfigs) { queueConfigs.forEach((confKey, confValue) -> { switch (confKey) { case MAX_SIZE: diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/common/AbstractTbQueueConsumerTemplate.java b/common/queue/src/main/java/org/thingsboard/server/queue/common/AbstractTbQueueConsumerTemplate.java index 86146dabd1..2ebe41850d 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/common/AbstractTbQueueConsumerTemplate.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/common/AbstractTbQueueConsumerTemplate.java @@ -44,6 +44,7 @@ public abstract class AbstractTbQueueConsumerTemplate i protected volatile Set partitions; protected final ReentrantLock consumerLock = new ReentrantLock(); //NonfairSync final Queue> subscribeQueue = new ConcurrentLinkedQueue<>(); + protected volatile boolean queueDeleted = false; @Getter private final String topic; @@ -94,7 +95,7 @@ public abstract class AbstractTbQueueConsumerTemplate i partitions = subscribeQueue.poll(); } if (!subscribed) { - List topicNames = partitions.stream().map(TopicPartitionInfo::getFullTopicName).collect(Collectors.toList()); + List topicNames = getFullTopicNames(); doSubscribe(topicNames); subscribed = true; } @@ -103,7 +104,9 @@ public abstract class AbstractTbQueueConsumerTemplate i consumerLock.unlock(); } - if (records.isEmpty()) { return sleepAndReturnEmpty(startNanos, durationInMillis); } + if (records.isEmpty() && !isLongPollingSupported()) { + return sleepAndReturnEmpty(startNanos, durationInMillis); + } return decodeRecords(records); } @@ -162,7 +165,9 @@ public abstract class AbstractTbQueueConsumerTemplate i @Override public void unsubscribe() { - log.info("unsubscribe topic and stop consumer {}", getTopic()); + log.info("Unsubscribing from topics and stopping consumer for topics {}", partitions.stream() + .map(TopicPartitionInfo::getFullTopicName) + .collect(Collectors.joining(", "))); stopped = true; consumerLock.lock(); try { @@ -187,4 +192,22 @@ public abstract class AbstractTbQueueConsumerTemplate i abstract protected void doUnsubscribe(); + @Override + public void onQueueDelete() { + queueDeleted = true; + } + + public boolean isQueueDeleted() { + return queueDeleted; + } + + @Override + public List getFullTopicNames() { + return partitions.stream().map(TopicPartitionInfo::getFullTopicName).collect(Collectors.toList()); + } + + protected boolean isLongPollingSupported() { + return false; + } + } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/DefaultTbServiceInfoProvider.java b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/DefaultTbServiceInfoProvider.java index 3c85aef350..64a8b70a1f 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/DefaultTbServiceInfoProvider.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/DefaultTbServiceInfoProvider.java @@ -23,6 +23,7 @@ import org.springframework.context.ApplicationContext; import org.springframework.stereotype.Component; import org.thingsboard.server.common.data.StringUtils; import org.thingsboard.server.common.data.TbTransportService; +import org.thingsboard.server.common.data.util.CollectionsUtil; import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.gen.transport.TransportProtos.ServiceInfo; @@ -35,6 +36,8 @@ import java.util.Arrays; import java.util.Collection; import java.util.Collections; import java.util.List; +import java.util.Set; +import java.util.UUID; import java.util.stream.Collectors; import static org.thingsboard.common.util.SystemUtil.getCpuCount; @@ -57,6 +60,10 @@ public class DefaultTbServiceInfoProvider implements TbServiceInfoProvider { @Value("${service.type:monolith}") private String serviceType; + @Getter + @Value("${service.rule_engine.assigned_tenant_profiles:}") + private Set assignedTenantProfiles; + @Autowired private ApplicationContext applicationContext; @@ -78,6 +85,9 @@ public class DefaultTbServiceInfoProvider implements TbServiceInfoProvider { } else { serviceTypes = Collections.singletonList(ServiceType.of(serviceType)); } + if (!serviceTypes.contains(ServiceType.TB_RULE_ENGINE) || assignedTenantProfiles == null) { + assignedTenantProfiles = Collections.emptySet(); + } generateNewServiceInfoWithCurrentSystemInfo(); } @@ -111,7 +121,9 @@ public class DefaultTbServiceInfoProvider implements TbServiceInfoProvider { .setServiceId(serviceId) .addAllServiceTypes(serviceTypes.stream().map(ServiceType::name).collect(Collectors.toList())) .setSystemInfo(getCurrentSystemInfoProto()); - + if (CollectionsUtil.isNotEmpty(assignedTenantProfiles)) { + builder.addAllAssignedTenantProfiles(assignedTenantProfiles.stream().map(UUID::toString).collect(Collectors.toList())); + } return serviceInfo = builder.build(); } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/HashPartitionService.java b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/HashPartitionService.java index 14fb0369ee..2db342d4e6 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/HashPartitionService.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/HashPartitionService.java @@ -24,6 +24,7 @@ import org.springframework.stereotype.Service; import org.thingsboard.server.common.data.exception.TenantNotFoundException; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.id.TenantProfileId; import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; import org.thingsboard.server.gen.transport.TransportProtos; @@ -36,6 +37,7 @@ import org.thingsboard.server.queue.util.AfterStartUp; import javax.annotation.PostConstruct; import java.nio.charset.StandardCharsets; import java.util.ArrayList; +import java.util.Collection; import java.util.Collections; import java.util.Comparator; import java.util.HashMap; @@ -48,6 +50,8 @@ import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; import java.util.stream.Collectors; +import static org.thingsboard.server.common.data.DataConstants.MAIN_QUEUE_NAME; + @Service @Slf4j public class HashPartitionService implements PartitionService { @@ -68,15 +72,16 @@ public class HashPartitionService implements PartitionService { private final TenantRoutingInfoService tenantRoutingInfoService; private final QueueRoutingInfoService queueRoutingInfoService; - private volatile ConcurrentMap> myPartitions = new ConcurrentHashMap<>(); + protected volatile ConcurrentMap> myPartitions = new ConcurrentHashMap<>(); private final ConcurrentMap partitionTopicsMap = new ConcurrentHashMap<>(); private final ConcurrentMap partitionSizesMap = new ConcurrentHashMap<>(); private final ConcurrentMap tenantRoutingInfoMap = new ConcurrentHashMap<>(); - private Map> tbTransportServicesByType = new HashMap<>(); private List currentOtherServices; + private final Map> tbTransportServicesByType = new HashMap<>(); + private final Map> responsibleServices = new HashMap<>(); private HashFunction hashFunction; @@ -165,6 +170,9 @@ public class HashPartitionService implements PartitionService { partitionTopicsMap.put(queueKey, queueUpdateMsg.getQueueTopic()); partitionSizesMap.put(queueKey, queueUpdateMsg.getPartitions()); myPartitions.remove(queueKey); + if (!tenantId.isSysTenantId()) { + tenantRoutingInfoMap.remove(tenantId); + } } @Override @@ -178,12 +186,38 @@ public class HashPartitionService implements PartitionService { removeTenant(tenantId); } + @Override + public boolean isManagedByCurrentService(TenantId tenantId) { + Set assignedTenantProfiles = serviceInfoProvider.getAssignedTenantProfiles(); + if (assignedTenantProfiles.isEmpty()) { + // TODO: refactor this for common servers + return true; + } else { + if (tenantId.isSysTenantId()) { + return false; + } + TenantProfileId profileId = tenantRoutingInfoService.getRoutingInfo(tenantId).getProfileId(); + return assignedTenantProfiles.contains(profileId.getId()); + } + } + @Override public TopicPartitionInfo resolve(ServiceType serviceType, String queueName, TenantId tenantId, EntityId entityId) { TenantId isolatedOrSystemTenantId = getIsolatedOrSystemTenantId(serviceType, tenantId); + if (queueName == null) { + queueName = MAIN_QUEUE_NAME; + } QueueKey queueKey = new QueueKey(serviceType, queueName, isolatedOrSystemTenantId); if (!partitionSizesMap.containsKey(queueKey)) { - queueKey = new QueueKey(serviceType, isolatedOrSystemTenantId); + if (isolatedOrSystemTenantId.isSysTenantId()) { + queueKey = new QueueKey(serviceType, TenantId.SYS_TENANT_ID); + } else { + queueKey = new QueueKey(serviceType, queueName, TenantId.SYS_TENANT_ID); + if (!MAIN_QUEUE_NAME.equals(queueName) && !partitionSizesMap.containsKey(queueKey)) { + queueKey = new QueueKey(serviceType, TenantId.SYS_TENANT_ID); + } + log.warn("Using queue {} instead of isolated {} for tenant {}", queueKey, queueName, isolatedOrSystemTenantId); + } } return resolve(queueKey, entityId); } @@ -199,11 +233,12 @@ public class HashPartitionService implements PartitionService { } private TopicPartitionInfo resolve(QueueKey queueKey, EntityId entityId) { - int hash = hashFunction.newHasher() - .putLong(entityId.getId().getMostSignificantBits()) - .putLong(entityId.getId().getLeastSignificantBits()).hash().asInt(); - Integer partitionSize = partitionSizesMap.get(queueKey); + if (partitionSize == null) { + throw new IllegalStateException("Partitions info for queue " + queueKey + " is missing"); + } + + int hash = hash(entityId.getId()); int partition = Math.abs(hash % partitionSize); return buildTopicPartitionInfo(queueKey, partition); @@ -212,6 +247,7 @@ public class HashPartitionService implements PartitionService { @Override public synchronized void recalculatePartitions(ServiceInfo currentService, List otherServices) { tbTransportServicesByType.clear(); + responsibleServices.clear(); logServiceInfo(currentService); otherServices.forEach(this::logServiceInfo); @@ -221,6 +257,7 @@ public class HashPartitionService implements PartitionService { addNode(queueServicesMap, other); } queueServicesMap.values().forEach(list -> list.sort(Comparator.comparing(ServiceInfo::getServiceId))); + responsibleServices.values().forEach(list -> list.sort(Comparator.comparing(ServiceInfo::getServiceId))); final ConcurrentMap> newPartitions = new ConcurrentHashMap<>(); partitionSizesMap.forEach((queueKey, size) -> { @@ -268,6 +305,9 @@ public class HashPartitionService implements PartitionService { changes.addAll(newMap.keySet()); if (!changes.isEmpty()) { applicationEventPublisher.publishEvent(new ClusterTopologyChangeEvent(this, changes)); + responsibleServices.forEach((profileId, serviceInfos) -> { + log.info("Servers responsible for tenant profile {}: {}", profileId, toServiceIds(serviceInfos)); + }); } } @@ -305,9 +345,7 @@ public class HashPartitionService implements PartitionService { @Override public int resolvePartitionIndex(UUID entityId, int partitions) { - int hash = hashFunction.newHasher() - .putLong(entityId.getMostSignificantBits()) - .putLong(entityId.getLeastSignificantBits()).hash().asInt(); + int hash = hash(entityId); return Math.abs(hash % partitions); } @@ -358,16 +396,9 @@ public class HashPartitionService implements PartitionService { if (TenantId.SYS_TENANT_ID.equals(tenantId)) { return false; } - TenantRoutingInfo routingInfo = tenantRoutingInfoMap.get(tenantId); - if (routingInfo == null) { - synchronized (tenantRoutingInfoMap) { - routingInfo = tenantRoutingInfoMap.get(tenantId); - if (routingInfo == null) { - routingInfo = tenantRoutingInfoService.getRoutingInfo(tenantId); - tenantRoutingInfoMap.put(tenantId, routingInfo); - } - } - } + TenantRoutingInfo routingInfo = tenantRoutingInfoMap.computeIfAbsent(tenantId, k -> { + return tenantRoutingInfoService.getRoutingInfo(tenantId); + }); if (routingInfo == null) { throw new TenantNotFoundException(tenantId); } @@ -396,6 +427,19 @@ public class HashPartitionService implements PartitionService { queueServiceList.computeIfAbsent(key, k -> new ArrayList<>()).add(instance); } }); + + if (instance.getAssignedTenantProfilesCount() > 0) { + for (String profileIdStr : instance.getAssignedTenantProfilesList()) { + TenantProfileId profileId; + try { + profileId = new TenantProfileId(UUID.fromString(profileIdStr)); + } catch (IllegalArgumentException e) { + log.warn("Failed to parse '{}' as tenant profile id", profileIdStr); + continue; + } + responsibleServices.computeIfAbsent(profileId, k -> new ArrayList<>()).add(instance); + } + } } else if (ServiceType.TB_CORE.equals(serviceType) || ServiceType.TB_VC_EXECUTOR.equals(serviceType)) { queueServiceList.computeIfAbsent(new QueueKey(serviceType), key -> new ArrayList<>()).add(instance); } @@ -411,18 +455,51 @@ public class HashPartitionService implements PartitionService { return null; } - if (!ServiceType.TB_RULE_ENGINE.equals(queueKey.getType()) || TenantId.SYS_TENANT_ID.equals(queueKey.getTenantId())) { - return servers.get(partition % servers.size()); - } else { - int hash = hashFunction.newHasher().putLong(queueKey.getTenantId().getId().getMostSignificantBits()) - .putLong(queueKey.getTenantId().getId().getLeastSignificantBits()) + TenantId tenantId = queueKey.getTenantId(); + if (queueKey.getType() == ServiceType.TB_RULE_ENGINE) { + if (!responsibleServices.isEmpty()) { // if there are any dedicated servers + TenantProfileId profileId; + if (tenantId != null && !tenantId.isSysTenantId()) { + TenantRoutingInfo routingInfo = tenantRoutingInfoService.getRoutingInfo(tenantId); + profileId = routingInfo.getProfileId(); + } else { + profileId = null; + } + + List responsible = responsibleServices.get(profileId); + if (responsible == null) { + // if there are no dedicated servers for this tenant profile, or for system queues, + // using the servers that are not responsible for any profile + responsible = servers.stream() + .filter(serviceInfo -> serviceInfo.getAssignedTenantProfilesCount() == 0) + .sorted(Comparator.comparing(ServiceInfo::getServiceId)) + .collect(Collectors.toList()); + if (profileId != null) { + log.debug("Using servers {} for profile {}", toServiceIds(responsible), profileId); + } + responsibleServices.put(profileId, responsible); + } + servers = responsible; + } + + int hash = hashFunction.newHasher() + .putLong(tenantId.getId().getMostSignificantBits()) + .putLong(tenantId.getId().getLeastSignificantBits()) .putString(queueKey.getQueueName(), StandardCharsets.UTF_8) .hash().asInt(); - return servers.get(Math.abs((hash + partition) % servers.size())); + } else { + return servers.get(partition % servers.size()); } } + private int hash(UUID key) { + return hashFunction.newHasher() + .putLong(key.getMostSignificantBits()) + .putLong(key.getLeastSignificantBits()) + .hash().asInt(); + } + public static HashFunction forName(String name) { switch (name) { case "murmur3_32": @@ -436,4 +513,8 @@ public class HashPartitionService implements PartitionService { } } + private List toServiceIds(Collection serviceInfos) { + return serviceInfos.stream().map(ServiceInfo::getServiceId).collect(Collectors.toList()); + } + } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/PartitionService.java b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/PartitionService.java index faa4d956a8..b55ba79f67 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/PartitionService.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/PartitionService.java @@ -64,4 +64,7 @@ public interface PartitionService { void updateQueue(TransportProtos.QueueUpdateMsg queueUpdateMsg); void removeQueue(TransportProtos.QueueDeleteMsg queueDeleteMsg); + + boolean isManagedByCurrentService(TenantId tenantId); + } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/TbServiceInfoProvider.java b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/TbServiceInfoProvider.java index e49cbbcfd9..9c7d1630ec 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/TbServiceInfoProvider.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/TbServiceInfoProvider.java @@ -18,6 +18,9 @@ package org.thingsboard.server.queue.discovery; import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.gen.transport.TransportProtos.ServiceInfo; +import java.util.Set; +import java.util.UUID; + public interface TbServiceInfoProvider { String getServiceId(); @@ -30,4 +33,6 @@ public interface TbServiceInfoProvider { ServiceInfo generateNewServiceInfoWithCurrentSystemInfo(); + Set getAssignedTenantProfiles(); + } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/TenantRoutingInfo.java b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/TenantRoutingInfo.java index c1c0b49dab..8dee68da49 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/TenantRoutingInfo.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/TenantRoutingInfo.java @@ -17,9 +17,11 @@ package org.thingsboard.server.queue.discovery; import lombok.Data; import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.id.TenantProfileId; @Data public class TenantRoutingInfo { private final TenantId tenantId; + private final TenantProfileId profileId; private final boolean isolatedTbRuleEngine; } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/TenantRoutingInfoService.java b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/TenantRoutingInfoService.java index 8dd3ff95e7..e4c0ac8250 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/TenantRoutingInfoService.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/TenantRoutingInfoService.java @@ -20,4 +20,5 @@ import org.thingsboard.server.common.data.id.TenantId; public interface TenantRoutingInfoService { TenantRoutingInfo getRoutingInfo(TenantId tenantId); + } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaAdmin.java b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaAdmin.java index f15b9258e8..d486d04783 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaAdmin.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaAdmin.java @@ -21,6 +21,7 @@ import org.apache.kafka.clients.admin.CreateTopicsResult; import org.apache.kafka.clients.admin.NewTopic; import org.apache.kafka.common.errors.TopicExistsException; import org.thingsboard.server.queue.TbQueueAdmin; +import org.thingsboard.server.queue.util.PropertyUtils; import java.util.Collections; import java.util.Map; @@ -62,12 +63,12 @@ public class TbKafkaAdmin implements TbQueueAdmin { } @Override - public void createTopicIfNotExists(String topic) { + public void createTopicIfNotExists(String topic, String properties) { if (topics.contains(topic)) { return; } try { - NewTopic newTopic = new NewTopic(topic, numPartitions, replicationFactor).configs(topicConfigs); + NewTopic newTopic = new NewTopic(topic, numPartitions, replicationFactor).configs(PropertyUtils.getProps(topicConfigs, properties)); createTopic(newTopic).values().get(topic).get(); topics.add(topic); } catch (ExecutionException ee) { @@ -81,7 +82,6 @@ public class TbKafkaAdmin implements TbQueueAdmin { log.warn("[{}] Failed to create topic", topic, e); throw new RuntimeException(e); } - } @Override diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaConsumerTemplate.java b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaConsumerTemplate.java index c17a563d46..9f58446966 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaConsumerTemplate.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaConsumerTemplate.java @@ -114,7 +114,6 @@ public class TbKafkaConsumerTemplate extends AbstractTbQue @Override protected void doUnsubscribe() { - log.info("unsubscribe topic and close consumer for topic {}", getTopic()); if (consumer != null) { consumer.unsubscribe(); consumer.close(); @@ -123,4 +122,10 @@ public class TbKafkaConsumerTemplate extends AbstractTbQue statsService.unregisterClientGroup(groupId); } } + + @Override + public boolean isLongPollingSupported() { + return true; + } + } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaSettings.java b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaSettings.java index 41ae0d5ea3..55f1721c46 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaSettings.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaSettings.java @@ -115,6 +115,9 @@ public class TbKafkaSettings { @Value("${queue.kafka.session.timeout.ms:10000}") private int sessionTimeoutMs; + @Value("${queue.kafka.auto_offset_reset:earliest}") + private String autoOffsetReset; + @Value("${queue.kafka.use_confluent_cloud:false}") private boolean useConfluent; @@ -155,6 +158,8 @@ public class TbKafkaSettings { props.put(ConsumerConfig.MAX_PARTITION_FETCH_BYTES_CONFIG, maxPartitionFetchBytes); props.put(ConsumerConfig.FETCH_MAX_BYTES_CONFIG, fetchMaxBytes); props.put(ConsumerConfig.MAX_POLL_INTERVAL_MS_CONFIG, maxPollIntervalMs); + props.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, autoOffsetReset); + props.put(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG, false); props.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class); props.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, ByteArrayDeserializer.class); diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/memory/InMemoryTbQueueConsumer.java b/common/queue/src/main/java/org/thingsboard/server/queue/memory/InMemoryTbQueueConsumer.java index 081202315e..8711cbbcf1 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/memory/InMemoryTbQueueConsumer.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/memory/InMemoryTbQueueConsumer.java @@ -31,6 +31,7 @@ public class InMemoryTbQueueConsumer implements TbQueueCon private volatile Set partitions; private volatile boolean stopped; private volatile boolean subscribed; + private volatile boolean queueDeleted; public InMemoryTbQueueConsumer(InMemoryStorage storage, String topic) { this.storage = storage; @@ -103,4 +104,19 @@ public class InMemoryTbQueueConsumer implements TbQueueCon return stopped; } + @Override + public void onQueueDelete() { + queueDeleted = true; + } + + @Override + public boolean isQueueDeleted() { + return queueDeleted; + } + + @Override + public List getFullTopicNames() { + return partitions.stream().map(TopicPartitionInfo::getFullTopicName).collect(Collectors.toList()); + } + } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/InMemoryTbTransportQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/InMemoryTbTransportQueueFactory.java index 6b6253acc4..a789816c34 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/InMemoryTbTransportQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/InMemoryTbTransportQueueFactory.java @@ -74,7 +74,7 @@ public class InMemoryTbTransportQueueFactory implements TbTransportQueueFactory templateBuilder.queueAdmin(new TbQueueAdmin() { @Override - public void createTopicIfNotExists(String topic) {} + public void createTopicIfNotExists(String topic, String properties) {} @Override public void destroy() {} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaMonolithQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaMonolithQueueFactory.java index 364abcff4c..22a16de64f 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaMonolithQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaMonolithQueueFactory.java @@ -187,7 +187,7 @@ public class KafkaMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEngi consumerBuilder.settings(kafkaSettings); consumerBuilder.topic(configuration.getTopic()); consumerBuilder.clientId("re-" + queueName + "-consumer-" + serviceInfoProvider.getServiceId() + "-" + consumerCount.incrementAndGet()); - consumerBuilder.groupId("re-" + queueName + "-consumer"); + consumerBuilder.groupId("re-" + queueName + (configuration.getTenantId().isSysTenantId() ? "" : ("-" + configuration.getTenantId())) + "-consumer"); consumerBuilder.decoder(msg -> new TbProtoQueueMsg<>(msg.getKey(), ToRuleEngineMsg.parseFrom(msg.getData()), msg.getHeaders())); consumerBuilder.admin(ruleEngineAdmin); consumerBuilder.statsService(consumerStatsService); diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbRuleEngineQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbRuleEngineQueueFactory.java index b8e07a45f7..2e3bf784d7 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbRuleEngineQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbRuleEngineQueueFactory.java @@ -166,7 +166,7 @@ public class KafkaTbRuleEngineQueueFactory implements TbRuleEngineQueueFactory { consumerBuilder.settings(kafkaSettings); consumerBuilder.topic(configuration.getTopic()); consumerBuilder.clientId("re-" + queueName + "-consumer-" + serviceInfoProvider.getServiceId() + "-" + consumerCount.incrementAndGet()); - consumerBuilder.groupId("re-" + queueName + "-consumer"); + consumerBuilder.groupId("re-" + queueName + (configuration.getTenantId().isSysTenantId() ? "" : ("-" + configuration.getTenantId())) + "-consumer"); consumerBuilder.decoder(msg -> new TbProtoQueueMsg<>(msg.getKey(), ToRuleEngineMsg.parseFrom(msg.getData()), msg.getHeaders())); consumerBuilder.admin(ruleEngineAdmin); consumerBuilder.statsService(consumerStatsService); diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/pubsub/TbPubSubAdmin.java b/common/queue/src/main/java/org/thingsboard/server/queue/pubsub/TbPubSubAdmin.java index d1a4942ad3..f9f20c2448 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/pubsub/TbPubSubAdmin.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/pubsub/TbPubSubAdmin.java @@ -103,7 +103,7 @@ public class TbPubSubAdmin implements TbQueueAdmin { } @Override - public void createTopicIfNotExists(String partition) { + public void createTopicIfNotExists(String partition, String properties) { TopicName topicName = TopicName.newBuilder() .setTopic(partition) .setProject(pubSubSettings.getProjectId()) diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/rabbitmq/TbRabbitMqAdmin.java b/common/queue/src/main/java/org/thingsboard/server/queue/rabbitmq/TbRabbitMqAdmin.java index 00a2ee4c6c..fb646f383a 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/rabbitmq/TbRabbitMqAdmin.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/rabbitmq/TbRabbitMqAdmin.java @@ -18,9 +18,11 @@ package org.thingsboard.server.queue.rabbitmq; import com.rabbitmq.client.Channel; import com.rabbitmq.client.Connection; import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.StringUtils; import org.thingsboard.server.queue.TbQueueAdmin; import java.io.IOException; +import java.util.HashMap; import java.util.Map; import java.util.concurrent.TimeoutException; @@ -50,7 +52,12 @@ public class TbRabbitMqAdmin implements TbQueueAdmin { } @Override - public void createTopicIfNotExists(String topic) { + public void createTopicIfNotExists(String topic, String properties) { + Map arguments = this.arguments; + if (StringUtils.isNotBlank(properties)) { + arguments = new HashMap<>(arguments); + arguments.putAll(TbRabbitMqQueueArguments.getArgs(properties)); + } try { channel.queueDeclare(topic, false, false, false, arguments); } catch (IOException e) { diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/rabbitmq/TbRabbitMqQueueArguments.java b/common/queue/src/main/java/org/thingsboard/server/queue/rabbitmq/TbRabbitMqQueueArguments.java index cb96abdf3c..8fa8c537e6 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/rabbitmq/TbRabbitMqQueueArguments.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/rabbitmq/TbRabbitMqQueueArguments.java @@ -65,7 +65,7 @@ public class TbRabbitMqQueueArguments { vcArgs = getArgs(vcProperties); } - private Map getArgs(String properties) { + public static Map getArgs(String properties) { Map configs = new HashMap<>(); if (StringUtils.isNotEmpty(properties)) { for (String property : properties.split(";")) { @@ -78,7 +78,7 @@ public class TbRabbitMqQueueArguments { return configs; } - private Object getObjectValue(String str) { + private static Object getObjectValue(String str) { if (str.equalsIgnoreCase("true") || str.equalsIgnoreCase("false")) { return Boolean.valueOf(str); } else if (isNumeric(str)) { @@ -87,7 +87,7 @@ public class TbRabbitMqQueueArguments { return str; } - private Object getNumericValue(String str) { + private static Object getNumericValue(String str) { if (str.contains(".")) { return Double.valueOf(str); } else { @@ -97,7 +97,7 @@ public class TbRabbitMqQueueArguments { private static final Pattern PATTERN = Pattern.compile("-?\\d+(\\.\\d+)?"); - public boolean isNumeric(String strNum) { + private static boolean isNumeric(String strNum) { if (strNum == null) { return false; } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/sqs/TbAwsSqsAdmin.java b/common/queue/src/main/java/org/thingsboard/server/queue/sqs/TbAwsSqsAdmin.java index f88a34941a..ba4eeb6ca4 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/sqs/TbAwsSqsAdmin.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/sqs/TbAwsSqsAdmin.java @@ -26,6 +26,7 @@ import com.amazonaws.services.sqs.model.CreateQueueRequest; import com.amazonaws.services.sqs.model.GetQueueUrlResult; import lombok.extern.slf4j.Slf4j; import org.thingsboard.server.queue.TbQueueAdmin; +import org.thingsboard.server.queue.util.PropertyUtils; import java.util.Map; import java.util.function.Function; @@ -63,11 +64,12 @@ public class TbAwsSqsAdmin implements TbQueueAdmin { } @Override - public void createTopicIfNotExists(String topic) { + public void createTopicIfNotExists(String topic, String properties) { String queueName = convertTopicToQueueName(topic); if (queues.containsKey(queueName)) { return; } + Map attributes = PropertyUtils.getProps(this.attributes, properties, TbAwsSqsQueueAttributes::toConfigs); final CreateQueueRequest createQueueRequest = new CreateQueueRequest(queueName).withAttributes(attributes); String queueUrl = sqsClient.createQueue(createQueueRequest).getQueueUrl(); queues.put(getQueueNameFromUrl(queueUrl), queueUrl); diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/sqs/TbAwsSqsQueueAttributes.java b/common/queue/src/main/java/org/thingsboard/server/queue/sqs/TbAwsSqsQueueAttributes.java index 66110ade74..faa8eccc90 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/sqs/TbAwsSqsQueueAttributes.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/sqs/TbAwsSqsQueueAttributes.java @@ -76,6 +76,12 @@ public class TbAwsSqsQueueAttributes { private Map getConfigs(String properties) { Map configs = new HashMap<>(defaultAttributes); + configs.putAll(toConfigs(properties)); + return configs; + } + + public static Map toConfigs(String properties) { + Map configs = new HashMap<>(); if (StringUtils.isNotEmpty(properties)) { for (String property : properties.split(";")) { int delimiterPosition = property.indexOf(":"); @@ -88,7 +94,7 @@ public class TbAwsSqsQueueAttributes { return configs; } - private void validateAttributeName(String key) { + private static void validateAttributeName(String key) { QueueAttributeName.fromValue(key); } } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/util/PropertyUtils.java b/common/queue/src/main/java/org/thingsboard/server/queue/util/PropertyUtils.java index 089d7f2219..afee64f382 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/util/PropertyUtils.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/util/PropertyUtils.java @@ -19,6 +19,7 @@ import org.thingsboard.server.common.data.StringUtils; import java.util.HashMap; import java.util.Map; +import java.util.function.Function; public class PropertyUtils { @@ -37,4 +38,17 @@ public class PropertyUtils { return configs; } + public static Map getProps(Map defaultProperties, String propertiesStr) { + return getProps(defaultProperties, propertiesStr, PropertyUtils::getProps); + } + + public static Map getProps(Map defaultProperties, String propertiesStr, Function> parser) { + Map properties = defaultProperties; + if (StringUtils.isNotBlank(propertiesStr)) { + properties = new HashMap<>(properties); + properties.putAll(parser.apply(propertiesStr)); + } + return properties; + } + } diff --git a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/DefaultTransportTenantProfileCache.java b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/DefaultTransportTenantProfileCache.java index d73e445516..12f89f29e6 100644 --- a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/DefaultTransportTenantProfileCache.java +++ b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/DefaultTransportTenantProfileCache.java @@ -82,6 +82,7 @@ public class DefaultTransportTenantProfileCache implements TransportTenantProfil if (profileOpt.isPresent()) { TenantProfile newProfile = profileOpt.get(); log.trace("[{}] put: {}", newProfile.getId(), newProfile); + profiles.put(newProfile.getId(), newProfile); Set affectedTenants = tenantProfileIds.get(newProfile.getId()); return new TenantProfileUpdateResult(newProfile, affectedTenants != null ? affectedTenants : Collections.emptySet()); } else { diff --git a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/TransportTenantRoutingInfoService.java b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/TransportTenantRoutingInfoService.java index c9f126b808..e1192391d5 100644 --- a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/TransportTenantRoutingInfoService.java +++ b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/TransportTenantRoutingInfoService.java @@ -29,7 +29,7 @@ import org.thingsboard.server.queue.discovery.TenantRoutingInfoService; @ConditionalOnExpression("'${service.type:null}'=='tb-transport'") public class TransportTenantRoutingInfoService implements TenantRoutingInfoService { - private TransportTenantProfileCache tenantProfileCache; + private final TransportTenantProfileCache tenantProfileCache; public TransportTenantRoutingInfoService(TransportTenantProfileCache tenantProfileCache) { this.tenantProfileCache = tenantProfileCache; @@ -38,7 +38,7 @@ public class TransportTenantRoutingInfoService implements TenantRoutingInfoServi @Override public TenantRoutingInfo getRoutingInfo(TenantId tenantId) { TenantProfile profile = tenantProfileCache.get(tenantId); - return new TenantRoutingInfo(tenantId, profile.isIsolatedTbRuleEngine()); + return new TenantRoutingInfo(tenantId, profile.getId(), profile.isIsolatedTbRuleEngine()); } } diff --git a/dao/src/main/java/org/thingsboard/server/dao/asset/AssetProfileServiceImpl.java b/dao/src/main/java/org/thingsboard/server/dao/asset/AssetProfileServiceImpl.java index b3bc89c34d..9240bbc223 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/asset/AssetProfileServiceImpl.java +++ b/dao/src/main/java/org/thingsboard/server/dao/asset/AssetProfileServiceImpl.java @@ -111,10 +111,24 @@ public class AssetProfileServiceImpl extends AbstractCachedEntityService prefs = new EnumMap<>(NotificationType.class); + if (settings != null) { + prefs.putAll(settings.getPrefs()); + } + NotificationPref defaultPref = NotificationPref.createDefault(); + for (NotificationType notificationType : NotificationType.values()) { + NotificationPref pref = prefs.get(notificationType); + if (pref == null) { + prefs.put(notificationType, defaultPref); + } else { + var enabledDeliveryMethods = new LinkedHashMap<>(pref.getEnabledDeliveryMethods()); + // in case a new delivery method was added to the platform + UserNotificationSettings.deliveryMethods.forEach(deliveryMethod -> { + enabledDeliveryMethods.putIfAbsent(deliveryMethod, true); + }); + pref.setEnabledDeliveryMethods(enabledDeliveryMethods); + } + } + return new UserNotificationSettings(prefs); + } + @Transactional(propagation = Propagation.NOT_SUPPORTED) // so that parent transaction is not aborted on method failure @Override public void createDefaultNotificationConfigs(TenantId tenantId) { diff --git a/dao/src/main/java/org/thingsboard/server/dao/service/validator/TenantProfileDataValidator.java b/dao/src/main/java/org/thingsboard/server/dao/service/validator/TenantProfileDataValidator.java index 0477ccf32f..aa166ba12e 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/service/validator/TenantProfileDataValidator.java +++ b/dao/src/main/java/org/thingsboard/server/dao/service/validator/TenantProfileDataValidator.java @@ -99,8 +99,6 @@ public class TenantProfileDataValidator extends DataValidator { TenantProfile old = tenantProfileDao.findById(TenantId.SYS_TENANT_ID, tenantProfile.getId().getId()); if (old == null) { throw new DataValidationException("Can't update non existing tenant profile!"); - } else if (old.isIsolatedTbRuleEngine() != tenantProfile.isIsolatedTbRuleEngine()) { - throw new DataValidationException("Can't update isolatedTbRuleEngine property!"); } return old; } diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/TenantProfileServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/TenantProfileServiceTest.java index 51f8aabb57..7b5e016d14 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/TenantProfileServiceTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/TenantProfileServiceTest.java @@ -188,17 +188,6 @@ public class TenantProfileServiceTest extends AbstractServiceTest { }); } - @Test - public void testSaveSameTenantProfileWithDifferentIsolatedTbRuleEngine() { - TenantProfile tenantProfile = this.createTenantProfile("Tenant Profile"); - TenantProfile savedTenantProfile = tenantProfileService.saveTenantProfile(TenantId.SYS_TENANT_ID, tenantProfile); - savedTenantProfile.setIsolatedTbRuleEngine(true); - addMainQueueConfig(savedTenantProfile); - Assertions.assertThrows(DataValidationException.class, () -> { - tenantProfileService.saveTenantProfile(TenantId.SYS_TENANT_ID, savedTenantProfile); - }); - } - @Test public void testDeleteTenantProfileWithExistingTenant() { TenantProfile tenantProfile = this.createTenantProfile("Tenant Profile"); diff --git a/msa/vc-executor/src/main/java/org/thingsboard/server/vc/service/VersionControlTenantRoutingInfoService.java b/msa/vc-executor/src/main/java/org/thingsboard/server/vc/service/VersionControlTenantRoutingInfoService.java index fb33ff9931..b343a4791f 100644 --- a/msa/vc-executor/src/main/java/org/thingsboard/server/vc/service/VersionControlTenantRoutingInfoService.java +++ b/msa/vc-executor/src/main/java/org/thingsboard/server/vc/service/VersionControlTenantRoutingInfoService.java @@ -25,6 +25,6 @@ public class VersionControlTenantRoutingInfoService implements TenantRoutingInfo @Override public TenantRoutingInfo getRoutingInfo(TenantId tenantId) { //This dummy implementation is ok since Version Control service does not produce any rule engine messages. - return new TenantRoutingInfo(tenantId, false); + return new TenantRoutingInfo(tenantId, null, false); } } diff --git a/msa/vc-executor/src/main/resources/tb-vc-executor.yml b/msa/vc-executor/src/main/resources/tb-vc-executor.yml index 418184e732..22ad6c60f8 100644 --- a/msa/vc-executor/src/main/resources/tb-vc-executor.yml +++ b/msa/vc-executor/src/main/resources/tb-vc-executor.yml @@ -73,6 +73,7 @@ queue: fetch_max_bytes: "${TB_QUEUE_KAFKA_FETCH_MAX_BYTES:134217728}" request.timeout.ms: "${TB_QUEUE_KAFKA_REQUEST_TIMEOUT_MS:30000}" # (30 seconds) # refer to https://docs.confluent.io/platform/current/installation/configuration/producer-configs.html#producerconfigs_request.timeout.ms session.timeout.ms: "${TB_QUEUE_KAFKA_SESSION_TIMEOUT_MS:10000}" # (10 seconds) # refer to https://docs.confluent.io/platform/current/installation/configuration/consumer-configs.html#consumerconfigs_session.timeout.ms + auto_offset_reset: "${TB_QUEUE_KAFKA_AUTO_OFFSET_RESET:earliest}" # earliest, latest or none use_confluent_cloud: "${TB_QUEUE_KAFKA_USE_CONFLUENT_CLOUD:false}" confluent: ssl.algorithm: "${TB_QUEUE_KAFKA_CONFLUENT_SSL_ALGORITHM:https}" diff --git a/pom.xml b/pom.xml index df8c6ed8d6..de4baadb8a 100755 --- a/pom.xml +++ b/pom.xml @@ -78,7 +78,7 @@ 3.8.1 3.21.9 1.42.1 - 1.0.7 + 1.0.8 1.18.18 1.2.4 1.2.5 diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/math/TbMathNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/math/TbMathNode.java index fe6deebbcf..f7b4eb163c 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/math/TbMathNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/math/TbMathNode.java @@ -19,6 +19,8 @@ import com.fasterxml.jackson.databind.node.ObjectNode; import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; import com.google.common.util.concurrent.MoreExecutors; +import lombok.Data; +import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import net.objecthunter.exp4j.Expression; import net.objecthunter.exp4j.ExpressionBuilder; @@ -44,9 +46,10 @@ import java.math.BigDecimal; import java.math.RoundingMode; import java.util.List; import java.util.Optional; +import java.util.Queue; +import java.util.concurrent.ConcurrentLinkedQueue; import java.util.concurrent.ConcurrentMap; import java.util.concurrent.Semaphore; -import java.util.concurrent.TimeUnit; import java.util.function.BiFunction; import java.util.function.Function; import java.util.stream.Collectors; @@ -81,9 +84,8 @@ import static org.thingsboard.rule.engine.math.TbMathArgumentType.CONSTANT; ) public class TbMathNode implements TbNode { - private static final ConcurrentMap semaphores = new ConcurrentReferenceHashMap<>(); + private static final ConcurrentMap> locks = new ConcurrentReferenceHashMap<>(16, ConcurrentReferenceHashMap.ReferenceType.WEAK); private final ThreadLocal customExpression = new ThreadLocal<>(); - private TbMathNodeConfiguration config; private boolean msgBodyToJsonConversionRequired; @@ -108,51 +110,69 @@ public class TbMathNode implements TbNode { @Override public void onMsg(TbContext ctx, TbMsg msg) { - var originator = msg.getOriginator(); - var originatorSemaphore = semaphores.computeIfAbsent(originator, tmp -> new Semaphore(1, true)); - boolean acquired = tryAcquire(originator, originatorSemaphore); + var semaphoreWithQueue = locks.computeIfAbsent(msg.getOriginator(), SemaphoreWithQueue::new); + semaphoreWithQueue.getQueue().add(new TbMsgTbContext(msg, ctx)); - if (!acquired) { - ctx.tellFailure(msg, new RuntimeException("Failed to process message for originator synchronously")); - return; - } + tryProcessQueue(semaphoreWithQueue); + } - try { - var arguments = config.getArguments(); - Optional msgBodyOpt = convertMsgBodyIfRequired(msg); - var argumentValues = Futures.allAsList(arguments.stream() - .map(arg -> resolveArguments(ctx, msg, msgBodyOpt, arg)).collect(Collectors.toList())); - ListenableFuture resultMsgFuture = Futures.transformAsync(argumentValues, args -> - updateMsgAndDb(ctx, msg, msgBodyOpt, calculateResult(args)), ctx.getDbCallbackExecutor()); - DonAsynchron.withCallback(resultMsgFuture, resultMsg -> { - try { - ctx.tellSuccess(resultMsg); - } finally { - originatorSemaphore.release(); + void tryProcessQueue(SemaphoreWithQueue lockAndQueue) { + final Semaphore semaphore = lockAndQueue.getSemaphore(); + final Queue queue = lockAndQueue.getQueue(); + while (!queue.isEmpty()) { + // The semaphore have to be acquired before EACH poll and released before NEXT poll. + // Otherwise, some message will remain unprocessed in queue + if (!semaphore.tryAcquire()) { + return; + } + TbMsgTbContext tbMsgTbContext = null; + try { + tbMsgTbContext = queue.poll(); + if (tbMsgTbContext == null) { + semaphore.release(); + continue; } - }, t -> { - try { - ctx.tellFailure(msg, t); - } finally { - originatorSemaphore.release(); + final TbMsg msg = tbMsgTbContext.getMsg(); + if (!msg.getCallback().isMsgValid()) { + log.trace("[{}] Skipping non-valid message [{}]", lockAndQueue.getEntityId(), msg); + semaphore.release(); + continue; } - }, ctx.getDbCallbackExecutor()); - } catch (Throwable e) { - originatorSemaphore.release(); - log.warn("[{}] Failed to process message: {}", originator, msg, e); - throw e; + //DO PROCESSING + final TbContext ctx = tbMsgTbContext.getCtx(); + final ListenableFuture resultMsgFuture = processMsgAsync(ctx, msg); + DonAsynchron.withCallback(resultMsgFuture, resultMsg -> { + try { + ctx.tellSuccess(resultMsg); + } finally { + lockAndQueue.getSemaphore().release(); + tryProcessQueue(lockAndQueue); + } + }, t -> { + try { + ctx.tellFailure(msg, t); + } finally { + lockAndQueue.getSemaphore().release(); + tryProcessQueue(lockAndQueue); + } + }, ctx.getDbCallbackExecutor()); + } catch (Throwable e) { + semaphore.release(); + log.warn("[{}] Failed to process message: {}", lockAndQueue.getEntityId(), tbMsgTbContext == null ? null : tbMsgTbContext.getMsg(), e); + throw e; + } + break; //submitted async exact one task. next poll will try on callback } } - private boolean tryAcquire(EntityId originator, Semaphore originatorSemaphore) { - boolean acquired; - try { - acquired = originatorSemaphore.tryAcquire(20, TimeUnit.SECONDS); - } catch (InterruptedException e) { - acquired = false; - log.debug("[{}] Failed to acquire semaphore", originator, e); - } - return acquired; + ListenableFuture processMsgAsync(TbContext ctx, TbMsg msg) { + var arguments = config.getArguments(); + Optional msgBodyOpt = convertMsgBodyIfRequired(msg); + var argumentValues = Futures.allAsList(arguments.stream() + .map(arg -> resolveArguments(ctx, msg, msgBodyOpt, arg)).collect(Collectors.toList())); + ListenableFuture resultMsgFuture = Futures.transformAsync(argumentValues, args -> + updateMsgAndDb(ctx, msg, msgBodyOpt, calculateResult(args)), ctx.getDbCallbackExecutor()); + return resultMsgFuture; } private ListenableFuture updateMsgAndDb(TbContext ctx, TbMsg msg, Optional msgBodyOpt, double result) { @@ -402,4 +422,20 @@ public class TbMathNode implements TbNode { @Override public void destroy() { } + + @Data + @RequiredArgsConstructor + static public class SemaphoreWithQueue { + final EntityId entityId; + final Semaphore semaphore = new Semaphore(1); + final Queue queue = new ConcurrentLinkedQueue<>(); + } + + @Data + @RequiredArgsConstructor + static public class TbMsgTbContext { + final TbMsg msg; + final TbContext ctx; + } + } diff --git a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/math/TbMathNodeTest.java b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/math/TbMathNodeTest.java index 0a42349433..983bc6574c 100644 --- a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/math/TbMathNodeTest.java +++ b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/math/TbMathNodeTest.java @@ -15,20 +15,20 @@ */ package org.thingsboard.rule.engine.math; -import com.datastax.oss.driver.api.core.uuid.Uuids; import com.fasterxml.jackson.databind.node.ObjectNode; import com.google.common.util.concurrent.Futures; import lombok.extern.slf4j.Slf4j; import org.awaitility.Awaitility; -import org.junit.After; import org.junit.Assert; -import org.junit.Before; -import org.junit.Test; -import org.junit.runner.RunWith; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.ArgumentCaptor; import org.mockito.Mock; import org.mockito.Mockito; -import org.mockito.junit.MockitoJUnitRunner; +import org.mockito.junit.jupiter.MockitoExtension; +import org.mockito.verification.Timeout; import org.springframework.test.util.ReflectionTestUtils; import org.thingsboard.common.util.AbstractListeningExecutor; import org.thingsboard.common.util.JacksonUtil; @@ -54,24 +54,35 @@ import org.thingsboard.server.dao.timeseries.TimeseriesService; import java.util.Arrays; import java.util.List; import java.util.Optional; +import java.util.UUID; import java.util.concurrent.ConcurrentMap; -import java.util.concurrent.Semaphore; +import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; +import java.util.stream.IntStream; +import static org.assertj.core.api.Assertions.assertThat; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyDouble; import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.ArgumentMatchers.argThat; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.BDDMockito.willAnswer; import static org.mockito.Mockito.lenient; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.spy; import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; @Slf4j -@RunWith(MockitoJUnitRunner.class) +@ExtendWith(MockitoExtension.class) public class TbMathNodeTest { - private EntityId originator = new DeviceId(Uuids.timeBased()); - private TenantId tenantId = TenantId.fromUUID(Uuids.timeBased()); + static final int RULE_DISPATCHER_POOL_SIZE = 2; + static final int DB_CALLBACK_POOL_SIZE = 3; + private final EntityId originator = DeviceId.fromString("ccd71696-0586-422d-940e-755a41ec3b0d"); + private final TenantId tenantId = TenantId.fromUUID(UUID.fromString("e7f46b23-0c7d-42f5-9b06-fc35ab17af8a")); @Mock private TbContext ctx; @@ -81,35 +92,31 @@ public class TbMathNodeTest { private TimeseriesService tsService; @Mock private RuleEngineTelemetryService telemetryService; - private AbstractListeningExecutor dbExecutor; + private AbstractListeningExecutor dbCallbackExecutor; + private AbstractListeningExecutor ruleEngineDispatcherExecutor; - @Before + @BeforeEach public void before() { - dbExecutor = new AbstractListeningExecutor() { - @Override - protected int getThreadPollSize() { - return 3; - } - }; - dbExecutor.init(); - initMocks(); + dbCallbackExecutor = new DBCallbackExecutor(); + dbCallbackExecutor.init(); + ruleEngineDispatcherExecutor = new RuleDispatcherExecutor(); + ruleEngineDispatcherExecutor.init(); + + lenient().when(ctx.getAttributesService()).thenReturn(attributesService); + lenient().when(ctx.getTelemetryService()).thenReturn(telemetryService); + lenient().when(ctx.getTimeseriesService()).thenReturn(tsService); + lenient().when(ctx.getTenantId()).thenReturn(tenantId); + lenient().when(ctx.getDbCallbackExecutor()).thenReturn(dbCallbackExecutor); } - @After + @AfterEach public void after() { - dbExecutor.destroy(); + ruleEngineDispatcherExecutor.executor().shutdownNow(); + dbCallbackExecutor.executor().shutdownNow(); } private void initMocks() { - Mockito.reset(ctx); - Mockito.reset(attributesService); - Mockito.reset(tsService); - Mockito.reset(telemetryService); - lenient().when(ctx.getAttributesService()).thenReturn(attributesService); - lenient().when(ctx.getTelemetryService()).thenReturn(telemetryService); - lenient().when(ctx.getTimeseriesService()).thenReturn(tsService); - lenient().when(ctx.getTenantId()).thenReturn(tenantId); - lenient().when(ctx.getDbCallbackExecutor()).thenReturn(dbExecutor); + Mockito.clearInvocations(ctx, attributesService, tsService, telemetryService); } private TbMathNode initNode(TbRuleNodeMathFunctionType operation, TbMathResult result, TbMathArgument... arguments) { @@ -154,10 +161,8 @@ public class TbMathNodeTest { node.onMsg(ctx, msg); - ConcurrentMap semaphores = (ConcurrentMap) ReflectionTestUtils.getField(node, "semaphores"); + ConcurrentMap> semaphores = (ConcurrentMap>) ReflectionTestUtils.getField(node, "locks"); Assert.assertNotNull(semaphores); - Semaphore originatorSemaphore = semaphores.get(originator); - Assert.assertNotNull(originatorSemaphore); metaData.putValue("key1", "secondMsgResult"); metaData.putValue("key2", "argumentC"); @@ -167,7 +172,7 @@ public class TbMathNodeTest { node.onMsg(ctx, msg); - Awaitility.await("Semaphore released").atMost(5, TimeUnit.SECONDS).until(semaphores.get(originator)::tryAcquire); + Awaitility.await("Semaphore released").atMost(5, TimeUnit.SECONDS).until(() -> semaphores.get(originator).semaphore.tryAcquire()); ArgumentCaptor msgCaptor = ArgumentCaptor.forClass(TbMsg.class); Mockito.verify(ctx, Mockito.times(2)).tellSuccess(msgCaptor.capture()); @@ -534,4 +539,87 @@ public class TbMathNodeTest { }); Assert.assertNotNull(thrown.getMessage()); } + + @Test + public void testExp4j_concurrent() { + TbMathNode node = spy(initNodeWithCustomFunction("2a+3b", + new TbMathResult(TbMathArgumentType.MESSAGE_BODY, "result", 2, false, false, null), + new TbMathArgument(TbMathArgumentType.MESSAGE_BODY, "a"), + new TbMathArgument(TbMathArgumentType.MESSAGE_BODY, "b") + )); + EntityId originatorSlow = DeviceId.fromString("7f01170d-6bba-419c-b95c-2b4c3ba32f30"); + EntityId originatorFast = DeviceId.fromString("c45360ff-7906-4102-a2ae-3495a86168d0"); + CountDownLatch slowProcessingLatch = new CountDownLatch(1); + + List slowMsgList = IntStream.range(0, 5) + .mapToObj(x -> TbMsg.newMsg("TEST", originatorSlow, new TbMsgMetaData(), JacksonUtil.newObjectNode().put("a", 2).put("b", 2).toString())) + .collect(Collectors.toList()); + List fastMsgList = IntStream.range(0, 2) + .mapToObj(x -> TbMsg.newMsg("TEST", originatorFast, new TbMsgMetaData(), JacksonUtil.newObjectNode().put("a", 2).put("b", 2).toString())) + .collect(Collectors.toList()); + + assertThat(slowMsgList.size()).as("slow msgs >= rule-dispatcher pool size").isGreaterThanOrEqualTo(RULE_DISPATCHER_POOL_SIZE); + + log.debug("rule-dispatcher [{}], db-callback [{}], slowMsg [{}], fastMsg [{}]", RULE_DISPATCHER_POOL_SIZE, DB_CALLBACK_POOL_SIZE, slowMsgList.size(), fastMsgList.size()); + + willAnswer(invocation -> { + TbMsg msg = invocation.getArgument(1); + log.debug("\uD83D\uDC0C processMsgAsync slow originator [{}][{}]", msg.getOriginator(), msg); + try { + assertThat(slowProcessingLatch.await(30, TimeUnit.SECONDS)).as("await on slowProcessingLatch").isTrue(); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + return invocation.callRealMethod(); + }).given(node).processMsgAsync(eq(ctx), argThat(slowMsgList::contains)); + + willAnswer(invocation -> { + TbMsg msg = invocation.getArgument(1); + log.debug("\u26A1\uFE0F processMsgAsync FAST originator [{}][{}]", msg.getOriginator(), msg); + return invocation.callRealMethod(); + }).given(node).processMsgAsync(eq(ctx), argThat(fastMsgList::contains)); + + willAnswer(invocation -> { + TbMsg msg = invocation.getArgument(1); + log.debug("submit slow originator onMsg [{}][{}]", msg.getOriginator(), msg); + return invocation.callRealMethod(); + }).given(node).onMsg(eq(ctx), argThat(slowMsgList::contains)); + + willAnswer(invocation -> { + TbMsg msg = invocation.getArgument(1); + log.debug("submit FAST originator onMsg [{}][{}]", msg.getOriginator(), msg); + return invocation.callRealMethod(); + }).given(node).onMsg(eq(ctx), argThat(fastMsgList::contains)); + + // submit slow msg may block all rule engine dispatcher threads + slowMsgList.forEach(msg -> ruleEngineDispatcherExecutor.executeAsync(() -> node.onMsg(ctx, msg))); + // wait until dispatcher threads started with all slowMsg + verify(node, new Timeout(TimeUnit.SECONDS.toMillis(5), times(slowMsgList.size()))).onMsg(eq(ctx), argThat(slowMsgList::contains)); + + // submit fast have to return immediately + fastMsgList.forEach(msg -> ruleEngineDispatcherExecutor.executeAsync(() -> node.onMsg(ctx, msg))); + // wait until all fast messages processed + verify(ctx, new Timeout(TimeUnit.SECONDS.toMillis(5), times(fastMsgList.size()))).tellSuccess(any()); + + slowProcessingLatch.countDown(); + + verify(ctx, new Timeout(TimeUnit.SECONDS.toMillis(5), times(fastMsgList.size() + slowMsgList.size()))).tellSuccess(any()); + + verify(ctx, never()).tellFailure(any(), any()); + } + + static class RuleDispatcherExecutor extends AbstractListeningExecutor { + @Override + protected int getThreadPollSize() { + return RULE_DISPATCHER_POOL_SIZE; + } + } + + static class DBCallbackExecutor extends AbstractListeningExecutor { + @Override + protected int getThreadPollSize() { + return DB_CALLBACK_POOL_SIZE; + } + } + } diff --git a/transport/coap/src/main/resources/tb-coap-transport.yml b/transport/coap/src/main/resources/tb-coap-transport.yml index bbded8aec9..082b6c0536 100644 --- a/transport/coap/src/main/resources/tb-coap-transport.yml +++ b/transport/coap/src/main/resources/tb-coap-transport.yml @@ -184,6 +184,7 @@ queue: fetch_max_bytes: "${TB_QUEUE_KAFKA_FETCH_MAX_BYTES:134217728}" request.timeout.ms: "${TB_QUEUE_KAFKA_REQUEST_TIMEOUT_MS:30000}" # (30 seconds) # refer to https://docs.confluent.io/platform/current/installation/configuration/producer-configs.html#producerconfigs_request.timeout.ms session.timeout.ms: "${TB_QUEUE_KAFKA_SESSION_TIMEOUT_MS:10000}" # (10 seconds) # refer to https://docs.confluent.io/platform/current/installation/configuration/consumer-configs.html#consumerconfigs_session.timeout.ms + auto_offset_reset: "${TB_QUEUE_KAFKA_AUTO_OFFSET_RESET:earliest}" # earliest, latest or none use_confluent_cloud: "${TB_QUEUE_KAFKA_USE_CONFLUENT_CLOUD:false}" confluent: ssl.algorithm: "${TB_QUEUE_KAFKA_CONFLUENT_SSL_ALGORITHM:https}" diff --git a/transport/http/src/main/resources/tb-http-transport.yml b/transport/http/src/main/resources/tb-http-transport.yml index 396d95e63c..6c5ddb9c52 100644 --- a/transport/http/src/main/resources/tb-http-transport.yml +++ b/transport/http/src/main/resources/tb-http-transport.yml @@ -169,6 +169,7 @@ queue: fetch_max_bytes: "${TB_QUEUE_KAFKA_FETCH_MAX_BYTES:134217728}" request.timeout.ms: "${TB_QUEUE_KAFKA_REQUEST_TIMEOUT_MS:30000}" # (30 seconds) # refer to https://docs.confluent.io/platform/current/installation/configuration/producer-configs.html#producerconfigs_request.timeout.ms session.timeout.ms: "${TB_QUEUE_KAFKA_SESSION_TIMEOUT_MS:10000}" # (10 seconds) # refer to https://docs.confluent.io/platform/current/installation/configuration/consumer-configs.html#consumerconfigs_session.timeout.ms + auto_offset_reset: "${TB_QUEUE_KAFKA_AUTO_OFFSET_RESET:earliest}" # earliest, latest or none use_confluent_cloud: "${TB_QUEUE_KAFKA_USE_CONFLUENT_CLOUD:false}" confluent: ssl.algorithm: "${TB_QUEUE_KAFKA_CONFLUENT_SSL_ALGORITHM:https}" diff --git a/transport/lwm2m/src/main/resources/tb-lwm2m-transport.yml b/transport/lwm2m/src/main/resources/tb-lwm2m-transport.yml index 7c0cd950ff..4c0a33929a 100644 --- a/transport/lwm2m/src/main/resources/tb-lwm2m-transport.yml +++ b/transport/lwm2m/src/main/resources/tb-lwm2m-transport.yml @@ -250,6 +250,7 @@ queue: fetch_max_bytes: "${TB_QUEUE_KAFKA_FETCH_MAX_BYTES:134217728}" request.timeout.ms: "${TB_QUEUE_KAFKA_REQUEST_TIMEOUT_MS:30000}" # (30 seconds) # refer to https://docs.confluent.io/platform/current/installation/configuration/producer-configs.html#producerconfigs_request.timeout.ms session.timeout.ms: "${TB_QUEUE_KAFKA_SESSION_TIMEOUT_MS:10000}" # (10 seconds) # refer to https://docs.confluent.io/platform/current/installation/configuration/consumer-configs.html#consumerconfigs_session.timeout.ms + auto_offset_reset: "${TB_QUEUE_KAFKA_AUTO_OFFSET_RESET:earliest}" # earliest, latest or none use_confluent_cloud: "${TB_QUEUE_KAFKA_USE_CONFLUENT_CLOUD:false}" confluent: ssl.algorithm: "${TB_QUEUE_KAFKA_CONFLUENT_SSL_ALGORITHM:https}" diff --git a/transport/mqtt/src/main/resources/tb-mqtt-transport.yml b/transport/mqtt/src/main/resources/tb-mqtt-transport.yml index e6f2b0af60..67986e4bfc 100644 --- a/transport/mqtt/src/main/resources/tb-mqtt-transport.yml +++ b/transport/mqtt/src/main/resources/tb-mqtt-transport.yml @@ -199,6 +199,7 @@ queue: fetch_max_bytes: "${TB_QUEUE_KAFKA_FETCH_MAX_BYTES:134217728}" request.timeout.ms: "${TB_QUEUE_KAFKA_REQUEST_TIMEOUT_MS:30000}" # (30 seconds) # refer to https://docs.confluent.io/platform/current/installation/configuration/producer-configs.html#producerconfigs_request.timeout.ms session.timeout.ms: "${TB_QUEUE_KAFKA_SESSION_TIMEOUT_MS:10000}" # (10 seconds) # refer to https://docs.confluent.io/platform/current/installation/configuration/consumer-configs.html#consumerconfigs_session.timeout.ms + auto_offset_reset: "${TB_QUEUE_KAFKA_AUTO_OFFSET_RESET:earliest}" # earliest, latest or none use_confluent_cloud: "${TB_QUEUE_KAFKA_USE_CONFLUENT_CLOUD:false}" confluent: ssl.algorithm: "${TB_QUEUE_KAFKA_CONFLUENT_SSL_ALGORITHM:https}" diff --git a/transport/snmp/src/main/resources/tb-snmp-transport.yml b/transport/snmp/src/main/resources/tb-snmp-transport.yml index 3fc11bbdfe..84fdf5ddbb 100644 --- a/transport/snmp/src/main/resources/tb-snmp-transport.yml +++ b/transport/snmp/src/main/resources/tb-snmp-transport.yml @@ -145,6 +145,7 @@ queue: fetch_max_bytes: "${TB_QUEUE_KAFKA_FETCH_MAX_BYTES:134217728}" request.timeout.ms: "${TB_QUEUE_KAFKA_REQUEST_TIMEOUT_MS:30000}" # (30 seconds) # refer to https://docs.confluent.io/platform/current/installation/configuration/producer-configs.html#producerconfigs_request.timeout.ms session.timeout.ms: "${TB_QUEUE_KAFKA_SESSION_TIMEOUT_MS:10000}" # (10 seconds) # refer to https://docs.confluent.io/platform/current/installation/configuration/consumer-configs.html#consumerconfigs_session.timeout.ms + auto_offset_reset: "${TB_QUEUE_KAFKA_AUTO_OFFSET_RESET:earliest}" # earliest, latest or none use_confluent_cloud: "${TB_QUEUE_KAFKA_USE_CONFLUENT_CLOUD:false}" confluent: ssl.algorithm: "${TB_QUEUE_KAFKA_CONFLUENT_SSL_ALGORITHM:https}" diff --git a/ui-ngx/src/app/core/http/notification.service.ts b/ui-ngx/src/app/core/http/notification.service.ts index ec64621aeb..20e10c91e2 100644 --- a/ui-ngx/src/app/core/http/notification.service.ts +++ b/ui-ngx/src/app/core/http/notification.service.ts @@ -31,6 +31,7 @@ import { NotificationTarget, NotificationTemplate, NotificationType, + NotificationUserSettings, SlackChanelType, SlackConversation } from '@shared/models/notification.models'; @@ -174,4 +175,12 @@ export class NotificationService { } return this.http.get>(url, defaultHttpOptionsFromConfig(config)); } + + public getNotificationUserSettings(config?: RequestConfig): Observable { + return this.http.get(`/api/notification/settings/user`, defaultHttpOptionsFromConfig(config)); + } + + public saveNotificationUserSettings(settings: NotificationUserSettings, config?: RequestConfig): Observable { + return this.http.post('/api/notification/settings/user', settings, defaultHttpOptionsFromConfig(config)); + } } diff --git a/ui-ngx/src/app/modules/home/components/profile/queue/tenant-profile-queues.component.ts b/ui-ngx/src/app/modules/home/components/profile/queue/tenant-profile-queues.component.ts index ac128b3ec5..298b4fe575 100644 --- a/ui-ngx/src/app/modules/home/components/profile/queue/tenant-profile-queues.component.ts +++ b/ui-ngx/src/app/modules/home/components/profile/queue/tenant-profile-queues.component.ts @@ -173,7 +173,8 @@ export class TenantProfileQueuesComponent implements ControlValueAccessor, Valid }, topic: '', additionalInfo: { - description: '' + description: '', + customProperties: '' } }; this.idMap.push(queue.id); diff --git a/ui-ngx/src/app/modules/home/components/profile/tenant-profile.component.ts b/ui-ngx/src/app/modules/home/components/profile/tenant-profile.component.ts index 0257fb35af..26d6611a3c 100644 --- a/ui-ngx/src/app/modules/home/components/profile/tenant-profile.component.ts +++ b/ui-ngx/src/app/modules/home/components/profile/tenant-profile.component.ts @@ -58,9 +58,9 @@ export class TenantProfileComponent extends EntityComponent { id: guid(), consumerPerPartition: true, name: 'Main', - packProcessingTimeout: 2000, - partitions: 10, - pollInterval: 25, + packProcessingTimeout: 10000, + partitions: 1, + pollInterval: 2000, processingStrategy: { failurePercentage: 0, maxPauseBetweenRetries: 3, @@ -74,7 +74,56 @@ export class TenantProfileComponent extends EntityComponent { }, topic: 'tb_rule_engine.main', additionalInfo: { - description: '' + description: '', + customProperties: '' + } + }, + { + id: guid(), + name: 'HighPriority', + topic: 'tb_rule_engine.hp', + pollInterval: 2000, + partitions: 1, + consumerPerPartition: true, + packProcessingTimeout: 10000, + submitStrategy: { + type: 'BURST', + batchSize: 100 + }, + processingStrategy: { + type: 'RETRY_FAILED_AND_TIMED_OUT', + retries: 0, + failurePercentage: 0, + pauseBetweenRetries: 5, + maxPauseBetweenRetries: 5 + }, + additionalInfo: { + description: '', + customProperties: '' + } + }, + { + id: guid(), + name: 'SequentialByOriginator', + topic: 'tb_rule_engine.sq', + pollInterval: 2000, + partitions: 1, + consumerPerPartition: true, + packProcessingTimeout: 10000, + submitStrategy: { + type: 'SEQUENTIAL_BY_ORIGINATOR', + batchSize: 100 + }, + processingStrategy: { + type: 'RETRY_FAILED_AND_TIMED_OUT', + retries: 3, + failurePercentage: 0, + pauseBetweenRetries: 5, + maxPauseBetweenRetries: 5 + }, + additionalInfo: { + description: '', + customProperties: '' } } ]; @@ -118,9 +167,6 @@ export class TenantProfileComponent extends EntityComponent { if (this.entityForm) { if (this.isEditValue) { this.entityForm.enable({emitEvent: false}); - if (!this.isAdd) { - this.entityForm.get('isolatedTbRuleEngine').disable({emitEvent: false}); - } } else { this.entityForm.disable({emitEvent: false}); } diff --git a/ui-ngx/src/app/modules/home/components/queue/queue-form.component.html b/ui-ngx/src/app/modules/home/components/queue/queue-form.component.html index 56b20d8aa8..4845f7f25a 100644 --- a/ui-ngx/src/app/modules/home/components/queue/queue-form.component.html +++ b/ui-ngx/src/app/modules/home/components/queue/queue-form.component.html @@ -203,6 +203,11 @@ + + queue.custom-properties + + queue.custom-properties-hint + queue.description diff --git a/ui-ngx/src/app/modules/home/components/queue/queue-form.component.ts b/ui-ngx/src/app/modules/home/components/queue/queue-form.component.ts index b123cf2ad5..e4fbcd031b 100644 --- a/ui-ngx/src/app/modules/home/components/queue/queue-form.component.ts +++ b/ui-ngx/src/app/modules/home/components/queue/queue-form.component.ts @@ -117,7 +117,8 @@ export class QueueFormComponent implements ControlValueAccessor, OnInit, OnDestr }), topic: [''], additionalInfo: this.fb.group({ - description: [''] + description: [''], + customProperties: [''] }) }); this.valueChange$ = this.queueFormGroup.valueChanges.subscribe(() => { diff --git a/ui-ngx/src/app/modules/home/pages/account/account-routing.module.ts b/ui-ngx/src/app/modules/home/pages/account/account-routing.module.ts index 9fcaecddfd..e2c997f654 100644 --- a/ui-ngx/src/app/modules/home/pages/account/account-routing.module.ts +++ b/ui-ngx/src/app/modules/home/pages/account/account-routing.module.ts @@ -23,6 +23,9 @@ import { profileRoutes } from '@home/pages/profile/profile-routing.module'; import { getCurrentAuthState } from '@core/auth/auth.selectors'; import { Store } from '@ngrx/store'; import { AppState } from '@core/core.state'; +import { + notificationUserSettingsRoutes +} from '@home/pages/notification/settings/notification-settings-routing.modules'; const routes: Routes = [ { @@ -49,7 +52,8 @@ const routes: Routes = [ } }, ...profileRoutes, - ...securityRoutes + ...securityRoutes, + ...notificationUserSettingsRoutes ] } ]; diff --git a/ui-ngx/src/app/modules/home/pages/notification/notification.module.ts b/ui-ngx/src/app/modules/home/pages/notification/notification.module.ts index 5ab7bfa957..389bf358d6 100644 --- a/ui-ngx/src/app/modules/home/pages/notification/notification.module.ts +++ b/ui-ngx/src/app/modules/home/pages/notification/notification.module.ts @@ -35,6 +35,10 @@ import { EscalationFormComponent } from '@home/pages/notification/rule/escalatio import { EscalationsComponent } from '@home/pages/notification/rule/escalations.component'; import { RuleNotificationDialogComponent } from '@home/pages/notification/rule/rule-notification-dialog.component'; import { RuleTableHeaderComponent } from '@home/pages/notification/rule/rule-table-header.component'; +import { NotificationSettingsComponent } from '@home/pages/notification/settings/notification-settings.component'; +import { + NotificationSettingFormComponent +} from '@home/pages/notification/settings/notification-setting-form.component'; @NgModule({ declarations: [ @@ -49,7 +53,9 @@ import { RuleTableHeaderComponent } from '@home/pages/notification/rule/rule-tab EscalationFormComponent, EscalationsComponent, RuleNotificationDialogComponent, - RuleTableHeaderComponent + RuleTableHeaderComponent, + NotificationSettingsComponent, + NotificationSettingFormComponent ], imports: [ CommonModule, diff --git a/ui-ngx/src/app/modules/home/pages/notification/settings/notification-setting-form.component.html b/ui-ngx/src/app/modules/home/pages/notification/settings/notification-setting-form.component.html new file mode 100644 index 0000000000..284cd67fa1 --- /dev/null +++ b/ui-ngx/src/app/modules/home/pages/notification/settings/notification-setting-form.component.html @@ -0,0 +1,40 @@ + +
+
+
+ + + {{notificationTemplateTypeTranslateMap.get(notificationSettingsFormGroup.get('name').value)?.name | translate}} + + +
+
+
+ +
+
+
+
diff --git a/ui-ngx/src/app/modules/home/pages/notification/settings/notification-setting-form.component.scss b/ui-ngx/src/app/modules/home/pages/notification/settings/notification-setting-form.component.scss new file mode 100644 index 0000000000..ee37181a09 --- /dev/null +++ b/ui-ngx/src/app/modules/home/pages/notification/settings/notification-setting-form.component.scss @@ -0,0 +1,24 @@ +/** + * Copyright © 2016-2023 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +:host { + .notification-type { + font-size: 14px; + + &-disabled { + color: rgba(0, 0, 0, 0.38) + } + } +} diff --git a/ui-ngx/src/app/modules/home/pages/notification/settings/notification-setting-form.component.ts b/ui-ngx/src/app/modules/home/pages/notification/settings/notification-setting-form.component.ts new file mode 100644 index 0000000000..76e9babf00 --- /dev/null +++ b/ui-ngx/src/app/modules/home/pages/notification/settings/notification-setting-form.component.ts @@ -0,0 +1,126 @@ +/// +/// Copyright © 2016-2023 The Thingsboard Authors +/// +/// Licensed under the Apache License, Version 2.0 (the "License"); +/// you may not use this file except in compliance with the License. +/// You may obtain a copy of the License at +/// +/// http://www.apache.org/licenses/LICENSE-2.0 +/// +/// Unless required by applicable law or agreed to in writing, software +/// distributed under the License is distributed on an "AS IS" BASIS, +/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +/// See the License for the specific language governing permissions and +/// limitations under the License. +/// + +import { Component, forwardRef, Input, OnDestroy, OnInit } from '@angular/core'; +import { ControlValueAccessor, NG_VALUE_ACCESSOR, UntypedFormBuilder, UntypedFormGroup } from '@angular/forms'; +import { UtilsService } from '@core/services/utils.service'; +import { isDefinedAndNotNull } from '@core/utils'; +import { Subscription } from 'rxjs'; +import { + NotificationDeliveryMethod, + NotificationTemplateTypeTranslateMap, + NotificationUserSetting +} from '@shared/models/notification.models'; + +@Component({ + selector: 'tb-notification-setting-form', + templateUrl: './notification-setting-form.component.html', + styleUrls: ['./notification-setting-form.component.scss'], + providers: [ + { + provide: NG_VALUE_ACCESSOR, + useExisting: forwardRef(() => NotificationSettingFormComponent), + multi: true + } + ] +}) +export class NotificationSettingFormComponent implements ControlValueAccessor, OnInit, OnDestroy { + + @Input() + disabled: boolean; + + @Input() + deliveryMethods: NotificationDeliveryMethod[] = []; + + @Input() + allowDeliveryMethods: NotificationDeliveryMethod[] = []; + + notificationSettingsFormGroup: UntypedFormGroup; + + notificationTemplateTypeTranslateMap = NotificationTemplateTypeTranslateMap; + + private propagateChange = null; + + private valueChange$: Subscription = null; + + constructor(private utils: UtilsService, + private fb: UntypedFormBuilder) { + } + + registerOnChange(fn: any): void { + this.propagateChange = fn; + } + + registerOnTouched(fn: any): void { + } + + ngOnInit() { + const deliveryMethod = {}; + this.deliveryMethods.forEach(value => { + deliveryMethod[value] = true; + }); + this.notificationSettingsFormGroup = this.fb.group( + { + name: [''], + enabled: [true], + enabledDeliveryMethods: this.fb.group({ + ...deliveryMethod + }) + }); + this.valueChange$ = this.notificationSettingsFormGroup.valueChanges.subscribe(() => { + this.updateModel(); + }); + } + + ngOnDestroy() { + if (this.valueChange$) { + this.valueChange$.unsubscribe(); + this.valueChange$ = null; + } + } + + setDisabledState(isDisabled: boolean): void { + this.disabled = isDisabled; + if (this.disabled) { + this.notificationSettingsFormGroup.disable({emitEvent: false}); + } else { + this.notificationSettingsFormGroup.enable({emitEvent: false}); + } + } + + toggleEnabled() { + this.notificationSettingsFormGroup.get('enabled').patchValue(!this.notificationSettingsFormGroup.get('enabled').value); + } + + getChecked(deliveryMethod: NotificationDeliveryMethod): boolean { + return this.notificationSettingsFormGroup.get('enabledDeliveryMethods').get(deliveryMethod).value; + } + + toggleDeliviryMethod(deliveryMethod: NotificationDeliveryMethod) { + this.notificationSettingsFormGroup.get('enabledDeliveryMethods').get(deliveryMethod) + .patchValue(!this.notificationSettingsFormGroup.get('enabledDeliveryMethods').get(deliveryMethod).value); + } + + writeValue(value: NotificationUserSetting): void { + if (isDefinedAndNotNull(value)) { + this.notificationSettingsFormGroup.patchValue(value, {emitEvent: false}); + } + } + + private updateModel() { + this.propagateChange(this.notificationSettingsFormGroup.value); + } +} diff --git a/ui-ngx/src/app/modules/home/pages/notification/settings/notification-settings-routing.modules.ts b/ui-ngx/src/app/modules/home/pages/notification/settings/notification-settings-routing.modules.ts new file mode 100644 index 0000000000..6a65e554bf --- /dev/null +++ b/ui-ngx/src/app/modules/home/pages/notification/settings/notification-settings-routing.modules.ts @@ -0,0 +1,41 @@ +/// +/// Copyright © 2016-2023 The Thingsboard Authors +/// +/// Licensed under the Apache License, Version 2.0 (the "License"); +/// you may not use this file except in compliance with the License. +/// You may obtain a copy of the License at +/// +/// http://www.apache.org/licenses/LICENSE-2.0 +/// +/// Unless required by applicable law or agreed to in writing, software +/// distributed under the License is distributed on an "AS IS" BASIS, +/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +/// See the License for the specific language governing permissions and +/// limitations under the License. +/// + +import { Routes } from '@angular/router'; +import { ConfirmOnExitGuard } from '@core/guards/confirm-on-exit.guard'; +import { Authority } from '@shared/models/authority.enum'; +import { inject, NgModule } from '@angular/core'; +import { NotificationSettingsComponent } from '@home/pages/notification/settings/notification-settings.component'; +import { NotificationService } from '@core/http/notification.service'; + +export const notificationUserSettingsRoutes: Routes = [ + { + path: 'notificationSettings', + component: NotificationSettingsComponent, + canDeactivate: [ConfirmOnExitGuard], + data: { + auth: [Authority.SYS_ADMIN, Authority.TENANT_ADMIN, Authority.CUSTOMER_USER], + title: 'account.notification-settings', + breadcrumb: { + label: 'account.notification-settings', + icon: 'settings' + } + }, + resolve: { + userSettings: () => inject(NotificationService).getNotificationUserSettings() + } + } +]; diff --git a/ui-ngx/src/app/modules/home/pages/notification/settings/notification-settings.component.html b/ui-ngx/src/app/modules/home/pages/notification/settings/notification-settings.component.html new file mode 100644 index 0000000000..c531f10d78 --- /dev/null +++ b/ui-ngx/src/app/modules/home/pages/notification/settings/notification-settings.component.html @@ -0,0 +1,84 @@ + +
+ + +
+
+ notification.settings.notification-settings +
+
+ +
+
+ + + +
+ +
+
+
+
+
+ + notification.settings.type + +
+
+
+ + {{ notificationDeliveryMethodTranslateMap.get(deliveryMethods) | translate }} + +
+
+
+ +
+ + + +
+
+
+
+ +
+ +
+ +
diff --git a/ui-ngx/src/app/modules/home/pages/notification/settings/notification-settings.component.scss b/ui-ngx/src/app/modules/home/pages/notification/settings/notification-settings.component.scss new file mode 100644 index 0000000000..a91c57ed65 --- /dev/null +++ b/ui-ngx/src/app/modules/home/pages/notification/settings/notification-settings.component.scss @@ -0,0 +1,41 @@ +/** + * Copyright © 2016-2023 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +@import "../../../../../../scss/constants"; + +:host { + .mat-mdc-card.settings-card { + margin: 8px; + @media #{$mat-gt-sm} { + width: 60%; + } + .mat-headline-5 { + margin: 0; + } + .notification-form { + height: 100%; + min-height: min-content; + max-height: min-content; + } + .notification-section { + height: 100%; + border: 1px solid rgba(0, 0, 0, 0.12); + overflow: scroll; + &-block { + min-width: 470px; + } + } + } +} diff --git a/ui-ngx/src/app/modules/home/pages/notification/settings/notification-settings.component.ts b/ui-ngx/src/app/modules/home/pages/notification/settings/notification-settings.component.ts new file mode 100644 index 0000000000..c28e030eec --- /dev/null +++ b/ui-ngx/src/app/modules/home/pages/notification/settings/notification-settings.component.ts @@ -0,0 +1,188 @@ +/// +/// Copyright © 2016-2023 The Thingsboard Authors +/// +/// Licensed under the Apache License, Version 2.0 (the "License"); +/// you may not use this file except in compliance with the License. +/// You may obtain a copy of the License at +/// +/// http://www.apache.org/licenses/LICENSE-2.0 +/// +/// Unless required by applicable law or agreed to in writing, software +/// distributed under the License is distributed on an "AS IS" BASIS, +/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +/// See the License for the specific language governing permissions and +/// limitations under the License. +/// + +import { Component, OnInit } from '@angular/core'; +import { PageComponent } from '@shared/components/page.component'; +import { Store } from '@ngrx/store'; +import { AppState } from '@core/core.state'; +import { AbstractControl, UntypedFormArray, UntypedFormBuilder, UntypedFormGroup, Validators } from '@angular/forms'; +import { HasConfirmForm } from '@core/guards/confirm-on-exit.guard'; +import { TranslateService } from '@ngx-translate/core'; +import { ActivatedRoute } from '@angular/router'; +import { deepClone, isDefinedAndNotNull } from '@core/utils'; +import { + NotificationDeliveryMethod, + NotificationDeliveryMethodTranslateMap, + NotificationUserSettings +} from '@shared/models/notification.models'; +import { NotificationService } from '@core/http/notification.service'; +import { DialogService } from '@core/services/dialog.service'; + +@Component({ + selector: 'tb-notification-settings', + templateUrl: './notification-settings.component.html', + styleUrls: ['./notification-settings.component.scss'] +}) +export class NotificationSettingsComponent extends PageComponent implements OnInit, HasConfirmForm { + + notificationSettings: UntypedFormGroup; + + notificationDeliveryMethods: NotificationDeliveryMethod[]; + notificationDeliveryMethodTranslateMap = NotificationDeliveryMethodTranslateMap; + + allowNotificationDeliveryMethods: Array; + + constructor(protected store: Store, + private route: ActivatedRoute, + private translate: TranslateService, + private dialogService: DialogService, + private notificationService: NotificationService, + private fb: UntypedFormBuilder,) { + super(store); + } + + ngOnInit() { + this.notificationDeliveryMethods = this.getNotificationDeliveryMethods(); + + this.notificationService.getAvailableDeliveryMethods({ignoreLoading: true}).subscribe(allowMethods => { + this.allowNotificationDeliveryMethods = allowMethods; + }); + + this.buildNotificationSettingsForm(); + this.patchNotificationSettings(this.route.snapshot.data.userSettings); + } + + private getNotificationDeliveryMethods(): NotificationDeliveryMethod[] { + const deliveryMethods = new Set([ + NotificationDeliveryMethod.SLACK + ]); + return Object.values(NotificationDeliveryMethod).filter(type => !deliveryMethods.has(type)); + } + + private buildNotificationSettingsForm() { + this.notificationSettings = this.fb.group({ + prefs: this.fb.array([]) + }); + } + + private patchNotificationSettings(settings: NotificationUserSettings) { + const notificationSettingsControls: Array = []; + let preparedSettings; + if (settings.prefs) { + preparedSettings = this.prepareNotificationSettings(settings.prefs); + preparedSettings.forEach((setting) => { + setting.enabledDeliveryMethods = Object.assign( + this.notificationDeliveryMethods.reduce((a, v) => ({ ...a, [v]: true}), {}), + setting.enabledDeliveryMethods + ); + notificationSettingsControls.push(this.fb.control(setting, [Validators.required])); + }); + } + this.notificationSettings.setControl('prefs', this.fb.array(notificationSettingsControls), {emitEvent: false}); + } + + private prepareNotificationSettings(prefs: any) { + return Object.entries(prefs).map((value: any) => { + value[1].name = value[0]; + return value[1]; + }); + } + + resetSettings() { + this.dialogService.confirm( + this.translate.instant('notification.settings.reset-all-title'), + this.translate.instant('notification.settings.reset-all-text'), + this.translate.instant('action.no'), + this.translate.instant('action.yes'), + true + ).subscribe( + result => { + if (result) { + const settings = this.prepareNotificationSettings(this.route.snapshot.data.userSettings.prefs); + const notificationSettingsControls: Array = []; + this.notificationSettings.reset({}); + if (settings) { + settings.forEach((setting) => { + setting.enabled = true; + setting.enabledDeliveryMethods = this.notificationDeliveryMethods.reduce((a, v) => ({ ...a, [v]: true}), {}); + notificationSettingsControls.push(this.fb.control(setting, [Validators.required])); + }); + } + this.notificationSettings.setControl('prefs', this.fb.array(notificationSettingsControls), {emitEvent: false}); + this.save(); + } + } + ); + } + + getChecked = (method: NotificationDeliveryMethod = null): boolean => { + const type = this.notificationSettings.get('prefs').value; + if (isDefinedAndNotNull(method)) { + return isDefinedAndNotNull(type) && type.every(resource => resource.enabledDeliveryMethods[method]); + } + return isDefinedAndNotNull(type) && type.every(resource => resource.enabled); + }; + + getSomeChecked = () => { + const type = this.notificationSettings.get('prefs').value; + return isDefinedAndNotNull(type) && type.some(resource => resource.enabled); + }; + + getIndeterminate = (deliveryMethod: NotificationDeliveryMethod = null): boolean => { + const type = this.notificationSettings.get('prefs').value; + if (isDefinedAndNotNull(type)) { + const checkedResource = isDefinedAndNotNull(deliveryMethod) ? + type.filter(resource => resource.enabledDeliveryMethods[deliveryMethod]) : + type.filter(resource => resource.enabled); + return checkedResource.length !== 0 && checkedResource.length !== type.length; + } + return false; + }; + + changeInstanceTypeCheckBox = (value: boolean, deliveryMethod: NotificationDeliveryMethod = null): void => { + const type = deepClone(this.notificationSettings.get('prefs').value); + if (isDefinedAndNotNull(deliveryMethod)) { + type.forEach(notificationType => notificationType.enabledDeliveryMethods[deliveryMethod] = value); + } else { + type.forEach(notificationType => notificationType.enabled = value); + } + this.notificationSettings.get('prefs').patchValue(type); + this.notificationSettings.markAsDirty(); + }; + + get notificationSettingsFormArray(): UntypedFormArray { + return this.notificationSettings.get('prefs') as UntypedFormArray; + } + + save(): void { + const settings = {prefs: {}}; + this.notificationSettings.getRawValue().prefs.forEach(value => { + const key = value.name; + delete value.name; + settings.prefs[key] = value; + }); + this.notificationService.saveNotificationUserSettings(settings).subscribe( + (userSettings) => { + this.notificationSettings.get('prefs').reset({}); + this.patchNotificationSettings(userSettings); + } + ); + } + + confirmForm(): UntypedFormGroup { + return this.notificationSettings; + } +} diff --git a/ui-ngx/src/app/shared/models/ace/tbel/worker-tbel.js b/ui-ngx/src/app/shared/models/ace/tbel/worker-tbel.js index d1d47d0c75..ec4133da2a 100644 --- a/ui-ngx/src/app/shared/models/ace/tbel/worker-tbel.js +++ b/ui-ngx/src/app/shared/models/ace/tbel/worker-tbel.js @@ -4656,10 +4656,12 @@ var JSHINT = (function() { case "else": case "finally": case "for": + case "foreach": case "if": case "in": case "instanceof": case "return": + case "until": case "switch": case "throw": case "try": @@ -5387,6 +5389,7 @@ var JSHINT = (function() { } switch (state.funct["(verb)"]) { case "break": + case "until": case "continue": case "return": case "throw": @@ -7691,6 +7694,20 @@ var JSHINT = (function() { return this; }).labelled = true; + blockstmt("until", function(context) { + var t = state.tokens.next; + state.funct["(breakage)"] += 1; + state.funct["(loopage)"] += 1; + increaseComplexityCount(); + advance("("); + checkCondAssignment(expression(context, 0)); + advance(")", t); + block(context, true, true); + state.funct["(breakage)"] -= 1; + state.funct["(loopage)"] -= 1; + return this; + }).labelled = true; + blockstmt("with", function(context) { var t = state.tokens.next; if (state.isStrict()) { @@ -7840,7 +7857,12 @@ var JSHINT = (function() { increaseComplexityCount(); this.first = block(context, true, true); - advance("while"); + var s = state.tokens.next; + if (s.value !== "while") { + advance("until"); + } else { + advance("while"); + } var t = state.tokens.next; advance("("); checkCondAssignment(expression(context, 0)); @@ -8135,7 +8157,7 @@ var JSHINT = (function() { state.funct["(breakage)"] += 1; state.funct["(loopage)"] += 1; - state.funct["(breakage)"] -= 1; + // state.funct["(breakage)"] -= 1; state.funct["(loopage)"] -= 1; } else { nolinebreak(state.tokens.curr); @@ -9755,9 +9777,9 @@ Lexer.prototype = { scanKeyword: function() { var result = /^[a-zA-Z_$][a-zA-Z0-9_$]*/.exec(this.input); var keywords = [ - "if", "in", "do", "var", "for", "new", + "if", "in", "do", "var", "for", "foreach", "new", "try", "let", "this", "else", "case", - "void", "with", "enum", "while", "break", + "void", "with", "enum", "while", "until", "break", "catch", "throw", "const", "yield", "class", "super", "return", "typeof", "delete", "switch", "export", "import", "default", diff --git a/ui-ngx/src/app/shared/models/notification.models.ts b/ui-ngx/src/app/shared/models/notification.models.ts index edafef4704..0d1a46443d 100644 --- a/ui-ngx/src/app/shared/models/notification.models.ts +++ b/ui-ngx/src/app/shared/models/notification.models.ts @@ -591,3 +591,12 @@ export const TriggerTypeTranslationMap = new Map([ [TriggerType.NEW_PLATFORM_VERSION, 'notification.trigger.new-platform-version'], [TriggerType.RATE_LIMITS, 'notification.trigger.rate-limits'], ]); + +export interface NotificationUserSettings { + prefs: {[key: string]: NotificationUserSetting}; +} + +export interface NotificationUserSetting { + enabled: boolean; + enabledDeliveryMethods: {[key: string]: boolean}; +} diff --git a/ui-ngx/src/app/shared/models/queue.models.ts b/ui-ngx/src/app/shared/models/queue.models.ts index 76ee0bf022..07a57e68c2 100644 --- a/ui-ngx/src/app/shared/models/queue.models.ts +++ b/ui-ngx/src/app/shared/models/queue.models.ts @@ -121,5 +121,6 @@ export interface QueueInfo extends BaseData { topic: string; additionalInfo: { description?: string; + customProperties?: string; }; } diff --git a/ui-ngx/src/assets/locale/locale.constant-en_US.json b/ui-ngx/src/assets/locale/locale.constant-en_US.json index 96a993170e..7f601189a0 100644 --- a/ui-ngx/src/assets/locale/locale.constant-en_US.json +++ b/ui-ngx/src/assets/locale/locale.constant-en_US.json @@ -11,7 +11,8 @@ "permission-denied-text": "You don't have permission to perform this operation!" }, "account": { - "account": "Account" + "account": "Account", + "notification-settings": "Notification settings" }, "action": { "activate": "Activate", @@ -3151,7 +3152,17 @@ "updated": "Updated", "use-template": "Use template", "view-all": "View all", - "warning": "Warning" + "warning": "Warning", + "settings": { + "notification-settings": "Notification settings", + "reset-all": "Reset all settings", + "reset-all-title": "Are you sure you want to reset form?", + "reset-all-text": "After the confirmation, the settings form will reset to the default value and save.", + "type": "Type", + "enable-all": "Enable all", + "disable-all": "Disable all", + "delivery-not-configured": "Delivery method is not configured" + } }, "ota-update": { "add": "Add package", @@ -3601,6 +3612,8 @@ "description": "Description", "description-hint": "This text will be displayed in the Queue description instead of the selected strategy", "alt-description": "Submit Strategy: {{submitStrategy}}, Processing Strategy: {{processingStrategy}}", + "custom-properties": "Custom properties", + "custom-properties-hint": "Custom queue (topic) creation properties, e.g. 'retention.ms:604800000;retention.bytes:1048576000'", "strategies": { "sequential-by-originator-label": "Sequential by originator", "sequential-by-originator-hint": "New message for e.g. device A is not submitted until previous message for device A is acknowledged", @@ -3668,8 +3681,8 @@ "tenant-required": "Tenant is required", "search": "Search tenants", "selected-tenants": "{ count, plural, =1 {1 tenant} other {# tenants} } selected", - "isolated-tb-rule-engine": "Processing in isolated ThingsBoard Rule Engine container", - "isolated-tb-rule-engine-details": "Requires separate microservice(s) per isolated Tenant" + "isolated-tb-rule-engine": "Use isolated ThingsBoard Rule Engine queues", + "isolated-tb-rule-engine-details": "Each tenant will have dedicated Rule Engine queues" }, "tenant-profile": { "tenant-profile": "Tenant profile",