Browse Source

Merge with queue API

pull/6591/head
Andrii Shvaika 4 years ago
parent
commit
62c5c73237
  1. 15
      application/src/main/data/upgrade/3.3.4/schema_update.sql
  2. 49
      application/src/main/data/upgrade/3.3.4/schema_update_device_profile.sql
  3. 14
      application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java
  4. 30
      application/src/main/java/org/thingsboard/server/actors/app/AppActor.java
  5. 75
      application/src/main/java/org/thingsboard/server/actors/ruleChain/DefaultTbContext.java
  6. 2
      application/src/main/java/org/thingsboard/server/actors/ruleChain/RuleChainActorMessageProcessor.java
  7. 2
      application/src/main/java/org/thingsboard/server/actors/service/DefaultActorService.java
  8. 18
      application/src/main/java/org/thingsboard/server/actors/tenant/TenantActor.java
  9. 27
      application/src/main/java/org/thingsboard/server/controller/BaseController.java
  10. 6
      application/src/main/java/org/thingsboard/server/controller/CustomerController.java
  11. 378
      application/src/main/java/org/thingsboard/server/controller/DashboardController.java
  12. 2
      application/src/main/java/org/thingsboard/server/controller/DeviceController.java
  13. 4
      application/src/main/java/org/thingsboard/server/controller/EdgeController.java
  14. 104
      application/src/main/java/org/thingsboard/server/controller/QueueController.java
  15. 1
      application/src/main/java/org/thingsboard/server/controller/TenantController.java
  16. 11
      application/src/main/java/org/thingsboard/server/controller/TenantProfileController.java
  17. 1
      application/src/main/java/org/thingsboard/server/install/ThingsboardInstallService.java
  18. 2
      application/src/main/java/org/thingsboard/server/service/edge/EdgeBulkImportService.java
  19. 33
      application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/TelemetryEdgeProcessor.java
  20. 10
      application/src/main/java/org/thingsboard/server/service/entitiy/AbstractTbEntityService.java
  21. 48
      application/src/main/java/org/thingsboard/server/service/entitiy/DefaultTbNotificationEntityService.java
  22. 4
      application/src/main/java/org/thingsboard/server/service/entitiy/SimpleTbEntityService.java
  23. 17
      application/src/main/java/org/thingsboard/server/service/entitiy/TbNotificationEntityService.java
  24. 14
      application/src/main/java/org/thingsboard/server/service/entitiy/alarm/DefaultTbAlarmService.java
  25. 6
      application/src/main/java/org/thingsboard/server/service/entitiy/alarm/TbAlarmService.java
  26. 6
      application/src/main/java/org/thingsboard/server/service/entitiy/asset/DefaultTbAssetService.java
  27. 5
      application/src/main/java/org/thingsboard/server/service/entitiy/asset/TbAssetService.java
  28. 18
      application/src/main/java/org/thingsboard/server/service/entitiy/customer/DefaultTbCustomerService.java
  29. 4
      application/src/main/java/org/thingsboard/server/service/entitiy/customer/TbCustomerService.java
  30. 275
      application/src/main/java/org/thingsboard/server/service/entitiy/dashboard/DefaultTbDashboardService.java
  31. 49
      application/src/main/java/org/thingsboard/server/service/entitiy/dashboard/TbDashboardService.java
  32. 2
      application/src/main/java/org/thingsboard/server/service/entitiy/device/DefaultTbDeviceService.java
  33. 2
      application/src/main/java/org/thingsboard/server/service/entitiy/device/TbDeviceService.java
  34. 8
      application/src/main/java/org/thingsboard/server/service/entitiy/edge/DefaultTbEdgeService.java
  35. 4
      application/src/main/java/org/thingsboard/server/service/entitiy/edge/TbEdgeService.java
  36. 269
      application/src/main/java/org/thingsboard/server/service/entitiy/queue/DefaultTbQueueService.java
  37. 34
      application/src/main/java/org/thingsboard/server/service/entitiy/queue/TbQueueService.java
  38. 27
      application/src/main/java/org/thingsboard/server/service/entitiy/tenant/DefaultTbTenantService.java
  39. 50
      application/src/main/java/org/thingsboard/server/service/entitiy/tenant_profile/DefaultTbTenantProfileService.java
  40. 23
      application/src/main/java/org/thingsboard/server/service/entitiy/tenant_profile/TbTenantProfileService.java
  41. 103
      application/src/main/java/org/thingsboard/server/service/install/DefaultSystemDataLoaderService.java
  42. 158
      application/src/main/java/org/thingsboard/server/service/install/SqlDatabaseUpgradeService.java
  43. 1
      application/src/main/java/org/thingsboard/server/service/install/SystemDataLoaderService.java
  44. 48
      application/src/main/java/org/thingsboard/server/service/install/TbRuleEngineQueueConfigService.java
  45. 44
      application/src/main/java/org/thingsboard/server/service/queue/DefaultQueueRoutingInfoService.java
  46. 120
      application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java
  47. 14
      application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java
  48. 184
      application/src/main/java/org/thingsboard/server/service/queue/DefaultTbRuleEngineConsumerService.java
  49. 2
      application/src/main/java/org/thingsboard/server/service/queue/TbTopicWithConsumerPerPartition.java
  50. 7
      application/src/main/java/org/thingsboard/server/service/queue/processing/AbstractConsumerService.java
  51. 7
      application/src/main/java/org/thingsboard/server/service/queue/processing/TbRuleEngineProcessingResult.java
  52. 44
      application/src/main/java/org/thingsboard/server/service/queue/processing/TbRuleEngineProcessingStrategyFactory.java
  53. 19
      application/src/main/java/org/thingsboard/server/service/queue/processing/TbRuleEngineSubmitStrategyFactory.java
  54. 3
      application/src/main/java/org/thingsboard/server/service/security/permission/Resource.java
  55. 1
      application/src/main/java/org/thingsboard/server/service/security/permission/SysAdminPermissions.java
  56. 17
      application/src/main/java/org/thingsboard/server/service/security/permission/TenantAdminPermissions.java
  57. 22
      application/src/main/java/org/thingsboard/server/service/subscription/DefaultSubscriptionManagerService.java
  58. 2
      application/src/main/java/org/thingsboard/server/service/subscription/DefaultTbLocalSubscriptionService.java
  59. 13
      application/src/main/java/org/thingsboard/server/service/transport/DefaultTbCoreToTransportService.java
  60. 25
      application/src/main/java/org/thingsboard/server/service/transport/DefaultTransportApiService.java
  61. 1
      application/src/main/resources/thingsboard.yml
  62. 5
      application/src/test/java/org/thingsboard/server/controller/AbstractWebTest.java
  63. 33
      application/src/test/java/org/thingsboard/server/controller/BaseTenantProfileControllerTest.java
  64. 25
      application/src/test/java/org/thingsboard/server/service/cluster/routing/HashPartitionServiceTest.java
  65. 10
      common/cluster-api/src/main/java/org/thingsboard/server/cluster/TbClusterService.java
  66. 2
      common/cluster-api/src/main/java/org/thingsboard/server/queue/TbQueueAdmin.java
  67. 12
      common/cluster-api/src/main/java/org/thingsboard/server/queue/TbQueueClusterService.java
  68. 48
      common/cluster-api/src/main/proto/queue.proto
  69. 45
      common/dao-api/src/main/java/org/thingsboard/server/dao/queue/QueueService.java
  70. 2
      common/dao-api/src/main/java/org/thingsboard/server/dao/rule/RuleChainService.java
  71. 13
      common/dao-api/src/main/java/org/thingsboard/server/dao/tenant/TenantService.java
  72. 5
      common/data/src/main/java/org/thingsboard/server/common/data/DeviceProfile.java
  73. 2
      common/data/src/main/java/org/thingsboard/server/common/data/EntityType.java
  74. 2
      common/data/src/main/java/org/thingsboard/server/common/data/TenantProfile.java
  75. 2
      common/data/src/main/java/org/thingsboard/server/common/data/id/EntityIdFactory.java
  76. 41
      common/data/src/main/java/org/thingsboard/server/common/data/id/QueueId.java
  77. 27
      common/data/src/main/java/org/thingsboard/server/common/data/queue/ProcessingStrategy.java
  78. 36
      common/data/src/main/java/org/thingsboard/server/common/data/queue/ProcessingStrategyType.java
  79. 61
      common/data/src/main/java/org/thingsboard/server/common/data/queue/Queue.java
  80. 24
      common/data/src/main/java/org/thingsboard/server/common/data/queue/SubmitStrategy.java
  81. 20
      common/data/src/main/java/org/thingsboard/server/common/data/queue/SubmitStrategyType.java
  82. 5
      common/data/src/main/java/org/thingsboard/server/common/data/tenant/profile/TenantProfileData.java
  83. 32
      common/data/src/main/java/org/thingsboard/server/common/data/tenant/profile/TenantProfileQueueConfiguration.java
  84. 63
      common/message/src/main/java/org/thingsboard/server/common/msg/TbMsg.java
  85. 2
      common/message/src/main/java/org/thingsboard/server/common/msg/queue/PartitionChangeMsg.java
  86. 62
      common/message/src/main/java/org/thingsboard/server/common/msg/queue/ServiceQueue.java
  87. 54
      common/message/src/main/java/org/thingsboard/server/common/msg/queue/ServiceQueueKey.java
  88. 2
      common/message/src/main/java/org/thingsboard/server/common/msg/queue/TopicPartitionInfo.java
  89. 62
      common/queue/src/main/java/org/thingsboard/server/queue/DefaultQueueService.java
  90. 114
      common/queue/src/main/java/org/thingsboard/server/queue/RuleEngineTbQueueAdminFactory.java
  91. 25
      common/queue/src/main/java/org/thingsboard/server/queue/azure/servicebus/TbServiceBusAdmin.java
  92. 36
      common/queue/src/main/java/org/thingsboard/server/queue/discovery/DefaultTbServiceInfoProvider.java
  93. 311
      common/queue/src/main/java/org/thingsboard/server/queue/discovery/HashPartitionService.java
  94. 54
      common/queue/src/main/java/org/thingsboard/server/queue/discovery/NotificationsTopicService.java
  95. 23
      common/queue/src/main/java/org/thingsboard/server/queue/discovery/PartitionService.java
  96. 56
      common/queue/src/main/java/org/thingsboard/server/queue/discovery/QueueKey.java
  97. 60
      common/queue/src/main/java/org/thingsboard/server/queue/discovery/QueueRoutingInfo.java
  98. 24
      common/queue/src/main/java/org/thingsboard/server/queue/discovery/QueueRoutingInfoService.java
  99. 7
      common/queue/src/main/java/org/thingsboard/server/queue/discovery/TbServiceInfoProvider.java
  100. 44
      common/queue/src/main/java/org/thingsboard/server/queue/discovery/TopicPartitionInfoKey.java

15
application/src/main/data/upgrade/3.3.4/schema_update.sql

@ -29,3 +29,18 @@ ALTER TABLE customer
ALTER TABLE admin_settings
ADD COLUMN IF NOT EXISTS tenant_id uuid NOT NULL DEFAULT '13814000-1dd2-11b2-8080-808080808080';
CREATE TABLE IF NOT EXISTS queue (
id uuid NOT NULL CONSTRAINT queue_pkey PRIMARY KEY,
created_time bigint NOT NULL,
tenant_id uuid,
name varchar(255),
topic varchar(255),
poll_interval int,
partitions int,
consumer_per_partition boolean,
pack_processing_timeout bigint,
submit_strategy varchar(255),
processing_strategy varchar(255),
additional_info varchar
);

49
application/src/main/data/upgrade/3.3.4/schema_update_device_profile.sql

@ -0,0 +1,49 @@
--
-- Copyright © 2016-2022 The Thingsboard Authors
--
-- Licensed under the Apache License, Version 2.0 (the "License");
-- you may not use this file except in compliance with the License.
-- You may obtain a copy of the License at
--
-- http://www.apache.org/licenses/LICENSE-2.0
--
-- Unless required by applicable law or agreed to in writing, software
-- distributed under the License is distributed on an "AS IS" BASIS,
-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-- See the License for the specific language governing permissions and
-- limitations under the License.
--
ALTER TABLE device_profile
ADD COLUMN IF NOT EXISTS default_queue_id uuid;
DO
$$
BEGIN
IF EXISTS
(SELECT column_name
FROM information_schema.columns
WHERE table_name = 'device_profile'
AND column_name = 'default_queue_name'
)
THEN
UPDATE device_profile
SET default_queue_id = q.id
FROM queue as q
WHERE default_queue_name = q.name;
END IF;
END
$$;
DO
$$
BEGIN
IF NOT EXISTS(SELECT 1 FROM pg_constraint WHERE conname = 'fk_default_queue_device_profile') THEN
ALTER TABLE device_profile
ADD CONSTRAINT fk_default_queue_device_profile FOREIGN KEY (default_queue_id) REFERENCES queue (id);
END IF;
END;
$$;
ALTER TABLE device_profile
DROP COLUMN IF EXISTS default_queue_name;

14
application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java

@ -40,6 +40,7 @@ import org.thingsboard.server.cluster.TbClusterService;
import org.thingsboard.server.common.data.DataConstants;
import org.thingsboard.server.common.data.Event;
import org.thingsboard.server.common.data.id.EntityId;
import org.thingsboard.server.common.data.id.QueueId;
import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.data.plugin.ComponentLifecycleEvent;
import org.thingsboard.server.common.msg.TbActorMsg;
@ -63,6 +64,7 @@ import org.thingsboard.server.dao.event.EventService;
import org.thingsboard.server.dao.nosql.CassandraBufferedRateReadExecutor;
import org.thingsboard.server.dao.nosql.CassandraBufferedRateWriteExecutor;
import org.thingsboard.server.dao.ota.OtaPackageService;
import org.thingsboard.server.dao.queue.QueueService;
import org.thingsboard.server.dao.relation.RelationService;
import org.thingsboard.server.dao.resource.ResourceService;
import org.thingsboard.server.dao.rule.RuleChainService;
@ -330,6 +332,11 @@ public class ActorSystemContext {
@Getter
private TbRpcService tbRpcService;
@Lazy
@Autowired(required = false)
@Getter
private QueueService queueService;
@Value("${actors.session.max_concurrent_sessions_per_device:1}")
@Getter
private long maxConcurrentSessionsPerDevice;
@ -495,8 +502,13 @@ public class ActorSystemContext {
return partitionService.resolve(serviceType, tenantId, entityId);
}
public TopicPartitionInfo resolve(ServiceType serviceType, QueueId queueId, TenantId tenantId, EntityId entityId) {
return partitionService.resolve(serviceType, queueId, tenantId, entityId);
}
@Deprecated
public TopicPartitionInfo resolve(ServiceType serviceType, String queueName, TenantId tenantId, EntityId entityId) {
return partitionService.resolve(serviceType, queueName, tenantId, entityId);
return partitionService.resolve(serviceType, tenantId, entityId, queueName);
}
public String getServiceId() {

30
application/src/main/java/org/thingsboard/server/actors/app/AppActor.java

@ -30,8 +30,6 @@ import org.thingsboard.server.actors.service.DefaultActorService;
import org.thingsboard.server.actors.tenant.TenantActor;
import org.thingsboard.server.common.data.EntityType;
import org.thingsboard.server.common.data.Tenant;
import org.thingsboard.server.common.data.TenantProfile;
import org.thingsboard.server.common.data.id.EntityId;
import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.data.page.PageDataIterable;
import org.thingsboard.server.common.data.plugin.ComponentLifecycleEvent;
@ -45,24 +43,20 @@ import org.thingsboard.server.common.msg.queue.RuleEngineException;
import org.thingsboard.server.common.msg.queue.ServiceType;
import org.thingsboard.server.dao.model.ModelConstants;
import org.thingsboard.server.dao.tenant.TenantService;
import org.thingsboard.server.dao.tenant.TbTenantProfileCache;
import org.thingsboard.server.service.transport.msg.TransportToDeviceActorMsgWrapper;
import java.util.HashSet;
import java.util.Optional;
import java.util.Set;
@Slf4j
public class AppActor extends ContextAwareActor {
private final TbTenantProfileCache tenantProfileCache;
private final TenantService tenantService;
private final Set<TenantId> deletedTenants;
private volatile boolean ruleChainsInitialized;
private AppActor(ActorSystemContext systemContext) {
super(systemContext);
this.tenantProfileCache = systemContext.getTenantProfileCache();
this.tenantService = systemContext.getTenantService();
this.deletedTenants = new HashSet<>();
}
@ -125,28 +119,12 @@ public class AppActor extends ContextAwareActor {
private void initTenantActors() {
log.info("Starting main system actor.");
try {
// This Service may be started for specific tenant only.
Optional<TenantId> isolatedTenantId = systemContext.getServiceInfoProvider().getIsolatedTenant();
if (isolatedTenantId.isPresent()) {
Tenant tenant = systemContext.getTenantService().findTenantById(isolatedTenantId.get());
if (tenant != null) {
log.debug("[{}] Creating tenant actor", tenant.getId());
getOrCreateTenantActor(tenant.getId());
log.debug("Tenant actor created.");
} else {
log.error("[{}] Tenant with such ID does not exist", isolatedTenantId.get());
}
} else if (systemContext.isTenantComponentsInitEnabled()) {
if (systemContext.isTenantComponentsInitEnabled()) {
PageDataIterable<Tenant> tenantIterator = new PageDataIterable<>(tenantService::findTenants, ENTITY_PACK_LIMIT);
boolean isRuleEngine = systemContext.getServiceInfoProvider().isService(ServiceType.TB_RULE_ENGINE);
boolean isCore = systemContext.getServiceInfoProvider().isService(ServiceType.TB_CORE);
for (Tenant tenant : tenantIterator) {
TenantProfile tenantProfile = tenantProfileCache.get(tenant.getTenantProfileId());
if (isCore || (isRuleEngine && !tenantProfile.isIsolatedTbRuleEngine())) {
log.debug("[{}] Creating tenant actor", tenant.getId());
getOrCreateTenantActor(tenant.getId());
log.debug("[{}] Tenant actor created.", tenant.getId());
}
log.debug("[{}] Creating tenant actor", tenant.getId());
getOrCreateTenantActor(tenant.getId());
log.debug("[{}] Tenant actor created.", tenant.getId());
}
}
log.info("Main system actor started.");

75
application/src/main/java/org/thingsboard/server/actors/ruleChain/DefaultTbContext.java

@ -46,6 +46,7 @@ import org.thingsboard.server.common.data.id.CustomerId;
import org.thingsboard.server.common.data.id.DeviceId;
import org.thingsboard.server.common.data.id.EdgeId;
import org.thingsboard.server.common.data.id.EntityId;
import org.thingsboard.server.common.data.id.QueueId;
import org.thingsboard.server.common.data.id.RuleChainId;
import org.thingsboard.server.common.data.id.RuleNodeId;
import org.thingsboard.server.common.data.id.TenantId;
@ -57,7 +58,6 @@ import org.thingsboard.server.common.msg.TbActorMsg;
import org.thingsboard.server.common.msg.TbMsg;
import org.thingsboard.server.common.msg.TbMsgMetaData;
import org.thingsboard.server.common.msg.TbMsgProcessingStackItem;
import org.thingsboard.server.common.msg.queue.ServiceQueue;
import org.thingsboard.server.common.msg.queue.ServiceType;
import org.thingsboard.server.common.msg.queue.TopicPartitionInfo;
import org.thingsboard.server.dao.asset.AssetService;
@ -72,6 +72,7 @@ import org.thingsboard.server.dao.entityview.EntityViewService;
import org.thingsboard.server.dao.nosql.CassandraStatementTask;
import org.thingsboard.server.dao.nosql.TbResultSetFuture;
import org.thingsboard.server.dao.ota.OtaPackageService;
import org.thingsboard.server.dao.queue.QueueService;
import org.thingsboard.server.dao.relation.RelationService;
import org.thingsboard.server.dao.resource.ResourceService;
import org.thingsboard.server.dao.rule.RuleChainService;
@ -161,11 +162,18 @@ class DefaultTbContext implements TbContext {
}
@Override
@Deprecated
public void enqueue(TbMsg tbMsg, String queueName, Runnable onSuccess, Consumer<Throwable> onFailure) {
TopicPartitionInfo tpi = resolvePartition(tbMsg, queueName);
enqueue(tpi, tbMsg, onFailure, onSuccess);
}
@Override
public void enqueue(TbMsg tbMsg, QueueId queueId, Runnable onSuccess, Consumer<Throwable> onFailure) {
TopicPartitionInfo tpi = resolvePartition(tbMsg, queueId);
enqueue(tpi, tbMsg, onFailure, onSuccess);
}
private void enqueue(TopicPartitionInfo tpi, TbMsg tbMsg, Consumer<Throwable> onFailure, Runnable onSuccess) {
if (!tbMsg.isValid()) {
log.trace("[{}] Skip invalid message: {}", getTenantId(), tbMsg);
@ -213,33 +221,35 @@ class DefaultTbContext implements TbContext {
}
@Override
public void enqueueForTellNext(TbMsg tbMsg, String queueName, String relationType, Runnable onSuccess, Consumer<Throwable> onFailure) {
TopicPartitionInfo tpi = resolvePartition(tbMsg, queueName);
enqueueForTellNext(tpi, queueName, tbMsg, Collections.singleton(relationType), null, onSuccess, onFailure);
public void enqueueForTellNext(TbMsg tbMsg, QueueId queueId, String relationType, Runnable onSuccess, Consumer<Throwable> onFailure) {
TopicPartitionInfo tpi = resolvePartition(tbMsg, queueId);
enqueueForTellNext(tpi, queueId, tbMsg, Collections.singleton(relationType), null, onSuccess, onFailure);
}
@Override
public void enqueueForTellNext(TbMsg tbMsg, String queueName, Set<String> relationTypes, Runnable onSuccess, Consumer<Throwable> onFailure) {
TopicPartitionInfo tpi = resolvePartition(tbMsg, queueName);
enqueueForTellNext(tpi, queueName, tbMsg, relationTypes, null, onSuccess, onFailure);
public void enqueueForTellNext(TbMsg tbMsg, QueueId queueId, Set<String> relationTypes, Runnable onSuccess, Consumer<Throwable> onFailure) {
TopicPartitionInfo tpi = resolvePartition(tbMsg, queueId);
enqueueForTellNext(tpi, queueId, tbMsg, relationTypes, null, onSuccess, onFailure);
}
private TopicPartitionInfo resolvePartition(TbMsg tbMsg, QueueId queueId) {
return mainCtx.resolve(ServiceType.TB_RULE_ENGINE, queueId, getTenantId(), tbMsg.getOriginator());
}
@Deprecated
private TopicPartitionInfo resolvePartition(TbMsg tbMsg, String queueName) {
if (StringUtils.isEmpty(queueName)) {
queueName = ServiceQueue.MAIN;
}
return mainCtx.resolve(ServiceType.TB_RULE_ENGINE, queueName, getTenantId(), tbMsg.getOriginator());
}
private TopicPartitionInfo resolvePartition(TbMsg tbMsg) {
return resolvePartition(tbMsg, tbMsg.getQueueName());
return resolvePartition(tbMsg, tbMsg.getQueueId());
}
private void enqueueForTellNext(TopicPartitionInfo tpi, TbMsg source, Set<String> relationTypes, String failureMessage, Runnable onSuccess, Consumer<Throwable> onFailure) {
enqueueForTellNext(tpi, source.getQueueName(), source, relationTypes, failureMessage, onSuccess, onFailure);
enqueueForTellNext(tpi, source.getQueueId(), source, relationTypes, failureMessage, onSuccess, onFailure);
}
private void enqueueForTellNext(TopicPartitionInfo tpi, String queueName, TbMsg source, Set<String> relationTypes, String failureMessage, Runnable onSuccess, Consumer<Throwable> onFailure) {
private void enqueueForTellNext(TopicPartitionInfo tpi, QueueId queueId, TbMsg source, Set<String> relationTypes, String failureMessage, Runnable onSuccess, Consumer<Throwable> onFailure) {
if (!source.isValid()) {
log.trace("[{}] Skip invalid message: {}", getTenantId(), source);
onFailure.accept(new IllegalArgumentException("Source message is no longer valid!"));
@ -247,7 +257,7 @@ class DefaultTbContext implements TbContext {
}
RuleChainId ruleChainId = nodeCtx.getSelf().getRuleChainId();
RuleNodeId ruleNodeId = nodeCtx.getSelf().getId();
TbMsg tbMsg = TbMsg.newMsg(source, queueName, ruleChainId, ruleNodeId);
TbMsg tbMsg = TbMsg.newMsg(source, queueId, ruleChainId, ruleNodeId);
TransportProtos.ToRuleEngineMsg.Builder msg = TransportProtos.ToRuleEngineMsg.newBuilder()
.setTenantIdMSB(getTenantId().getId().getMostSignificantBits())
.setTenantIdLSB(getTenantId().getId().getLeastSignificantBits())
@ -306,13 +316,13 @@ class DefaultTbContext implements TbContext {
}
@Override
public TbMsg newMsg(String queueName, String type, EntityId originator, TbMsgMetaData metaData, String data) {
return newMsg(queueName, type, originator, null, metaData, data);
public TbMsg newMsg(QueueId queueId, String type, EntityId originator, TbMsgMetaData metaData, String data) {
return newMsg(queueId, type, originator, null, metaData, data);
}
@Override
public TbMsg newMsg(String queueName, String type, EntityId originator, CustomerId customerId, TbMsgMetaData metaData, String data) {
return TbMsg.newMsg(queueName, type, originator, customerId, metaData, data, nodeCtx.getSelf().getRuleChainId(), nodeCtx.getSelf().getId());
public TbMsg newMsg(QueueId queueId, String type, EntityId originator, CustomerId customerId, TbMsgMetaData metaData, String data) {
return TbMsg.newMsg(queueId, type, originator, customerId, metaData, data, nodeCtx.getSelf().getRuleChainId(), nodeCtx.getSelf().getId());
}
@Override
@ -326,20 +336,17 @@ class DefaultTbContext implements TbContext {
public TbMsg deviceCreatedMsg(Device device, RuleNodeId ruleNodeId) {
RuleChainId ruleChainId = null;
String queueName = ServiceQueue.MAIN;
QueueId queueId = null;
if (device.getDeviceProfileId() != null) {
DeviceProfile deviceProfile = mainCtx.getDeviceProfileCache().find(device.getDeviceProfileId());
if (deviceProfile == null) {
log.warn("[{}] Device profile is null!", device.getDeviceProfileId());
ruleChainId = null;
queueName = ServiceQueue.MAIN;
} else {
ruleChainId = deviceProfile.getDefaultRuleChainId();
String defaultQueueName = deviceProfile.getDefaultQueueName();
queueName = defaultQueueName != null ? defaultQueueName : ServiceQueue.MAIN;
queueId = deviceProfile.getDefaultQueueId();
}
}
return entityActionMsg(device, device.getId(), ruleNodeId, DataConstants.ENTITY_CREATED, queueName, ruleChainId);
return entityActionMsg(device, device.getId(), ruleNodeId, DataConstants.ENTITY_CREATED, queueId, ruleChainId);
}
public TbMsg assetCreatedMsg(Asset asset, RuleNodeId ruleNodeId) {
@ -348,21 +355,18 @@ class DefaultTbContext implements TbContext {
public TbMsg alarmActionMsg(Alarm alarm, RuleNodeId ruleNodeId, String action) {
RuleChainId ruleChainId = null;
String queueName = ServiceQueue.MAIN;
QueueId queueId = null;
if (EntityType.DEVICE.equals(alarm.getOriginator().getEntityType())) {
DeviceId deviceId = new DeviceId(alarm.getOriginator().getId());
DeviceProfile deviceProfile = mainCtx.getDeviceProfileCache().get(getTenantId(), deviceId);
if (deviceProfile == null) {
log.warn("[{}] Device profile is null!", deviceId);
ruleChainId = null;
queueName = ServiceQueue.MAIN;
} else {
ruleChainId = deviceProfile.getDefaultRuleChainId();
String defaultQueueName = deviceProfile.getDefaultQueueName();
queueName = defaultQueueName != null ? defaultQueueName : ServiceQueue.MAIN;
queueId = deviceProfile.getDefaultQueueId();
}
}
return entityActionMsg(alarm, alarm.getId(), ruleNodeId, action, queueName, ruleChainId);
return entityActionMsg(alarm, alarm.getId(), ruleNodeId, action, queueId, ruleChainId);
}
@Override
@ -371,12 +375,12 @@ class DefaultTbContext implements TbContext {
}
public <E, I extends EntityId> TbMsg entityActionMsg(E entity, I id, RuleNodeId ruleNodeId, String action) {
return entityActionMsg(entity, id, ruleNodeId, action, ServiceQueue.MAIN, null);
return entityActionMsg(entity, id, ruleNodeId, action, null, null);
}
public <E, I extends EntityId> TbMsg entityActionMsg(E entity, I id, RuleNodeId ruleNodeId, String action, String queueName, RuleChainId ruleChainId) {
public <E, I extends EntityId> TbMsg entityActionMsg(E entity, I id, RuleNodeId ruleNodeId, String action, QueueId queueId, RuleChainId ruleChainId) {
try {
return TbMsg.newMsg(queueName, action, id, getActionMetaData(ruleNodeId), mapper.writeValueAsString(mapper.valueToTree(entity)), ruleChainId, null);
return TbMsg.newMsg(queueId, action, id, getActionMetaData(ruleNodeId), mapper.writeValueAsString(mapper.valueToTree(entity)), ruleChainId, null);
} catch (JsonProcessingException | IllegalArgumentException e) {
throw new RuntimeException("Failed to process " + id.getEntityType().name().toLowerCase() + " " + action + " msg: " + e);
}
@ -548,6 +552,11 @@ class DefaultTbContext implements TbContext {
return mainCtx.getEdgeEventService();
}
@Override
public QueueService getQueueService() {
return mainCtx.getQueueService();
}
@Override
public EventLoopGroup getSharedEventLoop() {
return mainCtx.getSharedEventLoopGroupService().getSharedEventLoopGroup();

2
application/src/main/java/org/thingsboard/server/actors/ruleChain/RuleChainActorMessageProcessor.java

@ -293,7 +293,7 @@ public class RuleChainActorMessageProcessor extends ComponentMsgProcessor<RuleCh
try {
checkComponentStateActive(msg);
EntityId entityId = msg.getOriginator();
TopicPartitionInfo tpi = systemContext.resolve(ServiceType.TB_RULE_ENGINE, msg.getQueueName(), tenantId, entityId);
TopicPartitionInfo tpi = systemContext.resolve(ServiceType.TB_RULE_ENGINE, msg.getQueueId(), tenantId, entityId);
List<RuleNodeRelation> ruleNodeRelations = nodeRoutes.get(originatorNodeId);
if (ruleNodeRelations == null) { // When unchecked, this will cause NullPointerException when rule node doesn't exist anymore

2
application/src/main/java/org/thingsboard/server/actors/service/DefaultActorService.java

@ -123,7 +123,7 @@ public class DefaultActorService extends TbApplicationEventListener<PartitionCha
@Override
protected void onTbApplicationEvent(PartitionChangeEvent event) {
log.info("Received partition change event.");
this.appActor.tellWithHighPriority(new PartitionChangeMsg(event.getServiceQueueKey(), event.getPartitions()));
this.appActor.tellWithHighPriority(new PartitionChangeMsg(event.getQueueKey().getType(), event.getPartitions()));
}
@PreDestroy

18
application/src/main/java/org/thingsboard/server/actors/tenant/TenantActor.java

@ -57,7 +57,6 @@ import org.thingsboard.server.service.edge.rpc.EdgeRpcService;
import org.thingsboard.server.service.transport.msg.TransportToDeviceActorMsgWrapper;
import java.util.List;
import java.util.Optional;
@Slf4j
public class TenantActor extends RuleChainManagerActor {
@ -82,24 +81,17 @@ public class TenantActor extends RuleChainManagerActor {
cantFindTenant = true;
log.info("[{}] Started tenant actor for missing tenant.", tenantId);
} else {
// This Service may be started for specific tenant only.
Optional<TenantId> isolatedTenantId = systemContext.getServiceInfoProvider().getIsolatedTenant();
TenantProfile tenantProfile = systemContext.getTenantProfileCache().get(tenant.getTenantProfileId());
isCore = systemContext.getServiceInfoProvider().isService(ServiceType.TB_CORE);
isRuleEngine = systemContext.getServiceInfoProvider().isService(ServiceType.TB_RULE_ENGINE);
if (isRuleEngine) {
try {
if (isolatedTenantId.map(id -> id.equals(tenantId)).orElseGet(() -> !tenantProfile.isIsolatedTbRuleEngine())) {
if (getApiUsageState().isReExecEnabled()) {
log.debug("[{}] Going to init rule chains", tenantId);
initRuleChains();
} else {
log.info("[{}] Skip init of the rule chains due to API limits", tenantId);
}
if (getApiUsageState().isReExecEnabled()) {
log.debug("[{}] Going to init rule chains", tenantId);
initRuleChains();
} else {
isRuleEngine = false;
log.info("[{}] Skip init of the rule chains due to API limits", tenantId);
}
} catch (Exception e) {
cantFindTenant = true;
@ -133,7 +125,7 @@ public class TenantActor extends RuleChainManagerActor {
switch (msg.getMsgType()) {
case PARTITION_CHANGE_MSG:
PartitionChangeMsg partitionChangeMsg = (PartitionChangeMsg) msg;
ServiceType serviceType = partitionChangeMsg.getServiceQueueKey().getServiceType();
ServiceType serviceType = partitionChangeMsg.getServiceType();
if (ServiceType.TB_RULE_ENGINE.equals(serviceType)) {
//To Rule Chain Actors
broadcast(msg);

27
application/src/main/java/org/thingsboard/server/controller/BaseController.java

@ -69,6 +69,7 @@ import org.thingsboard.server.common.data.id.EntityId;
import org.thingsboard.server.common.data.id.EntityIdFactory;
import org.thingsboard.server.common.data.id.EntityViewId;
import org.thingsboard.server.common.data.id.OtaPackageId;
import org.thingsboard.server.common.data.id.QueueId;
import org.thingsboard.server.common.data.id.RpcId;
import org.thingsboard.server.common.data.id.RuleChainId;
import org.thingsboard.server.common.data.id.RuleNodeId;
@ -84,6 +85,7 @@ import org.thingsboard.server.common.data.page.SortOrder;
import org.thingsboard.server.common.data.page.TimePageLink;
import org.thingsboard.server.common.data.plugin.ComponentDescriptor;
import org.thingsboard.server.common.data.plugin.ComponentType;
import org.thingsboard.server.common.data.queue.Queue;
import org.thingsboard.server.common.data.relation.EntityRelation;
import org.thingsboard.server.common.data.rpc.Rpc;
import org.thingsboard.server.common.data.rule.RuleChain;
@ -108,6 +110,7 @@ import org.thingsboard.server.dao.model.ModelConstants;
import org.thingsboard.server.dao.oauth2.OAuth2ConfigTemplateService;
import org.thingsboard.server.dao.oauth2.OAuth2Service;
import org.thingsboard.server.dao.ota.OtaPackageService;
import org.thingsboard.server.dao.queue.QueueService;
import org.thingsboard.server.dao.relation.RelationService;
import org.thingsboard.server.dao.rpc.RpcService;
import org.thingsboard.server.dao.rule.RuleChainService;
@ -147,6 +150,7 @@ import java.util.UUID;
import static org.thingsboard.server.controller.ControllerConstants.DEFAULT_PAGE_SIZE;
import static org.thingsboard.server.controller.ControllerConstants.INCORRECT_TENANT_ID;
import static org.thingsboard.server.controller.UserController.YOU_DON_T_HAVE_PERMISSION_TO_PERFORM_THIS_OPERATION;
import static org.thingsboard.server.dao.service.Validator.validateId;
@Slf4j
@ -271,6 +275,9 @@ public abstract class BaseController {
@Autowired
protected EntityActionService entityActionService;
@Autowired
protected QueueService queueService;
@Value("${server.log_controller_error_stack_trace}")
@Getter
private boolean logControllerErrorStackTrace;
@ -520,6 +527,9 @@ public abstract class BaseController {
case OTA_PACKAGE:
checkOtaPackageId(new OtaPackageId(entityId.getId()), operation);
return;
case QUEUE:
checkQueueId(new QueueId(entityId.getId()), operation);
return;
default:
throw new IllegalArgumentException("Unsupported entity type: " + entityId.getEntityType());
}
@ -811,7 +821,22 @@ public abstract class BaseController {
}
}
@SuppressWarnings("unchecked")
protected Queue checkQueueId(QueueId queueId, Operation operation) throws ThingsboardException {
validateId(queueId, "Incorrect queueId " + queueId);
Queue queue = queueService.findQueueById(getCurrentUser().getTenantId(), queueId);
checkNotNull(queue);
accessControlService.checkPermission(getCurrentUser(), Resource.QUEUE, operation, queueId, queue);
TenantId tenantId = getTenantId();
if (queue.getTenantId().isNullUid() && !tenantId.isNullUid()) {
TenantProfile tenantProfile = tenantProfileCache.get(tenantId);
if (tenantProfile.isIsolatedTbRuleEngine()) {
throw new ThingsboardException(YOU_DON_T_HAVE_PERMISSION_TO_PERFORM_THIS_OPERATION,
ThingsboardErrorCode.PERMISSION_DENIED);
}
}
return queue;
}
protected <I extends EntityId> I emptyId(EntityType entityType) {
return (I) EntityIdFactory.getByTypeAndUuid(entityType, ModelConstants.NULL_UUID);
}

6
application/src/main/java/org/thingsboard/server/controller/CustomerController.java

@ -32,22 +32,16 @@ import org.springframework.web.bind.annotation.ResponseBody;
import org.springframework.web.bind.annotation.ResponseStatus;
import org.springframework.web.bind.annotation.RestController;
import org.thingsboard.server.common.data.Customer;
import org.thingsboard.server.common.data.EntityType;
import org.thingsboard.server.common.data.audit.ActionType;
import org.thingsboard.server.common.data.exception.ThingsboardException;
import org.thingsboard.server.common.data.id.CustomerId;
import org.thingsboard.server.common.data.id.EdgeId;
import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.data.page.PageData;
import org.thingsboard.server.common.data.page.PageLink;
import org.thingsboard.server.common.data.plugin.ComponentLifecycleEvent;
import org.thingsboard.server.queue.util.TbCoreComponent;
import org.thingsboard.server.service.entitiy.customer.TbCustomerService;
import org.thingsboard.server.service.security.permission.Operation;
import org.thingsboard.server.service.security.permission.Resource;
import java.util.List;
import static org.thingsboard.server.controller.ControllerConstants.CUSTOMER_ID;
import static org.thingsboard.server.controller.ControllerConstants.CUSTOMER_ID_PARAM_DESCRIPTION;
import static org.thingsboard.server.controller.ControllerConstants.CUSTOMER_SORT_PROPERTY_ALLOWABLE_VALUES;

378
application/src/main/java/org/thingsboard/server/controller/DashboardController.java

@ -22,6 +22,7 @@ import io.swagger.annotations.ApiParam;
import io.swagger.annotations.ApiResponse;
import io.swagger.annotations.Example;
import io.swagger.annotations.ExampleProperty;
import lombok.RequiredArgsConstructor;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.http.HttpStatus;
import org.springframework.http.MediaType;
@ -38,15 +39,12 @@ import org.thingsboard.common.util.JacksonUtil;
import org.thingsboard.server.common.data.Customer;
import org.thingsboard.server.common.data.Dashboard;
import org.thingsboard.server.common.data.DashboardInfo;
import org.thingsboard.server.common.data.EntityType;
import org.thingsboard.server.common.data.HasName;
import org.thingsboard.server.common.data.HomeDashboard;
import org.thingsboard.server.common.data.HomeDashboardInfo;
import org.thingsboard.server.common.data.ShortCustomerInfo;
import org.thingsboard.server.common.data.Tenant;
import org.thingsboard.server.common.data.User;
import org.thingsboard.server.common.data.audit.ActionType;
import org.thingsboard.server.common.data.edge.Edge;
import org.thingsboard.server.common.data.edge.EdgeEventActionType;
import org.thingsboard.server.common.data.exception.ThingsboardException;
import org.thingsboard.server.common.data.id.CustomerId;
import org.thingsboard.server.common.data.id.DashboardId;
@ -55,6 +53,7 @@ import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.data.page.PageData;
import org.thingsboard.server.common.data.page.PageLink;
import org.thingsboard.server.queue.util.TbCoreComponent;
import org.thingsboard.server.service.entitiy.dashboard.TbDashboardService;
import org.thingsboard.server.service.security.model.SecurityUser;
import org.thingsboard.server.service.security.permission.Operation;
import org.thingsboard.server.service.security.permission.Resource;
@ -62,6 +61,7 @@ import org.thingsboard.server.service.security.permission.Resource;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
import java.util.UUID;
import java.util.stream.Collectors;
import static org.thingsboard.server.controller.ControllerConstants.CUSTOMER_ID;
@ -90,9 +90,11 @@ import static org.thingsboard.server.controller.ControllerConstants.UUID_WIKI_LI
@RestController
@TbCoreComponent
@RequiredArgsConstructor
@RequestMapping("/api")
public class DashboardController extends BaseController {
private final TbDashboardService tbDashboardService;
public static final String DASHBOARD_ID = "dashboardId";
private static final String HOME_DASHBOARD_ID = "homeDashboardId";
@ -180,28 +182,9 @@ public class DashboardController extends BaseController {
public Dashboard saveDashboard(
@ApiParam(value = "A JSON value representing the dashboard.")
@RequestBody Dashboard dashboard) throws ThingsboardException {
try {
dashboard.setTenantId(getCurrentUser().getTenantId());
checkEntity(dashboard.getId(), dashboard, Resource.DASHBOARD);
Dashboard savedDashboard = checkNotNull(dashboardService.saveDashboard(dashboard));
logEntityAction(savedDashboard.getId(), savedDashboard,
null,
dashboard.getId() == null ? ActionType.ADDED : ActionType.UPDATED, null);
if (dashboard.getId() != null) {
sendEntityNotificationMsg(savedDashboard.getTenantId(), savedDashboard.getId(), EdgeEventActionType.UPDATED);
}
return savedDashboard;
} catch (Exception e) {
logEntityAction(emptyId(EntityType.DASHBOARD), dashboard,
null, dashboard.getId() == null ? ActionType.ADDED : ActionType.UPDATED, e);
throw handleException(e);
}
dashboard.setTenantId(getCurrentUser().getTenantId());
checkEntity(dashboard.getId(), dashboard, Resource.DASHBOARD);
return tbDashboardService.save(dashboard, getCurrentUser());
}
@ApiOperation(value = "Delete the Dashboard (deleteDashboard)",
@ -213,28 +196,9 @@ public class DashboardController extends BaseController {
@ApiParam(value = DASHBOARD_ID_PARAM_DESCRIPTION)
@PathVariable(DASHBOARD_ID) String strDashboardId) throws ThingsboardException {
checkParameter(DASHBOARD_ID, strDashboardId);
try {
DashboardId dashboardId = new DashboardId(toUUID(strDashboardId));
Dashboard dashboard = checkDashboardId(dashboardId, Operation.DELETE);
List<EdgeId> relatedEdgeIds = findRelatedEdgeIds(getTenantId(), dashboardId);
dashboardService.deleteDashboard(getCurrentUser().getTenantId(), dashboardId);
logEntityAction(dashboardId, dashboard,
null,
ActionType.DELETED, null, strDashboardId);
sendDeleteNotificationMsg(getTenantId(), dashboardId, relatedEdgeIds);
} catch (Exception e) {
logEntityAction(emptyId(EntityType.DASHBOARD),
null,
null,
ActionType.DELETED, e, strDashboardId);
throw handleException(e);
}
DashboardId dashboardId = new DashboardId(toUUID(strDashboardId));
Dashboard dashboard = checkDashboardId(dashboardId, Operation.DELETE);
tbDashboardService.delete(dashboard, getCurrentUser());
}
@ApiOperation(value = "Assign the Dashboard (assignDashboardToCustomer)",
@ -251,30 +215,13 @@ public class DashboardController extends BaseController {
@PathVariable(DASHBOARD_ID) String strDashboardId) throws ThingsboardException {
checkParameter(CUSTOMER_ID, strCustomerId);
checkParameter(DASHBOARD_ID, strDashboardId);
try {
CustomerId customerId = new CustomerId(toUUID(strCustomerId));
Customer customer = checkCustomerId(customerId, Operation.READ);
DashboardId dashboardId = new DashboardId(toUUID(strDashboardId));
checkDashboardId(dashboardId, Operation.ASSIGN_TO_CUSTOMER);
Dashboard savedDashboard = checkNotNull(dashboardService.assignDashboardToCustomer(getCurrentUser().getTenantId(), dashboardId, customerId));
logEntityAction(dashboardId, savedDashboard,
customerId,
ActionType.ASSIGNED_TO_CUSTOMER, null, strDashboardId, strCustomerId, customer.getName());
CustomerId customerId = new CustomerId(toUUID(strCustomerId));
Customer customer = checkCustomerId(customerId, Operation.READ);
sendEntityAssignToCustomerNotificationMsg(savedDashboard.getTenantId(), savedDashboard.getId(), customerId, EdgeEventActionType.ASSIGNED_TO_CUSTOMER);
return savedDashboard;
} catch (Exception e) {
logEntityAction(emptyId(EntityType.DASHBOARD), null,
null,
ActionType.ASSIGNED_TO_CUSTOMER, e, strDashboardId, strCustomerId);
throw handleException(e);
}
DashboardId dashboardId = new DashboardId(toUUID(strDashboardId));
checkDashboardId(dashboardId, Operation.ASSIGN_TO_CUSTOMER);
return tbDashboardService.assignDashboardToCustomer(dashboardId, customer, getCurrentUser());
}
@ApiOperation(value = "Unassign the Dashboard (unassignDashboardFromCustomer)",
@ -291,29 +238,11 @@ public class DashboardController extends BaseController {
@PathVariable(DASHBOARD_ID) String strDashboardId) throws ThingsboardException {
checkParameter("customerId", strCustomerId);
checkParameter(DASHBOARD_ID, strDashboardId);
try {
CustomerId customerId = new CustomerId(toUUID(strCustomerId));
Customer customer = checkCustomerId(customerId, Operation.READ);
DashboardId dashboardId = new DashboardId(toUUID(strDashboardId));
Dashboard dashboard = checkDashboardId(dashboardId, Operation.UNASSIGN_FROM_CUSTOMER);
Dashboard savedDashboard = checkNotNull(dashboardService.unassignDashboardFromCustomer(getCurrentUser().getTenantId(), dashboardId, customerId));
logEntityAction(dashboardId, dashboard,
customerId,
ActionType.UNASSIGNED_FROM_CUSTOMER, null, strDashboardId, customer.getId().toString(), customer.getName());
sendEntityAssignToCustomerNotificationMsg(savedDashboard.getTenantId(), savedDashboard.getId(), customerId, EdgeEventActionType.UNASSIGNED_FROM_CUSTOMER);
return savedDashboard;
} catch (Exception e) {
logEntityAction(emptyId(EntityType.DASHBOARD), null,
null,
ActionType.UNASSIGNED_FROM_CUSTOMER, e, strDashboardId);
throw handleException(e);
}
CustomerId customerId = new CustomerId(toUUID(strCustomerId));
Customer customer = checkCustomerId(customerId, Operation.READ);
DashboardId dashboardId = new DashboardId(toUUID(strDashboardId));
Dashboard dashboard = checkDashboardId(dashboardId, Operation.UNASSIGN_FROM_CUSTOMER);
return tbDashboardService.unassignDashboardFromCustomer(dashboard, customer, getCurrentUser());
}
@ApiOperation(value = "Update the Dashboard Customers (updateDashboardCustomers)",
@ -331,69 +260,15 @@ public class DashboardController extends BaseController {
@ApiParam(value = "JSON array with the list of customer ids, or empty to remove all customers")
@RequestBody(required = false) String[] strCustomerIds) throws ThingsboardException {
checkParameter(DASHBOARD_ID, strDashboardId);
try {
DashboardId dashboardId = new DashboardId(toUUID(strDashboardId));
Dashboard dashboard = checkDashboardId(dashboardId, Operation.ASSIGN_TO_CUSTOMER);
Set<CustomerId> customerIds = new HashSet<>();
if (strCustomerIds != null) {
for (String strCustomerId : strCustomerIds) {
customerIds.add(new CustomerId(toUUID(strCustomerId)));
}
}
Set<CustomerId> addedCustomerIds = new HashSet<>();
Set<CustomerId> removedCustomerIds = new HashSet<>();
for (CustomerId customerId : customerIds) {
if (!dashboard.isAssignedToCustomer(customerId)) {
addedCustomerIds.add(customerId);
}
}
Set<ShortCustomerInfo> assignedCustomers = dashboard.getAssignedCustomers();
if (assignedCustomers != null) {
for (ShortCustomerInfo customerInfo : assignedCustomers) {
if (!customerIds.contains(customerInfo.getCustomerId())) {
removedCustomerIds.add(customerInfo.getCustomerId());
}
}
}
if (addedCustomerIds.isEmpty() && removedCustomerIds.isEmpty()) {
return dashboard;
} else {
Dashboard savedDashboard = null;
for (CustomerId customerId : addedCustomerIds) {
savedDashboard = checkNotNull(dashboardService.assignDashboardToCustomer(getCurrentUser().getTenantId(), dashboardId, customerId));
ShortCustomerInfo customerInfo = savedDashboard.getAssignedCustomerInfo(customerId);
logEntityAction(dashboardId, savedDashboard,
customerId,
ActionType.ASSIGNED_TO_CUSTOMER, null, strDashboardId, customerId.toString(), customerInfo.getTitle());
sendEntityAssignToCustomerNotificationMsg(savedDashboard.getTenantId(), savedDashboard.getId(), customerId, EdgeEventActionType.ASSIGNED_TO_CUSTOMER);
}
for (CustomerId customerId : removedCustomerIds) {
ShortCustomerInfo customerInfo = dashboard.getAssignedCustomerInfo(customerId);
savedDashboard = checkNotNull(dashboardService.unassignDashboardFromCustomer(getCurrentUser().getTenantId(), dashboardId, customerId));
logEntityAction(dashboardId, dashboard,
customerId,
ActionType.UNASSIGNED_FROM_CUSTOMER, null, strDashboardId, customerId.toString(), customerInfo.getTitle());
sendEntityAssignToCustomerNotificationMsg(savedDashboard.getTenantId(), savedDashboard.getId(), customerId, EdgeEventActionType.UNASSIGNED_FROM_CUSTOMER);
}
return savedDashboard;
}
} catch (Exception e) {
logEntityAction(emptyId(EntityType.DASHBOARD), null,
null,
ActionType.ASSIGNED_TO_CUSTOMER, e, strDashboardId);
throw handleException(e);
}
DashboardId dashboardId = new DashboardId(toUUID(strDashboardId));
Dashboard dashboard = checkDashboardId(dashboardId, Operation.ASSIGN_TO_CUSTOMER);
Set<CustomerId> customerIds = customerIdFromStr(strCustomerIds, dashboard);
return tbDashboardService.updateDashboardCustomers(dashboard, customerIds, getCurrentUser());
}
@ApiOperation(value = "Adds the Dashboard Customers (addDashboardCustomers)",
notes = "Adds the list of Customers to the existing list of assignments for the Dashboard. Keeps previous assignments to customers that are not in the provided list. " +
"Returns the Dashboard object." + TENANT_AUTHORITY_PARAGRAPH,
"Returns the Dashboard object." + TENANT_AUTHORITY_PARAGRAPH,
produces = MediaType.APPLICATION_JSON_VALUE,
consumes = MediaType.APPLICATION_JSON_VALUE)
@PreAuthorize("hasAuthority('TENANT_ADMIN')")
@ -405,42 +280,10 @@ public class DashboardController extends BaseController {
@ApiParam(value = "JSON array with the list of customer ids")
@RequestBody String[] strCustomerIds) throws ThingsboardException {
checkParameter(DASHBOARD_ID, strDashboardId);
try {
DashboardId dashboardId = new DashboardId(toUUID(strDashboardId));
Dashboard dashboard = checkDashboardId(dashboardId, Operation.ASSIGN_TO_CUSTOMER);
Set<CustomerId> customerIds = new HashSet<>();
if (strCustomerIds != null) {
for (String strCustomerId : strCustomerIds) {
CustomerId customerId = new CustomerId(toUUID(strCustomerId));
if (!dashboard.isAssignedToCustomer(customerId)) {
customerIds.add(customerId);
}
}
}
if (customerIds.isEmpty()) {
return dashboard;
} else {
Dashboard savedDashboard = null;
for (CustomerId customerId : customerIds) {
savedDashboard = checkNotNull(dashboardService.assignDashboardToCustomer(getCurrentUser().getTenantId(), dashboardId, customerId));
ShortCustomerInfo customerInfo = savedDashboard.getAssignedCustomerInfo(customerId);
logEntityAction(dashboardId, savedDashboard,
customerId,
ActionType.ASSIGNED_TO_CUSTOMER, null, strDashboardId, customerId.toString(), customerInfo.getTitle());
sendEntityAssignToCustomerNotificationMsg(savedDashboard.getTenantId(), savedDashboard.getId(), customerId, EdgeEventActionType.ASSIGNED_TO_CUSTOMER);
}
return savedDashboard;
}
} catch (Exception e) {
logEntityAction(emptyId(EntityType.DASHBOARD), null,
null,
ActionType.ASSIGNED_TO_CUSTOMER, e, strDashboardId);
throw handleException(e);
}
DashboardId dashboardId = new DashboardId(toUUID(strDashboardId));
Dashboard dashboard = checkDashboardId(dashboardId, Operation.ASSIGN_TO_CUSTOMER);
Set<CustomerId> customerIds = customerIdFromStr(strCustomerIds, dashboard);
return tbDashboardService.addDashboardCustomers(dashboard, customerIds, getCurrentUser());
}
@ApiOperation(value = "Remove the Dashboard Customers (removeDashboardCustomers)",
@ -457,42 +300,10 @@ public class DashboardController extends BaseController {
@ApiParam(value = "JSON array with the list of customer ids")
@RequestBody String[] strCustomerIds) throws ThingsboardException {
checkParameter(DASHBOARD_ID, strDashboardId);
try {
DashboardId dashboardId = new DashboardId(toUUID(strDashboardId));
Dashboard dashboard = checkDashboardId(dashboardId, Operation.UNASSIGN_FROM_CUSTOMER);
Set<CustomerId> customerIds = new HashSet<>();
if (strCustomerIds != null) {
for (String strCustomerId : strCustomerIds) {
CustomerId customerId = new CustomerId(toUUID(strCustomerId));
if (dashboard.isAssignedToCustomer(customerId)) {
customerIds.add(customerId);
}
}
}
if (customerIds.isEmpty()) {
return dashboard;
} else {
Dashboard savedDashboard = null;
for (CustomerId customerId : customerIds) {
ShortCustomerInfo customerInfo = dashboard.getAssignedCustomerInfo(customerId);
savedDashboard = checkNotNull(dashboardService.unassignDashboardFromCustomer(getCurrentUser().getTenantId(), dashboardId, customerId));
logEntityAction(dashboardId, dashboard,
customerId,
ActionType.UNASSIGNED_FROM_CUSTOMER, null, strDashboardId, customerId.toString(), customerInfo.getTitle());
sendEntityAssignToCustomerNotificationMsg(savedDashboard.getTenantId(), savedDashboard.getId(), customerId, EdgeEventActionType.UNASSIGNED_FROM_CUSTOMER);
}
return savedDashboard;
}
} catch (Exception e) {
logEntityAction(emptyId(EntityType.DASHBOARD), null,
null,
ActionType.UNASSIGNED_FROM_CUSTOMER, e, strDashboardId);
throw handleException(e);
}
DashboardId dashboardId = new DashboardId(toUUID(strDashboardId));
Dashboard dashboard = checkDashboardId(dashboardId, Operation.UNASSIGN_FROM_CUSTOMER);
Set<CustomerId> customerIds = customerIdFromStr(strCustomerIds, dashboard);
return tbDashboardService.removeDashboardCustomers(dashboard, customerIds, getCurrentUser());
}
@ApiOperation(value = "Assign the Dashboard to Public Customer (assignDashboardToPublicCustomer)",
@ -510,25 +321,9 @@ public class DashboardController extends BaseController {
@ApiParam(value = DASHBOARD_ID_PARAM_DESCRIPTION)
@PathVariable(DASHBOARD_ID) String strDashboardId) throws ThingsboardException {
checkParameter(DASHBOARD_ID, strDashboardId);
try {
DashboardId dashboardId = new DashboardId(toUUID(strDashboardId));
Dashboard dashboard = checkDashboardId(dashboardId, Operation.ASSIGN_TO_CUSTOMER);
Customer publicCustomer = customerService.findOrCreatePublicCustomer(dashboard.getTenantId());
Dashboard savedDashboard = checkNotNull(dashboardService.assignDashboardToCustomer(getCurrentUser().getTenantId(), dashboardId, publicCustomer.getId()));
logEntityAction(dashboardId, savedDashboard,
publicCustomer.getId(),
ActionType.ASSIGNED_TO_CUSTOMER, null, strDashboardId, publicCustomer.getId().toString(), publicCustomer.getName());
return savedDashboard;
} catch (Exception e) {
logEntityAction(emptyId(EntityType.DASHBOARD), null,
null,
ActionType.ASSIGNED_TO_CUSTOMER, e, strDashboardId);
throw handleException(e);
}
DashboardId dashboardId = new DashboardId(toUUID(strDashboardId));
checkDashboardId(dashboardId, Operation.ASSIGN_TO_CUSTOMER);
return tbDashboardService.assignDashboardToPublicCustomer(dashboardId, getCurrentUser());
}
@ApiOperation(value = "Unassign the Dashboard from Public Customer (unassignDashboardFromPublicCustomer)",
@ -542,26 +337,9 @@ public class DashboardController extends BaseController {
@ApiParam(value = DASHBOARD_ID_PARAM_DESCRIPTION)
@PathVariable(DASHBOARD_ID) String strDashboardId) throws ThingsboardException {
checkParameter(DASHBOARD_ID, strDashboardId);
try {
DashboardId dashboardId = new DashboardId(toUUID(strDashboardId));
Dashboard dashboard = checkDashboardId(dashboardId, Operation.UNASSIGN_FROM_CUSTOMER);
Customer publicCustomer = customerService.findOrCreatePublicCustomer(dashboard.getTenantId());
Dashboard savedDashboard = checkNotNull(dashboardService.unassignDashboardFromCustomer(getCurrentUser().getTenantId(), dashboardId, publicCustomer.getId()));
logEntityAction(dashboardId, dashboard,
publicCustomer.getId(),
ActionType.UNASSIGNED_FROM_CUSTOMER, null, strDashboardId, publicCustomer.getId().toString(), publicCustomer.getName());
return savedDashboard;
} catch (Exception e) {
logEntityAction(emptyId(EntityType.DASHBOARD), null,
null,
ActionType.UNASSIGNED_FROM_CUSTOMER, e, strDashboardId);
throw handleException(e);
}
DashboardId dashboardId = new DashboardId(toUUID(strDashboardId));
Dashboard dashboard = checkDashboardId(dashboardId, Operation.UNASSIGN_FROM_CUSTOMER);
return tbDashboardService.unassignDashboardFromPublicCustomer(dashboard, getCurrentUser());
}
@ApiOperation(value = "Get Tenant Dashboards by System Administrator (getTenantDashboards)",
@ -775,6 +553,7 @@ public class DashboardController extends BaseController {
public void setTenantHomeDashboardInfo(
@ApiParam(value = "A JSON object that represents home dashboard id and other parameters", required = true)
@RequestBody HomeDashboardInfo homeDashboardInfo) throws ThingsboardException {
try {
if (homeDashboardInfo.getDashboardId() != null) {
checkDashboardId(homeDashboardInfo.getDashboardId(), Operation.READ);
@ -847,30 +626,13 @@ public class DashboardController extends BaseController {
@PathVariable(DASHBOARD_ID) String strDashboardId) throws ThingsboardException {
checkParameter("edgeId", strEdgeId);
checkParameter(DASHBOARD_ID, strDashboardId);
try {
EdgeId edgeId = new EdgeId(toUUID(strEdgeId));
Edge edge = checkEdgeId(edgeId, Operation.READ);
DashboardId dashboardId = new DashboardId(toUUID(strDashboardId));
checkDashboardId(dashboardId, Operation.READ);
Dashboard savedDashboard = checkNotNull(dashboardService.assignDashboardToEdge(getCurrentUser().getTenantId(), dashboardId, edgeId));
logEntityAction(dashboardId, savedDashboard,
null,
ActionType.ASSIGNED_TO_EDGE, null, strDashboardId, strEdgeId, edge.getName());
EdgeId edgeId = new EdgeId(toUUID(strEdgeId));
Edge edge = checkEdgeId(edgeId, Operation.READ);
sendEntityAssignToEdgeNotificationMsg(getTenantId(), edgeId, savedDashboard.getId(), EdgeEventActionType.ASSIGNED_TO_EDGE);
return savedDashboard;
} catch (Exception e) {
logEntityAction(emptyId(EntityType.DASHBOARD), null,
null,
ActionType.ASSIGNED_TO_EDGE, e, strDashboardId, strEdgeId);
throw handleException(e);
}
DashboardId dashboardId = new DashboardId(toUUID(strDashboardId));
checkDashboardId(dashboardId, Operation.READ);
return tbDashboardService.asignDashboardToEdge(dashboardId, edge, getCurrentUser());
}
@ApiOperation(value = "Unassign dashboard from edge (unassignDashboardFromEdge)",
@ -886,37 +648,22 @@ public class DashboardController extends BaseController {
@ResponseBody
public Dashboard unassignDashboardFromEdge(@PathVariable("edgeId") String strEdgeId,
@PathVariable(DASHBOARD_ID) String strDashboardId) throws ThingsboardException {
checkParameter("edgeId", strEdgeId);
checkParameter(EDGE_ID, strEdgeId);
checkParameter(DASHBOARD_ID, strDashboardId);
try {
EdgeId edgeId = new EdgeId(toUUID(strEdgeId));
Edge edge = checkEdgeId(edgeId, Operation.READ);
DashboardId dashboardId = new DashboardId(toUUID(strDashboardId));
Dashboard dashboard = checkDashboardId(dashboardId, Operation.READ);
Dashboard savedDashboard = checkNotNull(dashboardService.unassignDashboardFromEdge(getCurrentUser().getTenantId(), dashboardId, edgeId));
logEntityAction(dashboardId, dashboard,
null,
ActionType.UNASSIGNED_FROM_EDGE, null, strDashboardId, strEdgeId, edge.getName());
EdgeId edgeId = new EdgeId(toUUID(strEdgeId));
Edge edge = checkEdgeId(edgeId, Operation.READ);
sendEntityAssignToEdgeNotificationMsg(getTenantId(), edgeId, savedDashboard.getId(), EdgeEventActionType.UNASSIGNED_FROM_EDGE);
DashboardId dashboardId = new DashboardId(toUUID(strDashboardId));
Dashboard dashboard = checkDashboardId(dashboardId, Operation.READ);
return savedDashboard;
} catch (Exception e) {
logEntityAction(emptyId(EntityType.DASHBOARD), null,
null,
ActionType.UNASSIGNED_FROM_EDGE, e, strDashboardId, strEdgeId);
throw handleException(e);
}
return tbDashboardService.unassignDashboardFromEdge(dashboard, edge, getCurrentUser());
}
@ApiOperation(value = "Get Edge Dashboards (getEdgeDashboards)",
notes = "Returns a page of dashboard info objects assigned to the specified edge. "
+ DASHBOARD_INFO_DEFINITION + " " + PAGE_DATA_PARAMETERS + TENANT_OR_CUSTOMER_AUTHORITY_PARAGRAPH,
produces = MediaType.APPLICATION_JSON_VALUE)
notes = "Returns a page of dashboard info objects assigned to the specified edge. "
+ DASHBOARD_INFO_DEFINITION + " " + PAGE_DATA_PARAMETERS + TENANT_OR_CUSTOMER_AUTHORITY_PARAGRAPH,
produces = MediaType.APPLICATION_JSON_VALUE)
@PreAuthorize("hasAnyAuthority('TENANT_ADMIN', 'CUSTOMER_USER')")
@RequestMapping(value = "/edge/{edgeId}/dashboards", params = {"pageSize", "page"}, method = RequestMethod.GET)
@ResponseBody
@ -957,4 +704,17 @@ public class DashboardController extends BaseController {
throw handleException(e);
}
}
private Set<CustomerId> customerIdFromStr(String [] strCustomerIds, Dashboard dashboard) {
Set<CustomerId> customerIds = new HashSet<>();
if (strCustomerIds != null) {
for (String strCustomerId : strCustomerIds) {
CustomerId customerId = new CustomerId(UUID.fromString(strCustomerId));
if (dashboard.isAssignedToCustomer(customerId)) {
customerIds.add(customerId);
}
}
}
return customerIds;
}
}

2
application/src/main/java/org/thingsboard/server/controller/DeviceController.java

@ -206,7 +206,7 @@ public class DeviceController extends BaseController {
DeviceId deviceId = new DeviceId(toUUID(strDeviceId));
Device device = checkDeviceId(deviceId, Operation.DELETE);
try {
tbDeviceService.deleteDevice(device, getCurrentUser()).get();
tbDeviceService.delete(device, getCurrentUser()).get();
} catch (Exception e) {
throw handleException(e);
}

4
application/src/main/java/org/thingsboard/server/controller/EdgeController.java

@ -164,7 +164,7 @@ public class EdgeController extends BaseController {
accessControlService.checkPermission(getCurrentUser(), Resource.EDGE, operation, edge.getId(), edge);
return tbEdgeService.saveEdge(edge, edgeTemplateRootRuleChain, getCurrentUser());
return tbEdgeService.save(edge, edgeTemplateRootRuleChain, getCurrentUser());
}
@ApiOperation(value = "Delete edge (deleteEdge)",
@ -177,7 +177,7 @@ public class EdgeController extends BaseController {
checkParameter(EDGE_ID, strEdgeId);
EdgeId edgeId = new EdgeId(toUUID(strEdgeId));
Edge edge = checkEdgeId(edgeId, Operation.DELETE);
tbEdgeService.deleteEdge(edge, getCurrentUser());
tbEdgeService.delete(edge, getCurrentUser());
}
@ApiOperation(value = "Get Tenant Edges (getEdges)",

104
application/src/main/java/org/thingsboard/server/controller/QueueController.java

@ -20,17 +20,28 @@ import io.swagger.annotations.ApiParam;
import lombok.RequiredArgsConstructor;
import org.springframework.http.MediaType;
import org.springframework.security.access.prepost.PreAuthorize;
import org.springframework.web.bind.annotation.PathVariable;
import org.springframework.web.bind.annotation.RequestBody;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RequestMethod;
import org.springframework.web.bind.annotation.RequestParam;
import org.springframework.web.bind.annotation.ResponseBody;
import org.springframework.web.bind.annotation.RestController;
import org.thingsboard.server.common.data.exception.ThingsboardException;
import org.thingsboard.server.common.data.id.QueueId;
import org.thingsboard.server.common.data.page.PageData;
import org.thingsboard.server.common.data.page.PageLink;
import org.thingsboard.server.common.data.queue.Queue;
import org.thingsboard.server.common.msg.queue.ServiceType;
import org.thingsboard.server.queue.QueueService;
import org.thingsboard.server.queue.util.TbCoreComponent;
import org.thingsboard.server.service.entitiy.queue.TbQueueService;
import org.thingsboard.server.service.security.permission.Operation;
import org.thingsboard.server.service.security.permission.Resource;
import java.util.Collections;
import java.util.Set;
import java.util.UUID;
import java.util.stream.Collectors;
import static org.thingsboard.server.controller.ControllerConstants.QUEUE_SERVICE_TYPE_ALLOWABLE_VALUES;
import static org.thingsboard.server.controller.ControllerConstants.QUEUE_SERVICE_TYPE_DESCRIPTION;
@ -42,18 +53,103 @@ import static org.thingsboard.server.controller.ControllerConstants.TENANT_AUTHO
@RequiredArgsConstructor
public class QueueController extends BaseController {
private final QueueService queueService;
private final TbQueueService tbQueueService;
@ApiOperation(value = "Get queue names (getTenantQueuesByServiceType)",
notes = "Returns a set of unique queue names based on service type. " + TENANT_AUTHORITY_PARAGRAPH)
@PreAuthorize("hasAuthority('TENANT_ADMIN')")
@RequestMapping(value = "/tenant/queues", params = {"serviceType"}, produces = MediaType.APPLICATION_JSON_VALUE, method = RequestMethod.GET)
@RequestMapping(value = "/queues", params = {"serviceType"}, produces = MediaType.APPLICATION_JSON_VALUE, method = RequestMethod.GET)
@ResponseBody()
public Set<String> getTenantQueuesByServiceType(@ApiParam(value = QUEUE_SERVICE_TYPE_DESCRIPTION, allowableValues = QUEUE_SERVICE_TYPE_ALLOWABLE_VALUES)
@RequestParam String serviceType) throws ThingsboardException {
checkParameter("serviceType", serviceType);
try {
return queueService.getQueuesByServiceType(ServiceType.valueOf(serviceType));
ServiceType type = ServiceType.valueOf(serviceType);
switch (type) {
case TB_RULE_ENGINE:
return queueService.findQueuesByTenantId(getTenantId()).stream().map(Queue::getName).collect(Collectors.toSet());
default:
return Collections.emptySet();
}
} catch (Exception e) {
throw handleException(e);
}
}
@PreAuthorize("hasAnyAuthority('SYS_ADMIN', 'TENANT_ADMIN')")
@RequestMapping(value = "/queues", params = {"serviceType", "pageSize", "page"}, method = RequestMethod.GET)
@ResponseBody
public PageData<Queue> getTenantQueuesByServiceType(@RequestParam String serviceType,
@RequestParam int pageSize,
@RequestParam int page,
@RequestParam(required = false) String textSearch,
@RequestParam(required = false) String sortProperty,
@RequestParam(required = false) String sortOrder) throws ThingsboardException {
checkParameter("serviceType", serviceType);
try {
PageLink pageLink = createPageLink(pageSize, page, textSearch, sortProperty, sortOrder);
ServiceType type = ServiceType.valueOf(serviceType);
switch (type) {
case TB_RULE_ENGINE:
return queueService.findQueuesByTenantId(getTenantId(), pageLink);
default:
return new PageData<>();
}
} catch (Exception e) {
throw handleException(e);
}
}
@PreAuthorize("hasAnyAuthority('SYS_ADMIN', 'TENANT_ADMIN')")
@RequestMapping(value = "/queues/{queueId}", method = RequestMethod.GET)
@ResponseBody
public Queue getQueueById(@PathVariable("queueId") String queueIdStr) throws ThingsboardException {
checkParameter("queueId", queueIdStr);
try {
QueueId queueId = new QueueId(UUID.fromString(queueIdStr));
checkQueueId(queueId, Operation.READ);
return checkNotNull(queueService.findQueueById(getTenantId(), queueId));
} catch (
Exception e) {
throw handleException(e);
}
}
@PreAuthorize("hasAnyAuthority('SYS_ADMIN')")
@RequestMapping(value = "/queues", params = {"serviceType"}, method = RequestMethod.POST)
@ResponseBody
public Queue saveQueue(@RequestBody Queue queue,
@RequestParam String serviceType) throws ThingsboardException {
checkParameter("serviceType", serviceType);
try {
queue.setTenantId(getCurrentUser().getTenantId());
checkEntity(queue.getId(), queue, Resource.QUEUE);
ServiceType type = ServiceType.valueOf(serviceType);
switch (type) {
case TB_RULE_ENGINE:
queue.setTenantId(getTenantId());
Queue savedQueue = tbQueueService.saveQueue(queue);
checkNotNull(savedQueue);
return savedQueue;
default:
return null;
}
} catch (Exception e) {
throw handleException(e);
}
}
@PreAuthorize("hasAnyAuthority('SYS_ADMIN')")
@RequestMapping(value = "/queues/{queueId}", method = RequestMethod.DELETE)
@ResponseBody
public void deleteQueue(@PathVariable("queueId") String queueIdStr) throws ThingsboardException {
checkParameter("queueId", queueIdStr);
try {
QueueId queueId = new QueueId(toUUID(queueIdStr));
checkQueueId(queueId, Operation.DELETE);
tbQueueService.deleteQueue(getTenantId(), queueId);
} catch (Exception e) {
throw handleException(e);
}

1
application/src/main/java/org/thingsboard/server/controller/TenantController.java

@ -18,7 +18,6 @@ package org.thingsboard.server.controller;
import com.fasterxml.jackson.databind.node.ObjectNode;
import io.swagger.annotations.ApiOperation;
import io.swagger.annotations.ApiParam;
import lombok.AllArgsConstructor;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.springframework.http.HttpStatus;

11
application/src/main/java/org/thingsboard/server/controller/TenantProfileController.java

@ -17,6 +17,7 @@ package org.thingsboard.server.controller;
import io.swagger.annotations.ApiOperation;
import io.swagger.annotations.ApiParam;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.springframework.http.HttpStatus;
import org.springframework.security.access.prepost.PreAuthorize;
@ -37,6 +38,7 @@ import org.thingsboard.server.common.data.page.PageData;
import org.thingsboard.server.common.data.page.PageLink;
import org.thingsboard.server.common.data.plugin.ComponentLifecycleEvent;
import org.thingsboard.server.queue.util.TbCoreComponent;
import org.thingsboard.server.service.entitiy.tenant_profile.TbTenantProfileService;
import org.thingsboard.server.service.security.permission.Operation;
import org.thingsboard.server.service.security.permission.Resource;
@ -59,10 +61,13 @@ import static org.thingsboard.server.controller.ControllerConstants.UUID_WIKI_LI
@TbCoreComponent
@RequestMapping("/api")
@Slf4j
@RequiredArgsConstructor
public class TenantProfileController extends BaseController {
private static final String TENANT_PROFILE_INFO_DESCRIPTION = "Tenant Profile Info is a lightweight object that contains only id and name of the profile. ";
private final TbTenantProfileService tbTenantProfileService;
@ApiOperation(value = "Get Tenant Profile (getTenantProfileById)",
notes = "Fetch the Tenant Profile object based on the provided Tenant Profile Id. " + SYSTEM_AUTHORITY_PARAGRAPH)
@PreAuthorize("hasAnyAuthority('SYS_ADMIN')")
@ -171,14 +176,16 @@ public class TenantProfileController extends BaseController {
@RequestBody TenantProfile tenantProfile) throws ThingsboardException {
try {
boolean newTenantProfile = tenantProfile.getId() == null;
TenantProfile oldProfile;
if (newTenantProfile) {
accessControlService
.checkPermission(getCurrentUser(), Resource.TENANT_PROFILE, Operation.CREATE);
oldProfile = null;
} else {
checkEntityId(tenantProfile.getId(), Operation.WRITE);
oldProfile = checkTenantProfileId(tenantProfile.getId(), Operation.WRITE);
}
tenantProfile = checkNotNull(tenantProfileService.saveTenantProfile(getTenantId(), tenantProfile));
tenantProfile = checkNotNull(tbTenantProfileService.saveTenantProfile(getTenantId(), tenantProfile, oldProfile));
tenantProfileCache.put(tenantProfile);
tbClusterService.onTenantProfileChange(tenantProfile, null);
tbClusterService.broadcastEntityStateChangeEvent(TenantId.SYS_TENANT_ID, tenantProfile.getId(),

1
application/src/main/java/org/thingsboard/server/install/ThingsboardInstallService.java

@ -258,6 +258,7 @@ public class ThingsboardInstallService {
systemDataLoaderService.createAdminSettings();
systemDataLoaderService.loadSystemWidgets();
systemDataLoaderService.createOAuth2Templates();
systemDataLoaderService.createQueues();
// systemDataLoaderService.loadSystemPlugins();
// systemDataLoaderService.loadSystemRules();

2
application/src/main/java/org/thingsboard/server/service/edge/EdgeBulkImportService.java

@ -76,7 +76,7 @@ public class EdgeBulkImportService extends AbstractBulkImportService<Edge> {
@Override
protected Edge saveEntity(SecurityUser user, Edge entity, Map<BulkImportColumnType, String> fields) {
RuleChain edgeTemplateRootRuleChain = ruleChainService.getEdgeTemplateRootRuleChain(user.getTenantId());
return tbEdgeService.saveEdge(entity, edgeTemplateRootRuleChain, user);
return tbEdgeService.save(entity, edgeTemplateRootRuleChain, user);
}
@Override

33
application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/TelemetryEdgeProcessor.java

@ -44,6 +44,7 @@ import org.thingsboard.server.common.data.id.DeviceId;
import org.thingsboard.server.common.data.id.EdgeId;
import org.thingsboard.server.common.data.id.EntityId;
import org.thingsboard.server.common.data.id.EntityViewId;
import org.thingsboard.server.common.data.id.QueueId;
import org.thingsboard.server.common.data.id.RuleChainId;
import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.data.id.UserId;
@ -51,7 +52,6 @@ import org.thingsboard.server.common.data.kv.AttributeKey;
import org.thingsboard.server.common.data.kv.AttributeKvEntry;
import org.thingsboard.server.common.msg.TbMsg;
import org.thingsboard.server.common.msg.TbMsgMetaData;
import org.thingsboard.server.common.msg.queue.ServiceQueue;
import org.thingsboard.server.common.msg.session.SessionMsgType;
import org.thingsboard.server.common.transport.adaptor.JsonConverter;
import org.thingsboard.server.common.transport.util.JsonUtils;
@ -134,24 +134,23 @@ public class TelemetryEdgeProcessor extends BaseEdgeProcessor {
return metaData;
}
private Pair<String, RuleChainId> getDefaultQueueNameAndRuleChainId(TenantId tenantId, EntityId entityId) {
private Pair<QueueId, RuleChainId> getDefaultQueueNameAndRuleChainId(TenantId tenantId, EntityId entityId) {
if (EntityType.DEVICE.equals(entityId.getEntityType())) {
DeviceProfile deviceProfile = deviceProfileCache.get(tenantId, new DeviceId(entityId.getId()));
RuleChainId ruleChainId;
String queueName;
QueueId queueId;
if (deviceProfile == null) {
log.warn("[{}] Device profile is null!", entityId);
ruleChainId = null;
queueName = ServiceQueue.MAIN;
queueId = null;
} else {
ruleChainId = deviceProfile.getDefaultRuleChainId();
String defaultQueueName = deviceProfile.getDefaultQueueName();
queueName = defaultQueueName != null ? defaultQueueName : ServiceQueue.MAIN;
queueId = deviceProfile.getDefaultQueueId();
}
return new ImmutablePair<>(queueName, ruleChainId);
return new ImmutablePair<>(queueId, ruleChainId);
} else {
return new ImmutablePair<>(ServiceQueue.MAIN, null);
return new ImmutablePair<>(null, null);
}
}
@ -160,10 +159,10 @@ public class TelemetryEdgeProcessor extends BaseEdgeProcessor {
for (TransportProtos.TsKvListProto tsKv : msg.getTsKvListList()) {
JsonObject json = JsonUtils.getJsonObject(tsKv.getKvList());
metaData.putValue("ts", tsKv.getTs() + "");
Pair<String, RuleChainId> defaultQueueAndRuleChain = getDefaultQueueNameAndRuleChainId(tenantId, entityId);
String queueName = defaultQueueAndRuleChain.getKey();
Pair<QueueId, RuleChainId> defaultQueueAndRuleChain = getDefaultQueueNameAndRuleChainId(tenantId, entityId);
QueueId queueId = defaultQueueAndRuleChain.getKey();
RuleChainId ruleChainId = defaultQueueAndRuleChain.getValue();
TbMsg tbMsg = TbMsg.newMsg(queueName, SessionMsgType.POST_TELEMETRY_REQUEST.name(), entityId, customerId, metaData, gson.toJson(json), ruleChainId, null);
TbMsg tbMsg = TbMsg.newMsg(queueId, SessionMsgType.POST_TELEMETRY_REQUEST.name(), entityId, customerId, metaData, gson.toJson(json), ruleChainId, null);
tbClusterService.pushMsgToRuleEngine(tenantId, tbMsg.getOriginator(), tbMsg, new TbQueueCallback() {
@Override
public void onSuccess(TbQueueMsgMetadata metadata) {
@ -183,10 +182,10 @@ public class TelemetryEdgeProcessor extends BaseEdgeProcessor {
private ListenableFuture<Void> processPostAttributes(TenantId tenantId, CustomerId customerId, EntityId entityId, TransportProtos.PostAttributeMsg msg, TbMsgMetaData metaData) {
SettableFuture<Void> futureToSet = SettableFuture.create();
JsonObject json = JsonUtils.getJsonObject(msg.getKvList());
Pair<String, RuleChainId> defaultQueueAndRuleChain = getDefaultQueueNameAndRuleChainId(tenantId, entityId);
String queueName = defaultQueueAndRuleChain.getKey();
Pair<QueueId, RuleChainId> defaultQueueAndRuleChain = getDefaultQueueNameAndRuleChainId(tenantId, entityId);
QueueId queueId = defaultQueueAndRuleChain.getKey();
RuleChainId ruleChainId = defaultQueueAndRuleChain.getValue();
TbMsg tbMsg = TbMsg.newMsg(queueName, SessionMsgType.POST_ATTRIBUTES_REQUEST.name(), entityId, customerId, metaData, gson.toJson(json), ruleChainId, null);
TbMsg tbMsg = TbMsg.newMsg(queueId, SessionMsgType.POST_ATTRIBUTES_REQUEST.name(), entityId, customerId, metaData, gson.toJson(json), ruleChainId, null);
tbClusterService.pushMsgToRuleEngine(tenantId, tbMsg.getOriginator(), tbMsg, new TbQueueCallback() {
@Override
public void onSuccess(TbQueueMsgMetadata metadata) {
@ -210,10 +209,10 @@ public class TelemetryEdgeProcessor extends BaseEdgeProcessor {
Futures.addCallback(future, new FutureCallback<>() {
@Override
public void onSuccess(@Nullable List<String> keys) {
Pair<String, RuleChainId> defaultQueueAndRuleChain = getDefaultQueueNameAndRuleChainId(tenantId, entityId);
String queueName = defaultQueueAndRuleChain.getKey();
Pair<QueueId, RuleChainId> defaultQueueAndRuleChain = getDefaultQueueNameAndRuleChainId(tenantId, entityId);
QueueId queueId = defaultQueueAndRuleChain.getKey();
RuleChainId ruleChainId = defaultQueueAndRuleChain.getValue();
TbMsg tbMsg = TbMsg.newMsg(queueName, DataConstants.ATTRIBUTES_UPDATED, entityId, customerId, metaData, gson.toJson(json), ruleChainId, null);
TbMsg tbMsg = TbMsg.newMsg(queueId, DataConstants.ATTRIBUTES_UPDATED, entityId, customerId, metaData, gson.toJson(json), ruleChainId, null);
tbClusterService.pushMsgToRuleEngine(tenantId, tbMsg.getOriginator(), tbMsg, new TbQueueCallback() {
@Override
public void onSuccess(TbQueueMsgMetadata metadata) {

10
application/src/main/java/org/thingsboard/server/service/entitiy/AbstractTbEntityService.java

@ -15,14 +15,12 @@
*/
package org.thingsboard.server.service.entitiy;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.google.common.util.concurrent.Futures;
import com.google.common.util.concurrent.ListenableFuture;
import lombok.Getter;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.thingsboard.server.cluster.TbClusterService;
import org.thingsboard.server.common.data.EntityType;
import org.thingsboard.server.common.data.HasName;
import org.thingsboard.server.common.data.User;
@ -43,6 +41,7 @@ import org.thingsboard.server.common.data.page.TimePageLink;
import org.thingsboard.server.dao.alarm.AlarmService;
import org.thingsboard.server.dao.asset.AssetService;
import org.thingsboard.server.dao.customer.CustomerService;
import org.thingsboard.server.dao.dashboard.DashboardService;
import org.thingsboard.server.dao.device.ClaimDevicesService;
import org.thingsboard.server.dao.device.DeviceCredentialsService;
import org.thingsboard.server.dao.device.DeviceService;
@ -50,6 +49,7 @@ import org.thingsboard.server.dao.edge.EdgeService;
import org.thingsboard.server.dao.exception.DataValidationException;
import org.thingsboard.server.dao.exception.IncorrectParameterException;
import org.thingsboard.server.dao.model.ModelConstants;
import org.thingsboard.server.dao.queue.QueueService;
import org.thingsboard.server.dao.rule.RuleChainService;
import org.thingsboard.server.dao.tenant.TbTenantProfileCache;
import org.thingsboard.server.dao.tenant.TenantService;
@ -69,8 +69,6 @@ public abstract class AbstractTbEntityService {
protected static final int DEFAULT_PAGE_SIZE = 1000;
private static final ObjectMapper json = new ObjectMapper();
@Value("${server.log_controller_error_stack_trace}")
@Getter
private boolean logControllerErrorStackTrace;
@ -106,6 +104,10 @@ public abstract class AbstractTbEntityService {
protected RuleChainService ruleChainService;
@Autowired
protected EdgeNotificationService edgeNotificationService;
@Autowired
protected QueueService queueService;
@Autowired
protected DashboardService dashboardService;
protected ListenableFuture<Void> removeAlarmsByEntityId(TenantId tenantId, EntityId entityId) {
ListenableFuture<PageData<AlarmInfo>> alarmsFuture =

48
application/src/main/java/org/thingsboard/server/service/entitiy/DefaultTbNotificationEntityService.java

@ -22,7 +22,6 @@ import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Service;
import org.thingsboard.rule.engine.api.msg.DeviceCredentialsUpdateNotificationMsg;
import org.thingsboard.server.cluster.TbClusterService;
import org.thingsboard.server.common.data.Customer;
import org.thingsboard.server.common.data.DataConstants;
import org.thingsboard.server.common.data.Device;
import org.thingsboard.server.common.data.HasName;
@ -73,7 +72,16 @@ public class DefaultTbNotificationEntityService implements TbNotificationEntityS
List<EdgeId> relatedEdgeIds,
SecurityUser user, Object... additionalInfo) {
logEntityAction(tenantId, entityId, entity, customerId, actionType, user, additionalInfo);
sendDeleteNotificationMsg(tenantId, entityId, entity, relatedEdgeIds);
sendDeleteNotificationMsg(tenantId, entityId, entity, relatedEdgeIds);
}
public void notifyDeleteAlarm(TenantId tenantId, Alarm alarm, EntityId originatorId,
CustomerId customerId, ActionType actionType,
List<EdgeId> relatedEdgeIds,
SecurityUser user,
String body, Object... additionalInfo) {
logEntityAction(tenantId, originatorId, alarm, customerId, actionType, user, additionalInfo);
sendAlarmDeleteNotificationMsg(tenantId, alarm, relatedEdgeIds, body);
}
@Override
@ -126,7 +134,7 @@ public class DefaultTbNotificationEntityService implements TbNotificationEntityS
gatewayNotificationsService.onDeviceDeleted(device);
tbClusterService.onDeviceDeleted(device, null);
notifyDeleteEntity(tenantId, deviceId, device, customerId, ActionType.DELETED, relatedEdgeIds, user, false, additionalInfo);
notifyDeleteEntity(tenantId, deviceId, device, customerId, ActionType.DELETED, relatedEdgeIds, user, additionalInfo);
}
@Override
@ -145,9 +153,9 @@ public class DefaultTbNotificationEntityService implements TbNotificationEntityS
}
@Override
public <E extends HasName, I extends EntityId> void notifyCreateOrUpdateEntity(TenantId tenantId, I entityId, E entity, CustomerId customerId, ActionType actionType, SecurityUser user, Object... additionalInfo) {
public <E extends HasName, I extends EntityId> void notifyCreateOrUpdateEntity(TenantId tenantId, I entityId, E entity, CustomerId customerId, ActionType actionType, SecurityUser user, Object... additionalInfo) {
logEntityAction(tenantId, entityId, entity, customerId, actionType, user, additionalInfo);
if (actionType == ActionType.UPDATED) {
if (actionType == ActionType.UPDATED) {
sendEntityNotificationMsg(tenantId, entityId, EdgeEventActionType.UPDATED);
}
}
@ -191,20 +199,7 @@ public class DefaultTbNotificationEntityService implements TbNotificationEntityS
@Override
public void notifyCreateOrUpdateAlarm(Alarm alarm, ActionType actionType, SecurityUser user, Object... additionalInfo) {
logEntityAction(alarm.getTenantId(), alarm.getOriginator(), alarm, alarm.getCustomerId(), actionType, user, additionalInfo);
sendEntityNotificationMsg(alarm.getTenantId(), alarm.getId(), edgeTypeByActionType (actionType));
}
@Override
public void notifyDeleteAlarm(Alarm alarm, SecurityUser user, List<EdgeId> relatedEdgeIds) {
logEntityAction(alarm.getTenantId(), alarm.getOriginator(), alarm, alarm.getCustomerId(), ActionType.ALARM_DELETE, user, null);
sendAlarmDeleteNotificationMsg(alarm, relatedEdgeIds);
}
@Override
public void notifyDeleteCustomer(Customer customer, SecurityUser user, List<EdgeId> edgeIds) {
logEntityAction(customer.getTenantId(), customer.getId(), customer, customer.getId(), ActionType.DELETED, user, null);
sendDeleteNotificationMsg(customer.getTenantId(), customer.getId(), customer, edgeIds);
tbClusterService.broadcastEntityStateChangeEvent(customer.getTenantId(), customer.getId(), ComponentLifecycleEvent.DELETED);
sendEntityNotificationMsg(alarm.getTenantId(), alarm.getId(), edgeTypeByActionType(actionType));
}
private <E extends HasName, I extends EntityId> void logEntityAction(TenantId tenantId, I entityId, E entity, CustomerId customerId,
@ -233,19 +228,20 @@ public class DefaultTbNotificationEntityService implements TbNotificationEntityS
}
}
protected <E extends HasName, I extends EntityId> void sendDeleteNotificationMsg(TenantId tenantId, I entityId, E entity, List<EdgeId> edgeIds) {
protected void sendAlarmDeleteNotificationMsg(TenantId tenantId, Alarm alarm, List<EdgeId> edgeIds, String body) {
try {
sendDeleteNotificationMsg(tenantId, entityId, edgeIds, null);
sendDeleteNotificationMsg(tenantId, alarm.getId(), edgeIds, body);
} catch (Exception e) {
log.warn("Failed to push delete " + entity.getClass().getName() + " msg to core: {}", entity, e);
log.warn("Failed to push delete msg to core: {}", alarm, e);
}
}
protected void sendAlarmDeleteNotificationMsg(Alarm alarm, List<EdgeId> relatedEdgeIds) {
protected <E extends HasName, I extends EntityId> void sendDeleteNotificationMsg(TenantId tenantId, I entityId, E entity,
List<EdgeId> edgeIds) {
try {
sendDeleteNotificationMsg(alarm.getTenantId(), alarm.getId(), relatedEdgeIds, json.writeValueAsString(alarm));
sendDeleteNotificationMsg(tenantId, entityId, edgeIds, null);
} catch (Exception e) {
log.warn("Failed to push delete alarm msg to core: {}", alarm, e);
log.warn("Failed to push delete msg to core: {}", entity, e);
}
}
@ -289,7 +285,7 @@ public class DefaultTbNotificationEntityService implements TbNotificationEntityS
return null;
}
private EdgeEventActionType edgeTypeByActionType (ActionType actionType) {
private EdgeEventActionType edgeTypeByActionType(ActionType actionType) {
switch (actionType) {
case ADDED:
return EdgeEventActionType.ADDED;

4
application/src/main/java/org/thingsboard/server/service/entitiy/SimpleTbEntityService.java

@ -20,6 +20,8 @@ import org.thingsboard.server.service.security.model.SecurityUser;
public interface SimpleTbEntityService<T> {
T save(T entity, SecurityUser user) throws ThingsboardException;
T save(T entity, SecurityUser user) throws ThingsboardException;
void delete (T entity, SecurityUser user) throws ThingsboardException;
}

17
application/src/main/java/org/thingsboard/server/service/entitiy/TbNotificationEntityService.java

@ -15,7 +15,6 @@
*/
package org.thingsboard.server.service.entitiy;
import org.thingsboard.server.common.data.Customer;
import org.thingsboard.server.common.data.Device;
import org.thingsboard.server.common.data.HasName;
import org.thingsboard.server.common.data.Tenant;
@ -44,9 +43,15 @@ public interface TbNotificationEntityService {
CustomerId customerId, ActionType actionType,
SecurityUser user, Object... additionalInfo);
<E extends HasName, I extends EntityId> void notifyDeleteEntity(TenantId tenantId, I entityId, E entity, CustomerId customerId,
ActionType actionType, List<EdgeId> relatedEdgeIds, SecurityUser user,
Object... additionalInfo);
<E extends HasName, I extends EntityId> void notifyDeleteEntity(TenantId tenantId, I entityId, E entity,
CustomerId customerId, ActionType actionType,
List<EdgeId> relatedEdgeIds,
SecurityUser user, Object... additionalInfo);
void notifyDeleteAlarm(TenantId tenantId, Alarm alarm, EntityId originatorId,
CustomerId customerId, ActionType actionType,
List<EdgeId> relatedEdgeIds,
SecurityUser user, String body, Object... additionalInfo);
<E extends HasName, I extends EntityId> void notifyAssignOrUnassignEntityToCustomer(TenantId tenantId, I entityId,
CustomerId customerId, E entity,
@ -80,8 +85,4 @@ public interface TbNotificationEntityService {
void notifyEdge(TenantId tenantId, EdgeId edgeId, CustomerId customerId, Edge edge, ActionType actionType, SecurityUser user, Object... additionalInfo);
void notifyCreateOrUpdateAlarm(Alarm alarm, ActionType actionType, SecurityUser user, Object... additionalInfo);
void notifyDeleteAlarm(Alarm alarm, SecurityUser user, List<EdgeId> relatedEdgeIds);
void notifyDeleteCustomer(Customer customer, SecurityUser user, List<EdgeId> relatedEdgeIds);
}

14
application/src/main/java/org/thingsboard/server/service/entitiy/alarm/DefaultTbAlarmService.java

@ -17,6 +17,7 @@ package org.thingsboard.server.service.entitiy.alarm;
import lombok.AllArgsConstructor;
import org.springframework.stereotype.Service;
import org.thingsboard.common.util.JacksonUtil;
import org.thingsboard.server.common.data.EntityType;
import org.thingsboard.server.common.data.alarm.Alarm;
import org.thingsboard.server.common.data.alarm.AlarmStatus;
@ -77,8 +78,13 @@ public class DefaultTbAlarmService extends AbstractTbEntityService implements Tb
@Override
public Boolean delete(Alarm alarm, SecurityUser user) throws ThingsboardException {
List<EdgeId> relatedEdgeIds = findRelatedEdgeIds(alarm.getTenantId(), alarm.getOriginator());
notificationEntityService.notifyDeleteAlarm(alarm, user, relatedEdgeIds);
return alarmService.deleteAlarm(alarm.getTenantId(), alarm.getId()).isSuccessful();
try {
List<EdgeId> relatedEdgeIds = findRelatedEdgeIds(user.getTenantId(), alarm.getOriginator());
notificationEntityService.notifyDeleteAlarm(user.getTenantId(), alarm, alarm.getOriginator(), user.getCustomerId(),
ActionType.DELETED, relatedEdgeIds, user, JacksonUtil.OBJECT_MAPPER.writeValueAsString(alarm));
return alarmService.deleteAlarm(user.getTenantId(), alarm.getId()).isSuccessful();
} catch (Exception e) {
throw handleException(e);
}
}
}
}

6
application/src/main/java/org/thingsboard/server/service/entitiy/alarm/TbAlarmService.java

@ -16,12 +16,12 @@
package org.thingsboard.server.service.entitiy.alarm;
import org.thingsboard.server.common.data.alarm.Alarm;
import org.thingsboard.server.common.data.asset.Asset;
import org.thingsboard.server.common.data.exception.ThingsboardException;
import org.thingsboard.server.service.entitiy.SimpleTbEntityService;
import org.thingsboard.server.service.security.model.SecurityUser;
public interface TbAlarmService extends SimpleTbEntityService<Alarm> {
public interface TbAlarmService {
Alarm save(Alarm entity, SecurityUser user) throws ThingsboardException;
void ack(Alarm alarm, SecurityUser user) throws ThingsboardException;

6
application/src/main/java/org/thingsboard/server/service/entitiy/asset/DefaultTbAssetService.java

@ -39,6 +39,7 @@ import java.util.List;
@TbCoreComponent
@AllArgsConstructor
public class DefaultTbAssetService extends AbstractTbEntityService implements TbAssetService {
@Override
public Asset save(Asset asset, SecurityUser user) throws ThingsboardException {
ActionType actionType = asset.getId() == null ? ActionType.ADDED : ActionType.UPDATED;
@ -60,12 +61,13 @@ public class DefaultTbAssetService extends AbstractTbEntityService implements Tb
try {
List<EdgeId> relatedEdgeIds = findRelatedEdgeIds(tenantId, assetId);
assetService.deleteAsset(tenantId, assetId);
notificationEntityService.notifyDeleteEntity(tenantId, assetId, asset, asset.getCustomerId(), ActionType.DELETED, relatedEdgeIds, user, false, asset.toString());
notificationEntityService.notifyDeleteEntity(tenantId, assetId, asset, asset.getCustomerId(), ActionType.DELETED,
relatedEdgeIds, user, assetId.toString());
return removeAlarmsByEntityId(tenantId, assetId);
} catch (Exception e) {
notificationEntityService.notifyEntity(tenantId, emptyId(EntityType.ASSET), null, null,
ActionType.DELETED, user, e, asset.toString());
ActionType.DELETED, user, e, assetId.toString());
throw handleException(e);
}
}

5
application/src/main/java/org/thingsboard/server/service/entitiy/asset/TbAssetService.java

@ -22,10 +22,11 @@ import org.thingsboard.server.common.data.edge.Edge;
import org.thingsboard.server.common.data.exception.ThingsboardException;
import org.thingsboard.server.common.data.id.AssetId;
import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.service.entitiy.SimpleTbEntityService;
import org.thingsboard.server.service.security.model.SecurityUser;
public interface TbAssetService extends SimpleTbEntityService<Asset> {
public interface TbAssetService {
Asset save(Asset asset, SecurityUser user) throws ThingsboardException;
ListenableFuture<Void> delete(Asset asset, SecurityUser user) throws ThingsboardException;

18
application/src/main/java/org/thingsboard/server/service/entitiy/customer/DefaultTbCustomerService.java

@ -17,12 +17,15 @@ package org.thingsboard.server.service.entitiy.customer;
import lombok.AllArgsConstructor;
import org.springframework.stereotype.Service;
import org.thingsboard.server.cluster.TbClusterService;
import org.thingsboard.server.common.data.Customer;
import org.thingsboard.server.common.data.EntityType;
import org.thingsboard.server.common.data.audit.ActionType;
import org.thingsboard.server.common.data.exception.ThingsboardException;
import org.thingsboard.server.common.data.id.CustomerId;
import org.thingsboard.server.common.data.id.EdgeId;
import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.data.plugin.ComponentLifecycleEvent;
import org.thingsboard.server.queue.util.TbCoreComponent;
import org.thingsboard.server.service.entitiy.AbstractTbEntityService;
import org.thingsboard.server.service.security.model.SecurityUser;
@ -34,6 +37,8 @@ import java.util.List;
@AllArgsConstructor
public class DefaultTbCustomerService extends AbstractTbEntityService implements TbCustomerService {
private final TbClusterService tbClusterService;
@Override
public Customer save(Customer customer, SecurityUser user) throws ThingsboardException {
ActionType actionType = customer.getId() == null ? ActionType.ADDED : ActionType.UPDATED;
@ -48,15 +53,20 @@ public class DefaultTbCustomerService extends AbstractTbEntityService implements
}
}
@Override
public void delete(Customer customer, SecurityUser user) throws ThingsboardException {
TenantId tenantId = customer.getTenantId();
CustomerId customerId = customer.getId();
try {
List<EdgeId> relatedEdgeIds = findRelatedEdgeIds(tenantId, customer.getId());
customerService.deleteCustomer(tenantId, customer.getId());
notificationEntityService.notifyDeleteCustomer(customer, user, relatedEdgeIds);
List<EdgeId> relatedEdgeIds = findRelatedEdgeIds(tenantId, customerId);
customerService.deleteCustomer(tenantId, customerId);
notificationEntityService.notifyDeleteEntity(tenantId, customerId, customer, customerId,
ActionType.DELETED, relatedEdgeIds, user, customerId.toString());
tbClusterService.broadcastEntityStateChangeEvent(tenantId, customerId, ComponentLifecycleEvent.DELETED);
} catch (Exception e) {
notificationEntityService.notifyEntity(tenantId, emptyId(EntityType.CUSTOMER), null, null, ActionType.DELETED, user, e);
notificationEntityService.notifyEntity(tenantId, emptyId(EntityType.CUSTOMER), null, null,
ActionType.DELETED, user, e, customerId.toString());
throw handleException(e);
}
}

4
application/src/main/java/org/thingsboard/server/service/entitiy/customer/TbCustomerService.java

@ -16,12 +16,8 @@
package org.thingsboard.server.service.entitiy.customer;
import org.thingsboard.server.common.data.Customer;
import org.thingsboard.server.common.data.exception.ThingsboardException;
import org.thingsboard.server.service.entitiy.SimpleTbEntityService;
import org.thingsboard.server.service.security.model.SecurityUser;
public interface TbCustomerService extends SimpleTbEntityService<Customer> {
void delete(Customer customer, SecurityUser user) throws ThingsboardException;
}

275
application/src/main/java/org/thingsboard/server/service/entitiy/dashboard/DefaultTbDashboardService.java

@ -0,0 +1,275 @@
/**
* Copyright © 2016-2022 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.service.entitiy.dashboard;
import lombok.AllArgsConstructor;
import org.springframework.stereotype.Service;
import org.thingsboard.server.common.data.Customer;
import org.thingsboard.server.common.data.Dashboard;
import org.thingsboard.server.common.data.EntityType;
import org.thingsboard.server.common.data.ShortCustomerInfo;
import org.thingsboard.server.common.data.audit.ActionType;
import org.thingsboard.server.common.data.edge.Edge;
import org.thingsboard.server.common.data.edge.EdgeEventActionType;
import org.thingsboard.server.common.data.exception.ThingsboardException;
import org.thingsboard.server.common.data.id.CustomerId;
import org.thingsboard.server.common.data.id.DashboardId;
import org.thingsboard.server.common.data.id.EdgeId;
import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.queue.util.TbCoreComponent;
import org.thingsboard.server.service.entitiy.AbstractTbEntityService;
import org.thingsboard.server.service.security.model.SecurityUser;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
@Service
@TbCoreComponent
@AllArgsConstructor
public class DefaultTbDashboardService extends AbstractTbEntityService implements TbDashboardService {
@Override
public Dashboard save(Dashboard dashboard, SecurityUser user) throws ThingsboardException {
ActionType actionType = dashboard.getId() == null ? ActionType.ADDED : ActionType.UPDATED;
TenantId tenantId = dashboard.getTenantId();
try {
Dashboard savedDashboard = checkNotNull(dashboardService.saveDashboard(dashboard));
notificationEntityService.notifyCreateOrUpdateEntity(tenantId, savedDashboard.getId(), savedDashboard,
null, actionType, user);
return savedDashboard;
} catch (Exception e) {
notificationEntityService.notifyEntity(tenantId, emptyId(EntityType.DASHBOARD), dashboard, null, actionType, user, e);
throw handleException(e);
}
}
@Override
public void delete(Dashboard dashboard, SecurityUser user) throws ThingsboardException {
TenantId tenantId = dashboard.getTenantId();
DashboardId dashboardId = dashboard.getId();
try {
List<EdgeId> relatedEdgeIds = findRelatedEdgeIds(tenantId, dashboardId);
dashboardService.deleteDashboard(tenantId, dashboardId);
notificationEntityService.notifyDeleteEntity(tenantId, dashboardId, dashboard, user.getCustomerId(),
ActionType.DELETED, relatedEdgeIds, user, dashboardId.toString());
} catch (Exception e) {
notificationEntityService.notifyEntity(tenantId, emptyId(EntityType.DASHBOARD), null, null,
ActionType.DELETED, user, e, dashboardId.toString());
throw handleException(e);
}
}
@Override
public Dashboard assignDashboardToCustomer(DashboardId dashboardId, Customer customer, SecurityUser user) throws ThingsboardException {
ActionType actionType = ActionType.ASSIGNED_TO_CUSTOMER;
CustomerId customerId = customer.getId();
try {
Dashboard savedDashboard = checkNotNull(dashboardService.assignDashboardToCustomer(user.getTenantId(), dashboardId, customerId));
notificationEntityService.notifyAssignOrUnassignEntityToCustomer(user.getTenantId(), dashboardId, customerId, savedDashboard,
actionType, EdgeEventActionType.ASSIGNED_TO_CUSTOMER, user, true, customerId.toString(), customer.getName());
return savedDashboard;
} catch (Exception e) {
notificationEntityService.notifyEntity(user.getTenantId(), emptyId(EntityType.DASHBOARD), null, null,
actionType, user, e, dashboardId.toString(), customerId.toString());
throw handleException(e);
}
}
@Override
public Dashboard assignDashboardToPublicCustomer(DashboardId dashboardId, SecurityUser user) throws ThingsboardException {
ActionType actionType = ActionType.ASSIGNED_TO_CUSTOMER;
try {
Customer publicCustomer = customerService.findOrCreatePublicCustomer(user.getTenantId());
Dashboard savedDashboard = checkNotNull(dashboardService.assignDashboardToCustomer(user.getTenantId(), dashboardId, publicCustomer.getId()));
notificationEntityService.notifyAssignOrUnassignEntityToCustomer(user.getTenantId(), dashboardId, user.getCustomerId(), savedDashboard,
actionType, null, user, false, dashboardId.toString(),
publicCustomer.getId().toString(), publicCustomer.getName());
return savedDashboard;
} catch (Exception e) {
notificationEntityService.notifyEntity(user.getTenantId(), emptyId(EntityType.DASHBOARD), null, null,
actionType, user, e, dashboardId.toString());
throw handleException(e);
}
}
@Override
public Dashboard unassignDashboardFromPublicCustomer(Dashboard dashboard, SecurityUser user) throws ThingsboardException {
ActionType actionType = ActionType.UNASSIGNED_FROM_CUSTOMER;
try {
Customer publicCustomer = customerService.findOrCreatePublicCustomer(dashboard.getTenantId());
Dashboard savedDashboard = checkNotNull(dashboardService.unassignDashboardFromCustomer(user.getTenantId(), dashboard.getId(), publicCustomer.getId()));
notificationEntityService.notifyAssignOrUnassignEntityToCustomer(user.getTenantId(), dashboard.getId(), user.getCustomerId(), dashboard,
actionType, null, user, false, dashboard.getId().toString(),
publicCustomer.getId().toString(), publicCustomer.getName());
return savedDashboard;
} catch (Exception e) {
notificationEntityService.notifyEntity(user.getTenantId(), emptyId(EntityType.DASHBOARD), null, null,
actionType, user, e, dashboard.getId().toString());
throw handleException(e);
}
}
@Override
public Dashboard updateDashboardCustomers(Dashboard dashboard, Set<CustomerId> customerIds, SecurityUser user) throws ThingsboardException {
ActionType actionType = ActionType.ASSIGNED_TO_CUSTOMER;
TenantId tenantId = user.getTenantId();
try {
Set<CustomerId> addedCustomerIds = new HashSet<>();
Set<CustomerId> removedCustomerIds = new HashSet<>();
for (CustomerId customerId : customerIds) {
if (!dashboard.isAssignedToCustomer(customerId)) {
addedCustomerIds.add(customerId);
}
}
Set<ShortCustomerInfo> assignedCustomers = dashboard.getAssignedCustomers();
if (assignedCustomers != null) {
for (ShortCustomerInfo customerInfo : assignedCustomers) {
if (!customerIds.contains(customerInfo.getCustomerId())) {
removedCustomerIds.add(customerInfo.getCustomerId());
}
}
}
if (addedCustomerIds.isEmpty() && removedCustomerIds.isEmpty()) {
return dashboard;
} else {
Dashboard savedDashboard = null;
for (CustomerId customerId : addedCustomerIds) {
savedDashboard = checkNotNull(dashboardService.assignDashboardToCustomer(tenantId, dashboard.getId(), customerId));
ShortCustomerInfo customerInfo = savedDashboard.getAssignedCustomerInfo(customerId);
notificationEntityService.notifyAssignOrUnassignEntityToCustomer(tenantId, savedDashboard.getId(), customerId, savedDashboard,
actionType, EdgeEventActionType.ASSIGNED_TO_CUSTOMER, user, true, customerInfo.getTitle());
}
for (CustomerId customerId : removedCustomerIds) {
ShortCustomerInfo customerInfo = dashboard.getAssignedCustomerInfo(customerId);
savedDashboard = checkNotNull(dashboardService.unassignDashboardFromCustomer(tenantId, dashboard.getId(), customerId));
notificationEntityService.notifyAssignOrUnassignEntityToCustomer(tenantId, savedDashboard.getId(), customerId, savedDashboard,
ActionType.UNASSIGNED_FROM_CUSTOMER, EdgeEventActionType.UNASSIGNED_FROM_CUSTOMER, user, true, customerInfo.getTitle());
}
return savedDashboard;
}
} catch (Exception e) {
notificationEntityService.notifyEntity(tenantId, emptyId(EntityType.DASHBOARD), null, null,
actionType, user, e, dashboard.getId().toString());
throw handleException(e);
}
}
@Override
public Dashboard addDashboardCustomers(Dashboard dashboard, Set<CustomerId> customerIds, SecurityUser user) throws ThingsboardException {
ActionType actionType = ActionType.ASSIGNED_TO_CUSTOMER;
TenantId tenantId = user.getTenantId();
try {
if (customerIds.isEmpty()) {
return dashboard;
} else {
Dashboard savedDashboard = null;
for (CustomerId customerId : customerIds) {
savedDashboard = checkNotNull(dashboardService.assignDashboardToCustomer(tenantId, dashboard.getId(), customerId));
ShortCustomerInfo customerInfo = savedDashboard.getAssignedCustomerInfo(customerId);
notificationEntityService.notifyAssignOrUnassignEntityToCustomer(tenantId, savedDashboard.getId(), customerId, savedDashboard,
actionType, EdgeEventActionType.ASSIGNED_TO_CUSTOMER, user, true, customerInfo.getTitle());
}
return savedDashboard;
}
} catch (Exception e) {
notificationEntityService.notifyEntity(tenantId, emptyId(EntityType.DASHBOARD), null, null,
actionType, user, e, dashboard.getId().toString());
throw handleException(e);
}
}
@Override
public Dashboard removeDashboardCustomers(Dashboard dashboard, Set<CustomerId> customerIds, SecurityUser user) throws ThingsboardException {
ActionType actionType = ActionType.UNASSIGNED_FROM_CUSTOMER;
TenantId tenantId = user.getTenantId();
try {
if (customerIds.isEmpty()) {
return dashboard;
} else {
Dashboard savedDashboard = null;
for (CustomerId customerId : customerIds) {
ShortCustomerInfo customerInfo = dashboard.getAssignedCustomerInfo(customerId);
savedDashboard = checkNotNull(dashboardService.unassignDashboardFromCustomer(tenantId, dashboard.getId(), customerId));
notificationEntityService.notifyAssignOrUnassignEntityToCustomer(tenantId, savedDashboard.getId(), customerId, savedDashboard,
actionType, EdgeEventActionType.UNASSIGNED_FROM_CUSTOMER, user, true, customerInfo.getTitle());
}
return savedDashboard;
}
} catch (Exception e) {
notificationEntityService.notifyEntity(tenantId, emptyId(EntityType.DASHBOARD), null, null,
actionType, user, e, dashboard.getId().toString());
throw handleException(e);
}
}
@Override
public Dashboard asignDashboardToEdge(DashboardId dashboardId, Edge edge, SecurityUser user) throws ThingsboardException {
ActionType actionType = ActionType.ASSIGNED_TO_EDGE;
TenantId tenantId = user.getTenantId();
EdgeId edgeId = edge.getId();
try {
Dashboard savedDashboard = checkNotNull(dashboardService.assignDashboardToEdge(tenantId, dashboardId, edgeId));
notificationEntityService.notifyAssignOrUnassignEntityToEdge(tenantId, dashboardId, user.getCustomerId(),
edgeId, savedDashboard, actionType, EdgeEventActionType.ASSIGNED_TO_EDGE, user, dashboardId.toString(),
edgeId.toString(), edge.getName());
return savedDashboard;
} catch (Exception e) {
notificationEntityService.notifyEntity(tenantId, emptyId(EntityType.DEVICE), null, null,
actionType, user, e, dashboardId.toString(), edgeId.toString());
throw handleException(e);
}
}
@Override
public Dashboard unassignDashboardFromEdge(Dashboard dashboard, Edge edge, SecurityUser user) throws ThingsboardException {
ActionType actionType = ActionType.UNASSIGNED_FROM_EDGE;
TenantId tenantId = dashboard.getTenantId();
DashboardId dashboardId = dashboard.getId();
EdgeId edgeId = edge.getId();
try {
Dashboard savedDevice = checkNotNull(dashboardService.unassignDashboardFromEdge(tenantId, dashboardId, edgeId));
notificationEntityService.notifyAssignOrUnassignEntityToEdge(tenantId, dashboardId, user.getCustomerId(),
edgeId, dashboard, actionType, EdgeEventActionType.UNASSIGNED_FROM_EDGE, user, dashboardId.toString(),
edgeId.toString(), edge.getName());
return savedDevice;
} catch (Exception e) {
notificationEntityService.notifyEntity(tenantId, emptyId(EntityType.DASHBOARD), null, null,
actionType, user, e, dashboardId.toString(), edgeId.toString());
throw handleException(e);
}
}
@Override
public Dashboard unassignDashboardFromCustomer(Dashboard dashboard, Customer customer, SecurityUser user) throws ThingsboardException {
ActionType actionType = ActionType.UNASSIGNED_FROM_CUSTOMER;
TenantId tenantId = dashboard.getTenantId();
try {
Dashboard savedDashboard = checkNotNull(dashboardService.unassignDashboardFromCustomer(tenantId, dashboard.getId(), customer.getId()));
notificationEntityService.notifyAssignOrUnassignEntityToCustomer(tenantId, dashboard.getId(), customer.getId(), savedDashboard,
actionType, EdgeEventActionType.UNASSIGNED_FROM_CUSTOMER, user, true, customer.getId().toString(), customer.getName());
return savedDashboard;
} catch (Exception e) {
notificationEntityService.notifyEntity(tenantId, emptyId(EntityType.DASHBOARD), null, null,
actionType, user, e, dashboard.getId().toString());
throw handleException(e);
}
}
}

49
application/src/main/java/org/thingsboard/server/service/entitiy/dashboard/TbDashboardService.java

@ -0,0 +1,49 @@
/**
* Copyright © 2016-2022 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.service.entitiy.dashboard;
import org.thingsboard.server.common.data.Customer;
import org.thingsboard.server.common.data.Dashboard;
import org.thingsboard.server.common.data.edge.Edge;
import org.thingsboard.server.common.data.exception.ThingsboardException;
import org.thingsboard.server.common.data.id.CustomerId;
import org.thingsboard.server.common.data.id.DashboardId;
import org.thingsboard.server.service.entitiy.SimpleTbEntityService;
import org.thingsboard.server.service.security.model.SecurityUser;
import java.util.Set;
public interface TbDashboardService extends SimpleTbEntityService<Dashboard> {
Dashboard assignDashboardToCustomer(DashboardId dashboardId, Customer customer, SecurityUser user) throws ThingsboardException;
Dashboard assignDashboardToPublicCustomer(DashboardId dashboardId, SecurityUser user) throws ThingsboardException;
Dashboard unassignDashboardFromPublicCustomer(Dashboard dashboard, SecurityUser user) throws ThingsboardException;
Dashboard updateDashboardCustomers(Dashboard dashboard, Set<CustomerId> customerIds, SecurityUser user) throws ThingsboardException;
Dashboard addDashboardCustomers(Dashboard dashboard, Set<CustomerId> customerIds, SecurityUser user) throws ThingsboardException;
Dashboard removeDashboardCustomers(Dashboard dashboard, Set<CustomerId> customerIds, SecurityUser user) throws ThingsboardException;
Dashboard asignDashboardToEdge(DashboardId dashboardId, Edge edge, SecurityUser user) throws ThingsboardException;
Dashboard unassignDashboardFromEdge(Dashboard dashboard, Edge edge, SecurityUser user) throws ThingsboardException;
Dashboard unassignDashboardFromCustomer(Dashboard dashboard, Customer customer, SecurityUser user) throws ThingsboardException;
}

2
application/src/main/java/org/thingsboard/server/service/entitiy/device/DefaultTbDeviceService.java

@ -80,7 +80,7 @@ public class DefaultTbDeviceService extends AbstractTbEntityService implements T
}
@Override
public ListenableFuture<Void> deleteDevice(Device device, SecurityUser user) throws ThingsboardException {
public ListenableFuture<Void> delete(Device device, SecurityUser user) throws ThingsboardException {
TenantId tenantId = device.getTenantId();
DeviceId deviceId = device.getId();
try {

2
application/src/main/java/org/thingsboard/server/service/entitiy/device/TbDeviceService.java

@ -35,7 +35,7 @@ public interface TbDeviceService {
Device saveDeviceWithCredentials(TenantId tenantId, Device device, DeviceCredentials deviceCredentials, SecurityUser user) throws ThingsboardException;
ListenableFuture<Void> deleteDevice(Device device, SecurityUser user) throws ThingsboardException;
ListenableFuture<Void> delete(Device device, SecurityUser user) throws ThingsboardException;
Device assignDeviceToCustomer(TenantId tenantId, DeviceId deviceId, Customer customer, SecurityUser user) throws ThingsboardException;

8
application/src/main/java/org/thingsboard/server/service/entitiy/edge/DefaultTbEdgeService.java

@ -39,7 +39,7 @@ import org.thingsboard.server.service.security.model.SecurityUser;
public class DefaultTbEdgeService extends AbstractTbEntityService implements TbEdgeService {
@Override
public Edge saveEdge(Edge edge, RuleChain edgeTemplateRootRuleChain, SecurityUser user) throws ThingsboardException {
public Edge save(Edge edge, RuleChain edgeTemplateRootRuleChain, SecurityUser user) throws ThingsboardException {
ActionType actionType = edge.getId() == null ? ActionType.ADDED : ActionType.UPDATED;
TenantId tenantId = edge.getTenantId();
try {
@ -62,16 +62,16 @@ public class DefaultTbEdgeService extends AbstractTbEntityService implements TbE
}
@Override
public void deleteEdge(Edge edge, SecurityUser user) throws ThingsboardException {
public void delete(Edge edge, SecurityUser user) throws ThingsboardException {
ActionType actionType = ActionType.DELETED;
EdgeId edgeId = edge.getId();
TenantId tenantId = edge.getTenantId();
try {
edgeService.deleteEdge(tenantId, edgeId);
notificationEntityService.notifyEdge(tenantId, edgeId, edge.getCustomerId(), edge, actionType, user, edgeId.toString());
} catch (Exception e) {
notificationEntityService.notifyEntity(tenantId, emptyId(EntityType.EDGE), edge, null, actionType, user, e);
notificationEntityService.notifyEntity(tenantId, emptyId(EntityType.EDGE), edge, null, actionType,
user, e, edgeId.toString());
throw handleException(e);
}
}

4
application/src/main/java/org/thingsboard/server/service/entitiy/edge/TbEdgeService.java

@ -25,9 +25,9 @@ import org.thingsboard.server.common.data.rule.RuleChain;
import org.thingsboard.server.service.security.model.SecurityUser;
public interface TbEdgeService {
Edge saveEdge(Edge edge, RuleChain edgeTemplateRootRuleChain, SecurityUser user) throws ThingsboardException;
Edge save(Edge edge, RuleChain edgeTemplateRootRuleChain, SecurityUser user) throws ThingsboardException;
void deleteEdge(Edge edge, SecurityUser user) throws ThingsboardException;
void delete(Edge edge, SecurityUser user) throws ThingsboardException;
Edge assignEdgeToCustomer(TenantId tenantId, EdgeId edgeId, Customer customer, SecurityUser user) throws ThingsboardException;

269
application/src/main/java/org/thingsboard/server/service/entitiy/queue/DefaultTbQueueService.java

@ -0,0 +1,269 @@
/**
* Copyright © 2016-2022 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.service.entitiy.queue;
import lombok.AllArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Service;
import org.thingsboard.server.cluster.TbClusterService;
import org.thingsboard.server.common.data.DeviceProfile;
import org.thingsboard.server.common.data.TenantProfile;
import org.thingsboard.server.common.data.id.QueueId;
import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.data.page.PageLink;
import org.thingsboard.server.common.data.plugin.ComponentLifecycleEvent;
import org.thingsboard.server.common.data.queue.Queue;
import org.thingsboard.server.common.data.tenant.profile.TenantProfileQueueConfiguration;
import org.thingsboard.server.common.msg.queue.TopicPartitionInfo;
import org.thingsboard.server.dao.device.DeviceProfileService;
import org.thingsboard.server.queue.TbQueueAdmin;
import org.thingsboard.server.queue.scheduler.SchedulerComponent;
import org.thingsboard.server.queue.util.TbCoreComponent;
import org.thingsboard.server.service.entitiy.AbstractTbEntityService;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.Map;
import java.util.concurrent.TimeUnit;
import java.util.stream.Collectors;
@Slf4j
@Service
@TbCoreComponent
@AllArgsConstructor
public class DefaultTbQueueService extends AbstractTbEntityService implements TbQueueService {
private static final String MAIN = "Main";
private static final long DELETE_DELAY = 30;
private final TbClusterService tbClusterService;
private final TbQueueAdmin tbQueueAdmin;
private final DeviceProfileService deviceProfileService;
private final SchedulerComponent scheduler;
@Override
public Queue saveQueue(Queue queue) {
boolean create = queue.getId() == null;
Queue oldQueue;
if (create) {
oldQueue = null;
} else {
oldQueue = queueService.findQueueById(queue.getTenantId(), queue.getId());
}
//TODO: add checkNotNull
Queue savedQueue = queueService.saveQueue(queue);
if (create) {
onQueueCreated(savedQueue);
} else {
onQueueUpdated(savedQueue, oldQueue);
}
return savedQueue;
}
@Override
public void deleteQueue(TenantId tenantId, QueueId queueId) {
Queue queue = queueService.findQueueById(tenantId, queueId);
queueService.deleteQueue(tenantId, queueId);
onQueueDeleted(queue);
}
@Override
public void deleteQueueByQueueName(TenantId tenantId, String queueName) {
Queue queue = queueService.findQueueByTenantIdAndNameInternal(tenantId, queueName);
queueService.deleteQueue(tenantId, queue.getId());
onQueueDeleted(queue);
}
private void onQueueCreated(Queue queue) {
for (int i = 0; i < queue.getPartitions(); i++) {
tbQueueAdmin.createTopicIfNotExists(
new TopicPartitionInfo(queue.getTopic(), queue.getTenantId(), i, false).getFullTopicName());
}
tbClusterService.onQueueChange(queue);
}
private void onQueueUpdated(Queue queue, Queue oldQueue) {
int oldPartitions = oldQueue.getPartitions();
int currentPartitions = queue.getPartitions();
if (currentPartitions != oldPartitions) {
if (currentPartitions > oldPartitions) {
log.info("Added [{}] new partitions to [{}] queue", currentPartitions - oldPartitions, queue.getName());
for (int i = oldPartitions; i < currentPartitions; i++) {
tbQueueAdmin.createTopicIfNotExists(
new TopicPartitionInfo(queue.getTopic(), queue.getTenantId(), i, false).getFullTopicName());
}
tbClusterService.onQueueChange(queue);
} else {
log.info("Removed [{}] partitions from [{}] queue", oldPartitions - currentPartitions, queue.getName());
tbClusterService.onQueueChange(queue);
scheduler.schedule(() -> {
for (int i = currentPartitions; i < oldPartitions; i++) {
String fullTopicName = new TopicPartitionInfo(queue.getTopic(), queue.getTenantId(), i, false).getFullTopicName();
log.info("Removed partition [{}]", fullTopicName);
tbQueueAdmin.deleteTopic(
fullTopicName);
}
}, DELETE_DELAY, TimeUnit.SECONDS);
}
} else if (!oldQueue.equals(queue)) {
tbClusterService.onQueueChange(queue);
}
}
private void onQueueDeleted(Queue queue) {
tbClusterService.onQueueDelete(queue);
// queueStatsService.deleteQueueStatsByQueueId(tenantId, queueId);
scheduler.schedule(() -> {
for (int i = 0; i < queue.getPartitions(); i++) {
String fullTopicName = new TopicPartitionInfo(queue.getTopic(), queue.getTenantId(), i, false).getFullTopicName();
log.info("Deleting queue [{}]", fullTopicName);
try {
tbQueueAdmin.deleteTopic(fullTopicName);
} catch (Exception e) {
log.error("Failed to delete queue [{}]", fullTopicName);
}
}
}, DELETE_DELAY, TimeUnit.SECONDS);
}
@Override
public void updateQueuesByTenants(List<TenantId> tenantIds, TenantProfile newTenantProfile, TenantProfile
oldTenantProfile) {
boolean oldIsolated = oldTenantProfile != null && oldTenantProfile.isIsolatedTbRuleEngine();
boolean newIsolated = newTenantProfile.isIsolatedTbRuleEngine();
if (!oldIsolated && !newIsolated) {
return;
}
if (newTenantProfile.equals(oldTenantProfile)) {
return;
}
Map<String, TenantProfileQueueConfiguration> oldQueues;
Map<String, TenantProfileQueueConfiguration> newQueues;
if (oldIsolated) {
oldQueues = oldTenantProfile.getProfileData().getQueueConfiguration().stream()
.collect(Collectors.toMap(TenantProfileQueueConfiguration::getName, q -> q));
} else {
oldQueues = Collections.emptyMap();
}
if (newIsolated) {
newQueues = newTenantProfile.getProfileData().getQueueConfiguration().stream()
.collect(Collectors.toMap(TenantProfileQueueConfiguration::getName, q -> q));
} else {
newQueues = Collections.emptyMap();
}
List<String> toRemove = new ArrayList<>();
List<String> toCreate = new ArrayList<>();
List<String> toUpdate = new ArrayList<>();
for (String oldQueue : oldQueues.keySet()) {
if (!newQueues.containsKey(oldQueue)) {
toRemove.add(oldQueue);
}
}
for (String newQueue : newQueues.keySet()) {
if (oldQueues.containsKey(newQueue)) {
toUpdate.add(newQueue);
} else {
toCreate.add(newQueue);
}
}
tenantIds.forEach(tenantId -> {
Map<QueueId, List<DeviceProfile>> deviceProfileQueues;
if (oldTenantProfile != null && !newTenantProfile.getId().equals(oldTenantProfile.getId()) || !toRemove.isEmpty()) {
List<DeviceProfile> deviceProfiles = deviceProfileService.findDeviceProfiles(tenantId, new PageLink(Integer.MAX_VALUE)).getData();
deviceProfileQueues = deviceProfiles.stream()
.filter(dp -> dp.getDefaultQueueId() != null)
.collect(Collectors.groupingBy(DeviceProfile::getDefaultQueueId));
} else {
deviceProfileQueues = Collections.emptyMap();
}
Map<String, QueueId> createdQueues = toCreate.stream()
.map(key -> saveQueue(new Queue(tenantId, newQueues.get(key))))
.collect(Collectors.toMap(Queue::getName, Queue::getId));
// assigning created queues to device profiles instead of system queues
if (oldTenantProfile != null && !oldTenantProfile.isIsolatedTbRuleEngine()) {
deviceProfileQueues.forEach((queueId, list) -> {
Queue queue = queueService.findQueueById(TenantId.SYS_TENANT_ID, queueId);
QueueId queueIdToAssign = createdQueues.get(queue.getName());
if (queueIdToAssign == null) {
queueIdToAssign = createdQueues.get(MAIN);
}
for (DeviceProfile deviceProfile : list) {
deviceProfile.setDefaultQueueId(queueIdToAssign);
saveDeviceProfile(deviceProfile);
}
});
}
toUpdate.forEach(key -> {
Queue queueToUpdate = new Queue(tenantId, newQueues.get(key));
Queue foundQueue = queueService.findQueueByTenantIdAndName(tenantId, key);
queueToUpdate.setId(foundQueue.getId());
queueToUpdate.setCreatedTime(foundQueue.getCreatedTime());
if (queueToUpdate.equals(foundQueue)) {
//Queue not changed
} else {
saveQueue(queueToUpdate);
}
});
toRemove.forEach(q -> {
Queue queue = queueService.findQueueByTenantIdAndNameInternal(tenantId, q);
QueueId queueIdForRemove = queue.getId();
if (deviceProfileQueues.containsKey(queueIdForRemove)) {
Queue foundQueue = queueService.findQueueByTenantIdAndName(tenantId, q);
if (foundQueue == null || queue.equals(foundQueue)) {
foundQueue = queueService.findQueueByTenantIdAndName(tenantId, MAIN);
}
QueueId newQueueId = foundQueue.getId();
deviceProfileQueues.get(queueIdForRemove).stream()
.peek(dp -> dp.setDefaultQueueId(newQueueId))
.forEach(this::saveDeviceProfile);
}
deleteQueue(tenantId, queueIdForRemove);
});
});
}
//TODO: remove after implementing TbDeviceProfileService
private void saveDeviceProfile(DeviceProfile deviceProfile) {
DeviceProfile savedDeviceProfile = deviceProfileService.saveDeviceProfile(deviceProfile);
tbClusterService.onDeviceProfileChange(savedDeviceProfile, null);
tbClusterService.broadcastEntityStateChangeEvent(deviceProfile.getTenantId(), savedDeviceProfile.getId(), ComponentLifecycleEvent.UPDATED);
}
}

34
application/src/main/java/org/thingsboard/server/service/entitiy/queue/TbQueueService.java

@ -0,0 +1,34 @@
/**
* Copyright © 2016-2022 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.service.entitiy.queue;
import org.thingsboard.server.common.data.TenantProfile;
import org.thingsboard.server.common.data.id.QueueId;
import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.data.queue.Queue;
import java.util.List;
public interface TbQueueService {
Queue saveQueue(Queue queue);
void deleteQueue(TenantId tenantId, QueueId queueId);
void deleteQueueByQueueName(TenantId tenantId, String queueName);
void updateQueuesByTenants(List<TenantId> tenantIds, TenantProfile newTenantProfile, TenantProfile oldTenantProfile);
}

27
application/src/main/java/org/thingsboard/server/service/entitiy/tenant/DefaultTbTenantService.java

@ -15,37 +15,48 @@
*/
package org.thingsboard.server.service.entitiy.tenant;
import lombok.AllArgsConstructor;
import org.springframework.beans.factory.annotation.Autowired;
import lombok.RequiredArgsConstructor;
import org.springframework.stereotype.Service;
import org.thingsboard.server.common.data.Tenant;
import org.thingsboard.server.common.data.TenantProfile;
import org.thingsboard.server.common.data.exception.ThingsboardException;
import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.data.plugin.ComponentLifecycleEvent;
import org.thingsboard.server.dao.tenant.TenantProfileService;
import org.thingsboard.server.queue.util.TbCoreComponent;
import org.thingsboard.server.service.entitiy.AbstractTbEntityService;
import org.thingsboard.server.service.entitiy.queue.TbQueueService;
import org.thingsboard.server.service.install.InstallScripts;
import java.util.Collections;
@Service
@TbCoreComponent
@AllArgsConstructor
@RequiredArgsConstructor
public class DefaultTbTenantService extends AbstractTbEntityService implements TbTenantService {
@Autowired
private InstallScripts installScripts;
private final InstallScripts installScripts;
private final TbQueueService tbQueueService;
private final TenantProfileService tenantProfileService;
@Override
public Tenant save(Tenant tenant) throws ThingsboardException {
try {
boolean newTenant = tenant.getId() == null;
boolean created = tenant.getId() == null;
Tenant oldTenant = !created ? tenantService.findTenantById(tenant.getId()) : null;
Tenant savedTenant = checkNotNull(tenantService.saveTenant(tenant));
if (newTenant) {
if (created) {
installScripts.createDefaultRuleChains(savedTenant.getId());
installScripts.createDefaultEdgeRuleChains(savedTenant.getId());
}
tenantProfileCache.evict(savedTenant.getId());
notificationEntityService.notifyCreateOruUpdateTenant(savedTenant, newTenant ?
notificationEntityService.notifyCreateOruUpdateTenant(savedTenant, created ?
ComponentLifecycleEvent.CREATED : ComponentLifecycleEvent.UPDATED);
TenantProfile oldTenantProfile = oldTenant != null ? tenantProfileService.findTenantProfileById(TenantId.SYS_TENANT_ID, oldTenant.getTenantProfileId()) : null;
TenantProfile newTenantProfile = tenantProfileService.findTenantProfileById(TenantId.SYS_TENANT_ID, savedTenant.getTenantProfileId());
tbQueueService.updateQueuesByTenants(Collections.singletonList(savedTenant.getTenantId()), newTenantProfile, oldTenantProfile);
return savedTenant;
} catch (Exception e) {
throw handleException(e);

50
application/src/main/java/org/thingsboard/server/service/entitiy/tenant_profile/DefaultTbTenantProfileService.java

@ -0,0 +1,50 @@
/**
* Copyright © 2016-2022 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.service.entitiy.tenant_profile;
import lombok.AllArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Service;
import org.thingsboard.server.common.data.TenantProfile;
import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.dao.tenant.TenantProfileService;
import org.thingsboard.server.dao.tenant.TenantService;
import org.thingsboard.server.queue.util.TbCoreComponent;
import org.thingsboard.server.service.entitiy.queue.TbQueueService;
import java.util.List;
@Slf4j
@Service
@TbCoreComponent
@AllArgsConstructor
public class DefaultTbTenantProfileService implements TbTenantProfileService {
private final TbQueueService tbQueueService;
private final TenantProfileService tenantProfileService;
private final TenantService tenantService;
@Override
public TenantProfile saveTenantProfile(TenantId tenantId, TenantProfile tenantProfile, TenantProfile oldTenantProfile) {
TenantProfile savedTenantProfile = tenantProfileService.saveTenantProfile(tenantId, tenantProfile);
if (oldTenantProfile != null && savedTenantProfile.isIsolatedTbRuleEngine()) {
List<TenantId> tenantIds = tenantService.findTenantIdsByTenantProfileId(savedTenantProfile.getId());
tbQueueService.updateQueuesByTenants(tenantIds, savedTenantProfile, oldTenantProfile);
}
return savedTenantProfile;
}
}

23
application/src/main/java/org/thingsboard/server/service/entitiy/tenant_profile/TbTenantProfileService.java

@ -0,0 +1,23 @@
/**
* Copyright © 2016-2022 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.service.entitiy.tenant_profile;
import org.thingsboard.server.common.data.TenantProfile;
import org.thingsboard.server.common.data.id.TenantId;
public interface TbTenantProfileService {
TenantProfile saveTenantProfile(TenantId tenantId, TenantProfile tenantProfile, TenantProfile oldTenantProfile);
}

103
application/src/main/java/org/thingsboard/server/service/install/DefaultSystemDataLoaderService.java

@ -68,12 +68,18 @@ import org.thingsboard.server.common.data.query.DynamicValueSourceType;
import org.thingsboard.server.common.data.query.EntityKeyValueType;
import org.thingsboard.server.common.data.query.FilterPredicateValue;
import org.thingsboard.server.common.data.query.NumericFilterPredicate;
import org.thingsboard.server.common.data.queue.ProcessingStrategy;
import org.thingsboard.server.common.data.queue.ProcessingStrategyType;
import org.thingsboard.server.common.data.queue.Queue;
import org.thingsboard.server.common.data.queue.SubmitStrategy;
import org.thingsboard.server.common.data.queue.SubmitStrategyType;
import org.thingsboard.server.common.data.rule.RuleChainType;
import org.thingsboard.server.common.data.security.Authority;
import org.thingsboard.server.common.data.security.DeviceCredentials;
import org.thingsboard.server.common.data.security.UserCredentials;
import org.thingsboard.server.common.data.tenant.profile.DefaultTenantProfileConfiguration;
import org.thingsboard.server.common.data.tenant.profile.TenantProfileData;
import org.thingsboard.server.common.data.tenant.profile.TenantProfileQueueConfiguration;
import org.thingsboard.server.common.data.widget.WidgetsBundle;
import org.thingsboard.server.dao.attributes.AttributesService;
import org.thingsboard.server.dao.customer.CustomerService;
@ -81,6 +87,7 @@ import org.thingsboard.server.dao.device.DeviceCredentialsService;
import org.thingsboard.server.dao.device.DeviceProfileService;
import org.thingsboard.server.dao.device.DeviceService;
import org.thingsboard.server.dao.exception.DataValidationException;
import org.thingsboard.server.dao.queue.QueueService;
import org.thingsboard.server.dao.rule.RuleChainService;
import org.thingsboard.server.dao.settings.AdminSettingsService;
import org.thingsboard.server.dao.tenant.TenantProfileService;
@ -155,6 +162,9 @@ public class DefaultSystemDataLoaderService implements SystemDataLoaderService {
@Getter
private boolean persistActivityToTelemetry;
@Autowired
private QueueService queueService;
@Bean
protected BCryptPasswordEncoder passwordEncoder() {
return new BCryptPasswordEncoder();
@ -199,13 +209,37 @@ public class DefaultSystemDataLoaderService implements SystemDataLoaderService {
log.warn(e.getMessage());
}
TenantProfileData isolatedRuleEngineTenantProfileData = new TenantProfileData();
isolatedRuleEngineTenantProfileData.setConfiguration(new DefaultTenantProfileConfiguration());
TenantProfileQueueConfiguration mainQueueConfiguration = new TenantProfileQueueConfiguration();
mainQueueConfiguration.setName("Main");
mainQueueConfiguration.setTopic("tb_rule_engine.main");
mainQueueConfiguration.setPollInterval(25);
mainQueueConfiguration.setPartitions(10);
mainQueueConfiguration.setConsumerPerPartition(true);
mainQueueConfiguration.setPackProcessingTimeout(2000);
SubmitStrategy mainQueueSubmitStrategy = new SubmitStrategy();
mainQueueSubmitStrategy.setType(SubmitStrategyType.BURST);
mainQueueSubmitStrategy.setBatchSize(1000);
mainQueueConfiguration.setSubmitStrategy(mainQueueSubmitStrategy);
ProcessingStrategy mainQueueProcessingStrategy = new ProcessingStrategy();
mainQueueProcessingStrategy.setType(ProcessingStrategyType.SKIP_ALL_FAILURES);
mainQueueProcessingStrategy.setRetries(3);
mainQueueProcessingStrategy.setFailurePercentage(0);
mainQueueProcessingStrategy.setPauseBetweenRetries(3);
mainQueueProcessingStrategy.setMaxPauseBetweenRetries(3);
mainQueueConfiguration.setProcessingStrategy(mainQueueProcessingStrategy);
isolatedRuleEngineTenantProfileData.setQueueConfiguration(Collections.singletonList(mainQueueConfiguration));
TenantProfile isolatedTbRuleEngineProfile = new TenantProfile();
isolatedTbRuleEngineProfile.setDefault(false);
isolatedTbRuleEngineProfile.setName("Isolated TB Rule Engine");
isolatedTbRuleEngineProfile.setDescription("Isolated TB Rule Engine tenant profile");
isolatedTbRuleEngineProfile.setIsolatedTbCore(false);
isolatedTbRuleEngineProfile.setIsolatedTbRuleEngine(true);
isolatedTbRuleEngineProfile.setProfileData(tenantProfileData);
isolatedTbRuleEngineProfile.setProfileData(isolatedRuleEngineTenantProfileData);
try {
tenantProfileService.saveTenantProfile(TenantId.SYS_TENANT_ID, isolatedTbRuleEngineProfile);
@ -219,7 +253,7 @@ public class DefaultSystemDataLoaderService implements SystemDataLoaderService {
isolatedTbCoreAndTbRuleEngineProfile.setDescription("Isolated TB Core and TB Rule Engine tenant profile");
isolatedTbCoreAndTbRuleEngineProfile.setIsolatedTbCore(true);
isolatedTbCoreAndTbRuleEngineProfile.setIsolatedTbRuleEngine(true);
isolatedTbCoreAndTbRuleEngineProfile.setProfileData(tenantProfileData);
isolatedTbCoreAndTbRuleEngineProfile.setProfileData(isolatedRuleEngineTenantProfileData);
try {
tenantProfileService.saveTenantProfile(TenantId.SYS_TENANT_ID, isolatedTbCoreAndTbRuleEngineProfile);
@ -577,4 +611,69 @@ public class DefaultSystemDataLoaderService implements SystemDataLoaderService {
}, tsCallBackExecutor);
}
@Override
public void createQueues() {
Queue mainQueue = new Queue();
mainQueue.setTenantId(TenantId.SYS_TENANT_ID);
mainQueue.setName("Main");
mainQueue.setTopic("tb_rule_engine.main");
mainQueue.setPollInterval(25);
mainQueue.setPartitions(10);
mainQueue.setConsumerPerPartition(true);
mainQueue.setPackProcessingTimeout(2000);
SubmitStrategy mainQueueSubmitStrategy = new SubmitStrategy();
mainQueueSubmitStrategy.setType(SubmitStrategyType.BURST);
mainQueueSubmitStrategy.setBatchSize(1000);
mainQueue.setSubmitStrategy(mainQueueSubmitStrategy);
ProcessingStrategy mainQueueProcessingStrategy = new ProcessingStrategy();
mainQueueProcessingStrategy.setType(ProcessingStrategyType.SKIP_ALL_FAILURES);
mainQueueProcessingStrategy.setRetries(3);
mainQueueProcessingStrategy.setFailurePercentage(0);
mainQueueProcessingStrategy.setPauseBetweenRetries(3);
mainQueueProcessingStrategy.setMaxPauseBetweenRetries(3);
mainQueue.setProcessingStrategy(mainQueueProcessingStrategy);
queueService.saveQueue(mainQueue);
Queue highPriorityQueue = new Queue();
highPriorityQueue.setTenantId(TenantId.SYS_TENANT_ID);
highPriorityQueue.setName("HighPriority");
highPriorityQueue.setTopic("tb_rule_engine.hp");
highPriorityQueue.setPollInterval(25);
highPriorityQueue.setPartitions(10);
highPriorityQueue.setConsumerPerPartition(true);
highPriorityQueue.setPackProcessingTimeout(2000);
SubmitStrategy highPriorityQueueSubmitStrategy = new SubmitStrategy();
highPriorityQueueSubmitStrategy.setType(SubmitStrategyType.BURST);
highPriorityQueueSubmitStrategy.setBatchSize(100);
highPriorityQueue.setSubmitStrategy(highPriorityQueueSubmitStrategy);
ProcessingStrategy highPriorityQueueProcessingStrategy = new ProcessingStrategy();
highPriorityQueueProcessingStrategy.setType(ProcessingStrategyType.RETRY_FAILED_AND_TIMED_OUT);
highPriorityQueueProcessingStrategy.setRetries(0);
highPriorityQueueProcessingStrategy.setFailurePercentage(0);
highPriorityQueueProcessingStrategy.setPauseBetweenRetries(5);
highPriorityQueueProcessingStrategy.setMaxPauseBetweenRetries(5);
highPriorityQueue.setProcessingStrategy(highPriorityQueueProcessingStrategy);
queueService.saveQueue(highPriorityQueue);
Queue sequentialByOriginatorQueue = new Queue();
sequentialByOriginatorQueue.setTenantId(TenantId.SYS_TENANT_ID);
sequentialByOriginatorQueue.setName("SequentialByOriginator");
sequentialByOriginatorQueue.setTopic("tb_rule_engine.sq");
sequentialByOriginatorQueue.setPollInterval(25);
sequentialByOriginatorQueue.setPartitions(10);
sequentialByOriginatorQueue.setPackProcessingTimeout(2000);
sequentialByOriginatorQueue.setConsumerPerPartition(true);
SubmitStrategy sequentialByOriginatorQueueSubmitStrategy = new SubmitStrategy();
sequentialByOriginatorQueueSubmitStrategy.setType(SubmitStrategyType.SEQUENTIAL_BY_ORIGINATOR);
sequentialByOriginatorQueueSubmitStrategy.setBatchSize(100);
sequentialByOriginatorQueue.setSubmitStrategy(sequentialByOriginatorQueueSubmitStrategy);
ProcessingStrategy sequentialByOriginatorQueueProcessingStrategy = new ProcessingStrategy();
sequentialByOriginatorQueueProcessingStrategy.setType(ProcessingStrategyType.RETRY_FAILED_AND_TIMED_OUT);
sequentialByOriginatorQueueProcessingStrategy.setRetries(3);
sequentialByOriginatorQueueProcessingStrategy.setFailurePercentage(0);
sequentialByOriginatorQueueProcessingStrategy.setPauseBetweenRetries(5);
sequentialByOriginatorQueueProcessingStrategy.setMaxPauseBetweenRetries(5);
sequentialByOriginatorQueue.setProcessingStrategy(sequentialByOriginatorQueueProcessingStrategy);
queueService.saveQueue(sequentialByOriginatorQueue);
}
}

158
application/src/main/java/org/thingsboard/server/service/install/SqlDatabaseUpgradeService.java

@ -15,20 +15,37 @@
*/
package org.thingsboard.server.service.install;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.node.ObjectNode;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.collections.CollectionUtils;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.context.annotation.Profile;
import org.springframework.stereotype.Service;
import org.thingsboard.server.common.data.EntitySubtype;
import org.thingsboard.server.common.data.Tenant;
import org.thingsboard.server.common.data.TenantProfile;
import org.thingsboard.server.common.data.id.QueueId;
import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.data.page.PageData;
import org.thingsboard.server.common.data.page.PageLink;
import org.thingsboard.server.common.data.queue.ProcessingStrategy;
import org.thingsboard.server.common.data.queue.ProcessingStrategyType;
import org.thingsboard.server.common.data.queue.Queue;
import org.thingsboard.server.common.data.queue.SubmitStrategy;
import org.thingsboard.server.common.data.queue.SubmitStrategyType;
import org.thingsboard.server.common.data.rule.RuleNode;
import org.thingsboard.server.common.data.tenant.profile.TenantProfileQueueConfiguration;
import org.thingsboard.server.dao.dashboard.DashboardService;
import org.thingsboard.server.dao.device.DeviceProfileService;
import org.thingsboard.server.dao.device.DeviceService;
import org.thingsboard.server.dao.queue.QueueService;
import org.thingsboard.server.dao.rule.RuleChainService;
import org.thingsboard.server.dao.tenant.TenantProfileService;
import org.thingsboard.server.dao.tenant.TenantService;
import org.thingsboard.server.dao.usagerecord.ApiUsageStateService;
import org.thingsboard.server.queue.settings.TbRuleEngineQueueConfiguration;
import org.thingsboard.server.service.install.sql.SqlDbHelper;
import java.nio.charset.Charset;
@ -42,8 +59,11 @@ import java.sql.SQLException;
import java.sql.SQLSyntaxErrorException;
import java.sql.SQLWarning;
import java.sql.Statement;
import java.util.Collections;
import java.util.List;
import java.util.Map;
import java.util.concurrent.TimeUnit;
import java.util.stream.Collectors;
import static org.thingsboard.server.service.install.DatabaseHelper.ADDITIONAL_INFO;
import static org.thingsboard.server.service.install.DatabaseHelper.ASSIGNED_CUSTOMERS;
@ -101,6 +121,17 @@ public class SqlDatabaseUpgradeService implements DatabaseEntitiesUpgradeService
@Autowired
private ApiUsageStateService apiUsageStateService;
@Autowired
private QueueService queueService;
@Autowired
private TbRuleEngineQueueConfigService queueConfig;
@Autowired
private RuleChainService ruleChainService;
@Autowired
private TenantProfileService tenantProfileService;
@Override
public void upgradeDatabase(String fromVersion) throws Exception {
@ -539,6 +570,88 @@ public class SqlDatabaseUpgradeService implements DatabaseEntitiesUpgradeService
log.info("Updating schema ...");
schemaUpdateFile = Paths.get(installScripts.getDataDir(), "upgrade", "3.3.4", SCHEMA_UPDATE_SQL);
loadSql(schemaUpdateFile, conn);
log.info("Loading queues...");
try {
if (!CollectionUtils.isEmpty(queueConfig.getQueues())) {
queueConfig.getQueues().forEach(queueSettings -> {
queueService.saveQueue(queueConfigToQueue(queueSettings));
});
} else {
systemDataLoaderService.createQueues();
}
} catch (Exception e) {
}
log.info("Updating device profiles...");
schemaUpdateFile = Paths.get(installScripts.getDataDir(), "upgrade", "3.3.4", "schema_update_device_profile.sql");
loadSql(schemaUpdateFile, conn);
log.info("Updating checkpoint rule nodes...");
PageLink pageLink = new PageLink(100);
PageData<Tenant> pageData;
do {
pageData = tenantService.findTenants(pageLink);
for (Tenant tenant : pageData.getData()) {
TenantId tenantId = tenant.getId();
Map<String, QueueId> queues =
queueService.findQueuesByTenantId(tenantId).stream().collect(Collectors.toMap(Queue::getName, Queue::getId));
try {
List<RuleNode> checkpointNodes =
ruleChainService.findRuleNodesByTenantIdAndType(tenantId, "org.thingsboard.rule.engine.flow.TbCheckpointNode");
checkpointNodes.forEach(node -> {
ObjectNode configuration = (ObjectNode) node.getConfiguration();
JsonNode queueNameNode = configuration.remove("queueName");
if (queueNameNode != null) {
String queueName = queueNameNode.asText();
configuration.put("queueId", queues.get(queueName).toString());
ruleChainService.saveRuleNode(tenantId, node);
}
});
} catch (Exception e) {
}
}
pageLink = pageLink.nextPageLink();
} while (pageData.hasNext());
log.info("Updating tenant profiles...");
PageLink profilePageLink = new PageLink(100);
PageData<TenantProfile> profilePageData;
do {
profilePageData = tenantProfileService.findTenantProfiles(TenantId.SYS_TENANT_ID, profilePageLink);
profilePageData.getData().forEach(profile -> {
try {
List<TenantProfileQueueConfiguration> queueConfiguration = profile.getProfileData().getQueueConfiguration();
if (profile.isIsolatedTbRuleEngine() && (queueConfiguration == null || queueConfiguration.isEmpty())) {
TenantProfileQueueConfiguration mainQueueConfig = getMainQueueConfiguration();
profile.getProfileData().setQueueConfiguration(Collections.singletonList((mainQueueConfig)));
tenantProfileService.saveTenantProfile(TenantId.SYS_TENANT_ID, profile);
List<TenantId> isolatedTenants = tenantService.findTenantIdsByTenantProfileId(profile.getId());
isolatedTenants.forEach(tenantId -> {
queueService.saveQueue(new Queue(tenantId, mainQueueConfig));
});
}
} catch (Exception e) {
}
});
profilePageLink = profilePageLink.nextPageLink();
} while (profilePageData.hasNext());
log.info("Updating schema settings...");
conn.createStatement().execute("UPDATE tb_schema_settings SET schema_version = 3004000;");
log.info("Schema updated.");
} catch (Exception e) {
log.error("Failed updating schema!!!", e);
}
break;
case "3.3.4":
try (Connection conn = DriverManager.getConnection(dbUrl, dbUserName, dbPassword)) {
log.info("Updating schema ...");
schemaUpdateFile = Paths.get(installScripts.getDataDir(), "upgrade", "3.3.4", SCHEMA_UPDATE_SQL);
loadSql(schemaUpdateFile, conn);
log.info("Updating schema settings...");
conn.createStatement().execute("UPDATE tb_schema_settings SET schema_version = 3004000;");
log.info("Schema updated.");
@ -591,4 +704,49 @@ public class SqlDatabaseUpgradeService implements DatabaseEntitiesUpgradeService
}
return isOldSchema;
}
private Queue queueConfigToQueue(TbRuleEngineQueueConfiguration queueSettings) {
Queue queue = new Queue();
queue.setTenantId(TenantId.SYS_TENANT_ID);
queue.setName(queueSettings.getName());
queue.setTopic(queueSettings.getTopic());
queue.setPollInterval(queueSettings.getPollInterval());
queue.setPartitions(queueSettings.getPartitions());
queue.setPackProcessingTimeout(queueSettings.getPackProcessingTimeout());
SubmitStrategy submitStrategy = new SubmitStrategy();
submitStrategy.setBatchSize(queueSettings.getSubmitStrategy().getBatchSize());
submitStrategy.setType(SubmitStrategyType.valueOf(queueSettings.getSubmitStrategy().getType()));
queue.setSubmitStrategy(submitStrategy);
ProcessingStrategy processingStrategy = new ProcessingStrategy();
processingStrategy.setType(ProcessingStrategyType.valueOf(queueSettings.getProcessingStrategy().getType()));
processingStrategy.setRetries(queueSettings.getProcessingStrategy().getRetries());
processingStrategy.setFailurePercentage(queueSettings.getProcessingStrategy().getFailurePercentage());
processingStrategy.setPauseBetweenRetries(queueSettings.getProcessingStrategy().getPauseBetweenRetries());
processingStrategy.setMaxPauseBetweenRetries(queueSettings.getProcessingStrategy().getMaxPauseBetweenRetries());
queue.setProcessingStrategy(processingStrategy);
queue.setConsumerPerPartition(queueSettings.isConsumerPerPartition());
return queue;
}
private TenantProfileQueueConfiguration getMainQueueConfiguration() {
TenantProfileQueueConfiguration mainQueueConfiguration = new TenantProfileQueueConfiguration();
mainQueueConfiguration.setName("Main");
mainQueueConfiguration.setTopic("tb_rule_engine.main");
mainQueueConfiguration.setPollInterval(25);
mainQueueConfiguration.setPartitions(10);
mainQueueConfiguration.setConsumerPerPartition(true);
mainQueueConfiguration.setPackProcessingTimeout(2000);
SubmitStrategy mainQueueSubmitStrategy = new SubmitStrategy();
mainQueueSubmitStrategy.setType(SubmitStrategyType.BURST);
mainQueueSubmitStrategy.setBatchSize(1000);
mainQueueConfiguration.setSubmitStrategy(mainQueueSubmitStrategy);
ProcessingStrategy mainQueueProcessingStrategy = new ProcessingStrategy();
mainQueueProcessingStrategy.setType(ProcessingStrategyType.SKIP_ALL_FAILURES);
mainQueueProcessingStrategy.setRetries(3);
mainQueueProcessingStrategy.setFailurePercentage(0);
mainQueueProcessingStrategy.setPauseBetweenRetries(3);
mainQueueProcessingStrategy.setMaxPauseBetweenRetries(3);
mainQueueConfiguration.setProcessingStrategy(mainQueueProcessingStrategy);
return mainQueueConfiguration;
}
}

1
application/src/main/java/org/thingsboard/server/service/install/SystemDataLoaderService.java

@ -33,4 +33,5 @@ public interface SystemDataLoaderService {
void deleteSystemWidgetBundle(String bundleAlias) throws Exception;
void createQueues();
}

48
application/src/main/java/org/thingsboard/server/service/install/TbRuleEngineQueueConfigService.java

@ -0,0 +1,48 @@
/**
* Copyright © 2016-2022 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.service.install;
import lombok.Data;
import lombok.extern.slf4j.Slf4j;
import org.springframework.boot.autoconfigure.EnableAutoConfiguration;
import org.springframework.boot.context.properties.ConfigurationProperties;
import org.springframework.context.annotation.Configuration;
import org.springframework.context.annotation.Profile;
import org.thingsboard.server.queue.settings.TbRuleEngineQueueConfiguration;
import javax.annotation.PostConstruct;
import java.util.List;
@Slf4j
@Data
@EnableAutoConfiguration
@Configuration
@ConfigurationProperties(prefix = "queue.rule-engine")
@Profile("install")
public class TbRuleEngineQueueConfigService {
private String topic;
private List<TbRuleEngineQueueConfiguration> queues;
@PostConstruct
public void validate() {
queues.stream().filter(queue -> queue.getName().equals("Main")).findFirst().orElseThrow(() -> {
log.error("Main queue is not configured in thingsboard.yml");
return new RuntimeException("No \"Main\" queue configured!");
});
}
}

44
application/src/main/java/org/thingsboard/server/service/queue/DefaultQueueRoutingInfoService.java

@ -0,0 +1,44 @@
/**
* Copyright © 2016-2022 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.service.queue;
import lombok.extern.slf4j.Slf4j;
import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression;
import org.springframework.stereotype.Service;
import org.thingsboard.server.dao.queue.QueueService;
import org.thingsboard.server.queue.discovery.QueueRoutingInfo;
import org.thingsboard.server.queue.discovery.QueueRoutingInfoService;
import java.util.List;
import java.util.stream.Collectors;
@Slf4j
@Service
@ConditionalOnExpression("'${service.type:null}'=='monolith' || '${service.type:null}'=='tb-core' || '${service.type:null}'=='tb-rule-engine'")
public class DefaultQueueRoutingInfoService implements QueueRoutingInfoService {
private final QueueService queueService;
public DefaultQueueRoutingInfoService(QueueService queueService) {
this.queueService = queueService;
}
@Override
public List<QueueRoutingInfo> getAllQueuesRoutingInfo() {
return queueService.findAllQueues().stream().map(QueueRoutingInfo::new).collect(Collectors.toList());
}
}

120
application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java

@ -18,36 +18,41 @@ package org.thingsboard.server.service.queue;
import com.google.protobuf.ByteString;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.context.annotation.Lazy;
import org.springframework.scheduling.annotation.Scheduled;
import org.springframework.stereotype.Service;
import org.thingsboard.rule.engine.api.msg.DeviceEdgeUpdateMsg;
import org.thingsboard.rule.engine.api.msg.DeviceNameOrTypeUpdateMsg;
import org.thingsboard.server.cluster.TbClusterService;
import org.thingsboard.server.common.data.EdgeUtils;
import org.thingsboard.server.common.data.edge.EdgeEventActionType;
import org.thingsboard.server.common.data.edge.EdgeEventType;
import org.thingsboard.server.common.msg.ToDeviceActorNotificationMsg;
import org.thingsboard.server.common.data.ApiUsageState;
import org.thingsboard.server.common.data.Device;
import org.thingsboard.server.common.data.DeviceProfile;
import org.thingsboard.server.common.data.EdgeUtils;
import org.thingsboard.server.common.data.EntityType;
import org.thingsboard.server.common.data.HasName;
import org.thingsboard.server.common.data.TbResource;
import org.thingsboard.server.common.data.Tenant;
import org.thingsboard.server.common.data.TenantProfile;
import org.thingsboard.server.common.data.edge.EdgeEventActionType;
import org.thingsboard.server.common.data.edge.EdgeEventType;
import org.thingsboard.server.common.data.id.DeviceId;
import org.thingsboard.server.common.data.id.DeviceProfileId;
import org.thingsboard.server.common.data.id.EdgeId;
import org.thingsboard.server.common.data.id.EntityId;
import org.thingsboard.server.common.data.id.QueueId;
import org.thingsboard.server.common.data.id.RuleChainId;
import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.data.plugin.ComponentLifecycleEvent;
import org.thingsboard.server.common.data.queue.Queue;
import org.thingsboard.server.common.msg.TbMsg;
import org.thingsboard.server.common.msg.ToDeviceActorNotificationMsg;
import org.thingsboard.server.common.msg.edge.EdgeEventUpdateMsg;
import org.thingsboard.server.common.msg.plugin.ComponentLifecycleMsg;
import org.thingsboard.server.common.msg.queue.ServiceType;
import org.thingsboard.server.common.msg.queue.TopicPartitionInfo;
import org.thingsboard.server.common.msg.rpc.FromDeviceRpcResponse;
import org.thingsboard.server.common.transport.util.DataDecodingEncodingService;
import org.thingsboard.server.gen.transport.TransportProtos;
import org.thingsboard.server.gen.transport.TransportProtos.FromDeviceRPCResponseProto;
@ -60,12 +65,12 @@ import org.thingsboard.server.queue.TbQueueCallback;
import org.thingsboard.server.queue.TbQueueProducer;
import org.thingsboard.server.queue.common.MultipleTbQueueCallbackWrapper;
import org.thingsboard.server.queue.common.TbProtoQueueMsg;
import org.thingsboard.server.queue.discovery.NotificationsTopicService;
import org.thingsboard.server.queue.discovery.PartitionService;
import org.thingsboard.server.queue.provider.TbQueueProducerProvider;
import org.thingsboard.server.service.gateway_device.GatewayNotificationsService;
import org.thingsboard.server.service.ota.OtaPackageStateService;
import org.thingsboard.server.service.profile.TbDeviceProfileCache;
import org.thingsboard.server.common.msg.rpc.FromDeviceRpcResponse;
import java.util.HashSet;
import java.util.Set;
@ -81,6 +86,8 @@ public class DefaultTbClusterService implements TbClusterService {
private boolean statsEnabled;
@Value("${edges.enabled}")
protected boolean edgesEnabled;
@Value("${service.type:monolith}")
private String serviceType;
private final AtomicInteger toCoreMsgs = new AtomicInteger(0);
private final AtomicInteger toCoreNfs = new AtomicInteger(0);
@ -88,11 +95,21 @@ public class DefaultTbClusterService implements TbClusterService {
private final AtomicInteger toRuleEngineNfs = new AtomicInteger(0);
private final AtomicInteger toTransportNfs = new AtomicInteger(0);
private final TbQueueProducerProvider producerProvider;
private final PartitionService partitionService;
@Autowired
@Lazy
private PartitionService partitionService;
@Autowired
@Lazy
private TbQueueProducerProvider producerProvider;
@Autowired
@Lazy
private OtaPackageStateService otaPackageStateService;
private final NotificationsTopicService notificationsTopicService;
private final DataDecodingEncodingService encodingService;
private final TbDeviceProfileCache deviceProfileCache;
private final OtaPackageStateService otaPackageStateService;
private final GatewayNotificationsService gatewayNotificationsService;
@Override
@ -120,7 +137,7 @@ public class DefaultTbClusterService implements TbClusterService {
@Override
public void pushNotificationToCore(String serviceId, FromDeviceRpcResponse response, TbQueueCallback callback) {
TopicPartitionInfo tpi = partitionService.getNotificationsTopic(ServiceType.TB_CORE, serviceId);
TopicPartitionInfo tpi = notificationsTopicService.getNotificationsTopic(ServiceType.TB_CORE, serviceId);
log.trace("PUSHING msg: {} to:{}", response, tpi);
FromDeviceRPCResponseProto.Builder builder = FromDeviceRPCResponseProto.newBuilder()
.setRequestIdMSB(response.getId().getMostSignificantBits())
@ -155,7 +172,7 @@ public class DefaultTbClusterService implements TbClusterService {
tbMsg = transformMsg(tbMsg, deviceProfileCache.get(tenantId, new DeviceProfileId(entityId.getId())));
}
}
TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_RULE_ENGINE, tbMsg.getQueueName(), tenantId, entityId);
TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_RULE_ENGINE, tbMsg.getQueueId(), tenantId, entityId);
log.trace("PUSHING msg: {} to:{}", tbMsg, tpi);
ToRuleEngineMsg msg = ToRuleEngineMsg.newBuilder()
.setTenantIdMSB(tenantId.getId().getMostSignificantBits())
@ -168,16 +185,16 @@ public class DefaultTbClusterService implements TbClusterService {
private TbMsg transformMsg(TbMsg tbMsg, DeviceProfile deviceProfile) {
if (deviceProfile != null) {
RuleChainId targetRuleChainId = deviceProfile.getDefaultRuleChainId();
String targetQueueName = deviceProfile.getDefaultQueueName();
QueueId targetQueueId = deviceProfile.getDefaultQueueId();
boolean isRuleChainTransform = targetRuleChainId != null && !targetRuleChainId.equals(tbMsg.getRuleChainId());
boolean isQueueTransform = targetQueueName != null && !targetQueueName.equals(tbMsg.getQueueName());
boolean isQueueTransform = targetQueueId != null && !targetQueueId.equals(tbMsg.getQueueId());
if (isRuleChainTransform && isQueueTransform) {
tbMsg = TbMsg.transformMsg(tbMsg, targetRuleChainId, targetQueueName);
tbMsg = TbMsg.transformMsg(tbMsg, targetRuleChainId, targetQueueId);
} else if (isRuleChainTransform) {
tbMsg = TbMsg.transformMsg(tbMsg, targetRuleChainId);
} else if (isQueueTransform) {
tbMsg = TbMsg.transformMsg(tbMsg, targetQueueName);
tbMsg = TbMsg.transformMsg(tbMsg, targetQueueId);
}
}
return tbMsg;
@ -185,7 +202,7 @@ public class DefaultTbClusterService implements TbClusterService {
@Override
public void pushNotificationToRuleEngine(String serviceId, FromDeviceRpcResponse response, TbQueueCallback callback) {
TopicPartitionInfo tpi = partitionService.getNotificationsTopic(ServiceType.TB_RULE_ENGINE, serviceId);
TopicPartitionInfo tpi = notificationsTopicService.getNotificationsTopic(ServiceType.TB_RULE_ENGINE, serviceId);
log.trace("PUSHING msg: {} to:{}", response, tpi);
FromDeviceRPCResponseProto.Builder builder = FromDeviceRPCResponseProto.newBuilder()
.setRequestIdMSB(response.getId().getMostSignificantBits())
@ -199,14 +216,14 @@ public class DefaultTbClusterService implements TbClusterService {
@Override
public void pushNotificationToTransport(String serviceId, ToTransportMsg response, TbQueueCallback callback) {
if (serviceId == null || serviceId.isEmpty()){
if (serviceId == null || serviceId.isEmpty()) {
log.trace("pushNotificationToTransport: skipping message without serviceId [{}], (ToTransportMsg) response [{}]", serviceId, response);
if (callback != null) {
callback.onSuccess(null); //callback that message already sent, no useful payload expected
}
return;
}
TopicPartitionInfo tpi = partitionService.getNotificationsTopic(ServiceType.TB_TRANSPORT, serviceId);
TopicPartitionInfo tpi = notificationsTopicService.getNotificationsTopic(ServiceType.TB_TRANSPORT, serviceId);
log.trace("PUSHING msg: {} to:{}", response, tpi);
producerProvider.getTransportNotificationsMsgProducer().send(tpi, new TbProtoQueueMsg<>(UUID.randomUUID(), response), callback);
toTransportNfs.incrementAndGet();
@ -314,7 +331,7 @@ public class DefaultTbClusterService implements TbClusterService {
Set<String> tbTransportServices = partitionService.getAllServiceIds(ServiceType.TB_TRANSPORT);
TbQueueCallback proxyCallback = callback != null ? new MultipleTbQueueCallbackWrapper(tbTransportServices.size(), callback) : null;
for (String transportServiceId : tbTransportServices) {
TopicPartitionInfo tpi = partitionService.getNotificationsTopic(ServiceType.TB_TRANSPORT, transportServiceId);
TopicPartitionInfo tpi = notificationsTopicService.getNotificationsTopic(ServiceType.TB_TRANSPORT, transportServiceId);
toTransportNfProducer.send(tpi, new TbProtoQueueMsg<>(UUID.randomUUID(), transportMsg), proxyCallback);
toTransportNfs.incrementAndGet();
}
@ -328,7 +345,7 @@ public class DefaultTbClusterService implements TbClusterService {
TbQueueProducer<TbProtoQueueMsg<ToCoreNotificationMsg>> toCoreNfProducer = producerProvider.getTbCoreNotificationsMsgProducer();
Set<String> tbCoreServices = partitionService.getAllServiceIds(ServiceType.TB_CORE);
for (String serviceId : tbCoreServices) {
TopicPartitionInfo tpi = partitionService.getNotificationsTopic(ServiceType.TB_CORE, serviceId);
TopicPartitionInfo tpi = notificationsTopicService.getNotificationsTopic(ServiceType.TB_CORE, serviceId);
ToCoreNotificationMsg toCoreMsg = ToCoreNotificationMsg.newBuilder().setEdgeEventUpdateMsg(ByteString.copyFrom(msgBytes)).build();
toCoreNfProducer.send(tpi, new TbProtoQueueMsg<>(msg.getEdgeId().getId(), toCoreMsg), null);
toCoreNfs.incrementAndGet();
@ -349,7 +366,7 @@ public class DefaultTbClusterService implements TbClusterService {
TbQueueProducer<TbProtoQueueMsg<ToCoreNotificationMsg>> toCoreNfProducer = producerProvider.getTbCoreNotificationsMsgProducer();
Set<String> tbCoreServices = partitionService.getAllServiceIds(ServiceType.TB_CORE);
for (String serviceId : tbCoreServices) {
TopicPartitionInfo tpi = partitionService.getNotificationsTopic(ServiceType.TB_CORE, serviceId);
TopicPartitionInfo tpi = notificationsTopicService.getNotificationsTopic(ServiceType.TB_CORE, serviceId);
ToCoreNotificationMsg toCoreMsg = ToCoreNotificationMsg.newBuilder().setComponentLifecycleMsg(ByteString.copyFrom(msgBytes)).build();
toCoreNfProducer.send(tpi, new TbProtoQueueMsg<>(msg.getEntityId().getId(), toCoreMsg), null);
toCoreNfs.incrementAndGet();
@ -358,7 +375,7 @@ public class DefaultTbClusterService implements TbClusterService {
tbRuleEngineServices.removeAll(tbCoreServices);
}
for (String serviceId : tbRuleEngineServices) {
TopicPartitionInfo tpi = partitionService.getNotificationsTopic(ServiceType.TB_RULE_ENGINE, serviceId);
TopicPartitionInfo tpi = notificationsTopicService.getNotificationsTopic(ServiceType.TB_RULE_ENGINE, serviceId);
ToRuleEngineNotificationMsg toRuleEngineMsg = ToRuleEngineNotificationMsg.newBuilder().setComponentLifecycleMsg(ByteString.copyFrom(msgBytes)).build();
toRuleEngineProducer.send(tpi, new TbProtoQueueMsg<>(msg.getEntityId().getId(), toRuleEngineMsg), null);
toRuleEngineNfs.incrementAndGet();
@ -473,4 +490,65 @@ public class DefaultTbClusterService implements TbClusterService {
break;
}
}
@Override
public void onQueueChange(Queue queue) {
log.trace("[{}][{}] Processing queue change [{}] event", queue.getTenantId(), queue.getId(), queue.getName());
TransportProtos.QueueUpdateMsg queueUpdateMsg = TransportProtos.QueueUpdateMsg.newBuilder()
.setTenantIdMSB(queue.getTenantId().getId().getMostSignificantBits())
.setTenantIdLSB(queue.getTenantId().getId().getLeastSignificantBits())
.setQueueIdMSB(queue.getId().getId().getMostSignificantBits())
.setQueueIdLSB(queue.getId().getId().getLeastSignificantBits())
.setQueueName(queue.getName())
.setQueueTopic(queue.getTopic())
.setPartitions(queue.getPartitions())
.build();
ToRuleEngineNotificationMsg ruleEngineMsg = ToRuleEngineNotificationMsg.newBuilder().setQueueUpdateMsg(queueUpdateMsg).build();
ToCoreNotificationMsg coreMsg = ToCoreNotificationMsg.newBuilder().setQueueUpdateMsg(queueUpdateMsg).build();
ToTransportMsg transportMsg = ToTransportMsg.newBuilder().setQueueUpdateMsg(queueUpdateMsg).build();
doSendQueueNotifications(ruleEngineMsg, coreMsg, transportMsg);
}
@Override
public void onQueueDelete(Queue queue) {
log.trace("[{}][{}] Processing queue delete [{}] event", queue.getTenantId(), queue.getId(), queue.getName());
TransportProtos.QueueDeleteMsg queueDeleteMsg = TransportProtos.QueueDeleteMsg.newBuilder()
.setTenantIdMSB(queue.getTenantId().getId().getMostSignificantBits())
.setTenantIdLSB(queue.getTenantId().getId().getLeastSignificantBits())
.setQueueIdMSB(queue.getId().getId().getMostSignificantBits())
.setQueueIdLSB(queue.getId().getId().getLeastSignificantBits())
.setQueueName(queue.getName())
.build();
ToRuleEngineNotificationMsg ruleEngineMsg = ToRuleEngineNotificationMsg.newBuilder().setQueueDeleteMsg(queueDeleteMsg).build();
ToCoreNotificationMsg coreMsg = ToCoreNotificationMsg.newBuilder().setQueueDeleteMsg(queueDeleteMsg).build();
ToTransportMsg transportMsg = ToTransportMsg.newBuilder().setQueueDeleteMsg(queueDeleteMsg).build();
doSendQueueNotifications(ruleEngineMsg, coreMsg, transportMsg);
}
private void doSendQueueNotifications(ToRuleEngineNotificationMsg ruleEngineMsg, ToCoreNotificationMsg coreMsg, ToTransportMsg transportMsg) {
Set<TransportProtos.ServiceInfo> tbRuleEngineServices = partitionService.getAllServices(ServiceType.TB_RULE_ENGINE);
for (TransportProtos.ServiceInfo ruleEngineService : tbRuleEngineServices) {
TopicPartitionInfo tpi = notificationsTopicService.getNotificationsTopic(ServiceType.TB_RULE_ENGINE, ruleEngineService.getServiceId());
producerProvider.getRuleEngineNotificationsMsgProducer().send(tpi, new TbProtoQueueMsg<>(UUID.randomUUID(), ruleEngineMsg), null);
toRuleEngineNfs.incrementAndGet();
}
if (!serviceType.equals("monolith")) {
Set<TransportProtos.ServiceInfo> tbCoreServices = partitionService.getAllServices(ServiceType.TB_CORE);
for (TransportProtos.ServiceInfo coreService : tbCoreServices) {
TopicPartitionInfo tpi = notificationsTopicService.getNotificationsTopic(ServiceType.TB_CORE, coreService.getServiceId());
producerProvider.getTbCoreNotificationsMsgProducer().send(tpi, new TbProtoQueueMsg<>(UUID.randomUUID(), coreMsg), null);
toCoreNfs.incrementAndGet();
}
Set<TransportProtos.ServiceInfo> tbTransportServices = partitionService.getAllServices(ServiceType.TB_TRANSPORT);
for (TransportProtos.ServiceInfo transportService : tbTransportServices) {
TopicPartitionInfo tpi = notificationsTopicService.getNotificationsTopic(ServiceType.TB_TRANSPORT, transportService.getServiceId());
producerProvider.getTransportNotificationsMsgProducer().send(tpi, new TbProtoQueueMsg<>(UUID.randomUUID(), transportMsg), null);
toTransportNfs.incrementAndGet();
}
}
}
}

14
application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java

@ -38,6 +38,7 @@ import org.thingsboard.server.common.msg.rpc.FromDeviceRpcResponse;
import org.thingsboard.server.common.stats.StatsFactory;
import org.thingsboard.server.common.transport.util.DataDecodingEncodingService;
import org.thingsboard.server.dao.tenant.TbTenantProfileCache;
import org.thingsboard.server.gen.transport.TransportProtos;
import org.thingsboard.server.gen.transport.TransportProtos.DeviceStateServiceMsgProto;
import org.thingsboard.server.gen.transport.TransportProtos.EdgeNotificationMsgProto;
import org.thingsboard.server.gen.transport.TransportProtos.FromDeviceRPCResponseProto;
@ -57,6 +58,7 @@ import org.thingsboard.server.gen.transport.TransportProtos.ToUsageStatsServiceM
import org.thingsboard.server.gen.transport.TransportProtos.TransportToDeviceActorMsg;
import org.thingsboard.server.queue.TbQueueConsumer;
import org.thingsboard.server.queue.common.TbProtoQueueMsg;
import org.thingsboard.server.queue.discovery.PartitionService;
import org.thingsboard.server.queue.discovery.event.PartitionChangeEvent;
import org.thingsboard.server.queue.provider.TbCoreQueueFactory;
import org.thingsboard.server.queue.util.TbCoreComponent;
@ -135,8 +137,8 @@ public class DefaultTbCoreConsumerService extends AbstractConsumerService<ToCore
TbTenantProfileCache tenantProfileCache,
TbApiUsageStateService apiUsageStateService,
EdgeNotificationService edgeNotificationService,
OtaPackageStateService firmwareStateService) {
super(actorContext, encodingService, tenantProfileCache, deviceProfileCache, apiUsageStateService, tbCoreQueueFactory.createToCoreNotificationsMsgConsumer());
OtaPackageStateService firmwareStateService, PartitionService partitionService) {
super(actorContext, encodingService, tenantProfileCache, deviceProfileCache, apiUsageStateService, partitionService, tbCoreQueueFactory.createToCoreNotificationsMsgConsumer());
this.mainConsumer = tbCoreQueueFactory.createToCoreMsgConsumer();
this.usageStatsConsumer = tbCoreQueueFactory.createToUsageStatsServiceMsgConsumer();
this.firmwareStatesConsumer = tbCoreQueueFactory.createToOtaPackageStateServiceMsgConsumer();
@ -312,6 +314,14 @@ public class DefaultTbCoreConsumerService extends AbstractConsumerService<ToCore
actorContext.tellWithHighPriority(actorMsg.get());
}
callback.onSuccess();
} else if (toCoreNotification.hasQueueUpdateMsg()) {
TransportProtos.QueueUpdateMsg queue = toCoreNotification.getQueueUpdateMsg();
partitionService.updateQueue(queue);
callback.onSuccess();
} else if (toCoreNotification.hasQueueDeleteMsg()) {
TransportProtos.QueueDeleteMsg queue = toCoreNotification.getQueueDeleteMsg();
partitionService.removeQueue(queue);
callback.onSuccess();
}
if (statsEnabled) {
stats.log(toCoreNotification);

184
application/src/main/java/org/thingsboard/server/service/queue/DefaultTbRuleEngineConsumerService.java

@ -21,33 +21,37 @@ import org.springframework.beans.factory.annotation.Value;
import org.springframework.scheduling.annotation.Scheduled;
import org.springframework.stereotype.Service;
import org.thingsboard.common.util.ThingsBoardThreadFactory;
import org.thingsboard.server.common.data.rpc.RpcError;
import org.thingsboard.server.actors.ActorSystemContext;
import org.thingsboard.server.common.data.id.QueueId;
import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.data.queue.Queue;
import org.thingsboard.server.common.data.rpc.RpcError;
import org.thingsboard.server.common.msg.TbMsg;
import org.thingsboard.server.common.msg.queue.QueueToRuleEngineMsg;
import org.thingsboard.server.common.msg.queue.RuleEngineException;
import org.thingsboard.server.common.msg.queue.RuleNodeInfo;
import org.thingsboard.server.common.msg.queue.ServiceQueue;
import org.thingsboard.server.common.msg.queue.ServiceType;
import org.thingsboard.server.common.msg.queue.TbCallback;
import org.thingsboard.server.common.msg.queue.TbMsgCallback;
import org.thingsboard.server.common.msg.queue.TopicPartitionInfo;
import org.thingsboard.server.common.msg.rpc.FromDeviceRpcResponse;
import org.thingsboard.server.common.stats.StatsFactory;
import org.thingsboard.server.common.transport.util.DataDecodingEncodingService;
import org.thingsboard.server.dao.queue.QueueService;
import org.thingsboard.server.dao.tenant.TbTenantProfileCache;
import org.thingsboard.server.gen.transport.TransportProtos;
import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg;
import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineNotificationMsg;
import org.thingsboard.server.queue.TbQueueConsumer;
import org.thingsboard.server.queue.common.TbProtoQueueMsg;
import org.thingsboard.server.queue.discovery.PartitionService;
import org.thingsboard.server.queue.discovery.QueueKey;
import org.thingsboard.server.queue.discovery.TbServiceInfoProvider;
import org.thingsboard.server.queue.discovery.event.PartitionChangeEvent;
import org.thingsboard.server.queue.provider.TbRuleEngineQueueFactory;
import org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings;
import org.thingsboard.server.queue.settings.TbRuleEngineQueueConfiguration;
import org.thingsboard.server.queue.util.TbRuleEngineComponent;
import org.thingsboard.server.service.apiusage.TbApiUsageStateService;
import org.thingsboard.server.service.profile.TbDeviceProfileCache;
import org.thingsboard.server.dao.tenant.TbTenantProfileCache;
import org.thingsboard.server.service.queue.processing.AbstractConsumerService;
import org.thingsboard.server.service.queue.processing.TbRuleEngineProcessingDecision;
import org.thingsboard.server.service.queue.processing.TbRuleEngineProcessingResult;
@ -55,25 +59,26 @@ import org.thingsboard.server.service.queue.processing.TbRuleEngineProcessingStr
import org.thingsboard.server.service.queue.processing.TbRuleEngineProcessingStrategyFactory;
import org.thingsboard.server.service.queue.processing.TbRuleEngineSubmitStrategy;
import org.thingsboard.server.service.queue.processing.TbRuleEngineSubmitStrategyFactory;
import org.thingsboard.server.common.msg.rpc.FromDeviceRpcResponse;
import org.thingsboard.server.service.rpc.TbRuleEngineDeviceRpcService;
import org.thingsboard.server.service.stats.RuleEngineStatisticsService;
import javax.annotation.PostConstruct;
import javax.annotation.PreDestroy;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Queue;
import java.util.Set;
import java.util.UUID;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.locks.ReentrantLock;
@Service
@TbRuleEngineComponent
@ -96,19 +101,20 @@ public class DefaultTbRuleEngineConsumerService extends AbstractConsumerService<
private final TbRuleEngineSubmitStrategyFactory submitStrategyFactory;
private final TbRuleEngineProcessingStrategyFactory processingStrategyFactory;
private final TbRuleEngineQueueFactory tbRuleEngineQueueFactory;
private final TbQueueRuleEngineSettings ruleEngineSettings;
private final RuleEngineStatisticsService statisticsService;
private final TbRuleEngineDeviceRpcService tbDeviceRpcService;
private final ConcurrentMap<String, TbQueueConsumer<TbProtoQueueMsg<ToRuleEngineMsg>>> consumers = new ConcurrentHashMap<>();
private final ConcurrentMap<String, TbRuleEngineQueueConfiguration> consumerConfigurations = new ConcurrentHashMap<>();
private final ConcurrentMap<String, TbRuleEngineConsumerStats> consumerStats = new ConcurrentHashMap<>();
private final ConcurrentMap<String, TbTopicWithConsumerPerPartition> topicsConsumerPerPartition = new ConcurrentHashMap<>();
private final TbServiceInfoProvider serviceInfoProvider;
private final QueueService queueService;
// private final TenantId tenantId;
private final ConcurrentMap<QueueKey, TbQueueConsumer<TbProtoQueueMsg<ToRuleEngineMsg>>> consumers = new ConcurrentHashMap<>();
private final ConcurrentMap<QueueKey, Queue> consumerConfigurations = new ConcurrentHashMap<>();
private final ConcurrentMap<QueueKey, TbRuleEngineConsumerStats> consumerStats = new ConcurrentHashMap<>();
private final ConcurrentMap<QueueKey, TbTopicWithConsumerPerPartition> topicsConsumerPerPartition = new ConcurrentHashMap<>();
final ExecutorService submitExecutor = Executors.newSingleThreadExecutor(ThingsBoardThreadFactory.forName("tb-rule-engine-consumer-submit"));
final ScheduledExecutorService repartitionExecutor = Executors.newScheduledThreadPool(1, ThingsBoardThreadFactory.forName("tb-rule-engine-consumer-repartition"));
public DefaultTbRuleEngineConsumerService(TbRuleEngineProcessingStrategyFactory processingStrategyFactory,
TbRuleEngineSubmitStrategyFactory submitStrategyFactory,
TbQueueRuleEngineSettings ruleEngineSettings,
TbRuleEngineQueueFactory tbRuleEngineQueueFactory,
RuleEngineStatisticsService statisticsService,
ActorSystemContext actorContext,
@ -117,28 +123,36 @@ public class DefaultTbRuleEngineConsumerService extends AbstractConsumerService<
StatsFactory statsFactory,
TbDeviceProfileCache deviceProfileCache,
TbTenantProfileCache tenantProfileCache,
TbApiUsageStateService apiUsageStateService) {
super(actorContext, encodingService, tenantProfileCache, deviceProfileCache, apiUsageStateService, tbRuleEngineQueueFactory.createToRuleEngineNotificationsMsgConsumer());
TbApiUsageStateService apiUsageStateService,
PartitionService partitionService, TbServiceInfoProvider serviceInfoProvider, QueueService queueService) {
super(actorContext, encodingService, tenantProfileCache, deviceProfileCache, apiUsageStateService, partitionService, tbRuleEngineQueueFactory.createToRuleEngineNotificationsMsgConsumer());
this.statisticsService = statisticsService;
this.ruleEngineSettings = ruleEngineSettings;
this.tbRuleEngineQueueFactory = tbRuleEngineQueueFactory;
this.submitStrategyFactory = submitStrategyFactory;
this.processingStrategyFactory = processingStrategyFactory;
this.tbDeviceRpcService = tbDeviceRpcService;
this.statsFactory = statsFactory;
this.serviceInfoProvider = serviceInfoProvider;
this.queueService = queueService;
}
@PostConstruct
public void init() {
super.init("tb-rule-engine-consumer", "tb-rule-engine-notifications-consumer");
for (TbRuleEngineQueueConfiguration configuration : ruleEngineSettings.getQueues()) {
consumerConfigurations.putIfAbsent(configuration.getName(), configuration);
consumerStats.put(configuration.getName(), new TbRuleEngineConsumerStats(configuration.getName(), statsFactory));
if (!configuration.isConsumerPerPartition()) {
consumers.computeIfAbsent(configuration.getName(), queueName -> tbRuleEngineQueueFactory.createToRuleEngineMsgConsumer(configuration));
} else {
topicsConsumerPerPartition.computeIfAbsent(configuration.getName(), TbTopicWithConsumerPerPartition::new);
}
List<Queue> queues = queueService.findAllQueues();
for (Queue configuration : queues) {
initConsumer(configuration);
}
}
private void initConsumer(Queue configuration) {
QueueKey queueKey = new QueueKey(ServiceType.TB_RULE_ENGINE, configuration);
consumerConfigurations.putIfAbsent(queueKey, configuration);
consumerStats.putIfAbsent(queueKey, new TbRuleEngineConsumerStats(configuration.getName(), statsFactory));
if (!configuration.isConsumerPerPartition()) {
consumers.computeIfAbsent(queueKey, queueName -> tbRuleEngineQueueFactory.createToRuleEngineMsgConsumer(configuration));
} else {
topicsConsumerPerPartition.computeIfAbsent(queueKey, k -> new TbTopicWithConsumerPerPartition(k.getQueueName()));
}
}
@ -147,38 +161,37 @@ public class DefaultTbRuleEngineConsumerService extends AbstractConsumerService<
super.destroy();
submitExecutor.shutdownNow();
repartitionExecutor.shutdownNow();
ruleEngineSettings.getQueues().forEach(config -> consumerConfigurations.put(config.getName(), config));
}
@Override
protected void onTbApplicationEvent(PartitionChangeEvent event) {
if (event.getServiceType().equals(getServiceType())) {
ServiceQueue serviceQueue = event.getServiceQueueKey().getServiceQueue();
log.info("[{}] Subscribing to partitions: {}", serviceQueue.getQueue(), event.getPartitions());
if (!consumerConfigurations.get(serviceQueue.getQueue()).isConsumerPerPartition()) {
consumers.get(serviceQueue.getQueue()).subscribe(event.getPartitions());
String serviceQueue = event.getQueueKey().getQueueName();
log.info("[{}] Subscribing to partitions: {}", serviceQueue, event.getPartitions());
if (!consumerConfigurations.get(event.getQueueKey()).isConsumerPerPartition()) {
consumers.get(event.getQueueKey()).subscribe(event.getPartitions());
} else {
log.info("[{}] Subscribing consumer per partition: {}", serviceQueue.getQueue(), event.getPartitions());
subscribeConsumerPerPartition(serviceQueue.getQueue(), event.getPartitions());
log.info("[{}] Subscribing consumer per partition: {}", serviceQueue, event.getPartitions());
subscribeConsumerPerPartition(event.getQueueKey(), event.getPartitions());
}
}
}
void subscribeConsumerPerPartition(String queue, Set<TopicPartitionInfo> partitions) {
void subscribeConsumerPerPartition(QueueKey queue, Set<TopicPartitionInfo> partitions) {
topicsConsumerPerPartition.get(queue).getSubscribeQueue().add(partitions);
scheduleTopicRepartition(queue);
}
private void scheduleTopicRepartition(String queue) {
private void scheduleTopicRepartition(QueueKey queue) {
repartitionExecutor.schedule(() -> repartitionTopicWithConsumerPerPartition(queue), 1, TimeUnit.SECONDS);
}
void repartitionTopicWithConsumerPerPartition(final String queueName) {
void repartitionTopicWithConsumerPerPartition(final QueueKey queueKey) {
if (stopped) {
return;
}
TbTopicWithConsumerPerPartition tbTopicWithConsumerPerPartition = topicsConsumerPerPartition.get(queueName);
Queue<Set<TopicPartitionInfo>> subscribeQueue = tbTopicWithConsumerPerPartition.getSubscribeQueue();
TbTopicWithConsumerPerPartition tbTopicWithConsumerPerPartition = topicsConsumerPerPartition.get(queueKey);
java.util.Queue<Set<TopicPartitionInfo>> subscribeQueue = tbTopicWithConsumerPerPartition.getSubscribeQueue();
if (subscribeQueue.isEmpty()) {
return;
}
@ -202,15 +215,15 @@ public class DefaultTbRuleEngineConsumerService extends AbstractConsumerService<
log.info("calculated removedPartitions {}", removedPartitions);
removedPartitions.forEach((tpi) -> {
removeConsumerForTopicByTpi(queueName, consumers, tpi);
removeConsumerForTopicByTpi(queueKey.getQueueName(), consumers, tpi);
});
addedPartitions.forEach((tpi) -> {
log.info("[{}] Adding consumer for topic: {}", queueName, tpi);
TbRuleEngineQueueConfiguration configuration = consumerConfigurations.get(queueName);
log.info("[{}] Adding consumer for topic: {}", queueKey, tpi);
Queue configuration = consumerConfigurations.get(queueKey);
TbQueueConsumer<TbProtoQueueMsg<ToRuleEngineMsg>> consumer = tbRuleEngineQueueFactory.createToRuleEngineMsgConsumer(configuration);
consumers.put(tpi, consumer);
launchConsumer(consumer, consumerConfigurations.get(queueName), consumerStats.get(queueName), "" + queueName + "-" + tpi.getPartition().orElse(-999999));
launchConsumer(consumer, consumerConfigurations.get(queueKey), consumerStats.get(queueKey), "" + queueKey + "-" + tpi.getPartition().orElse(-999999));
consumer.subscribe(Collections.singleton(tpi));
});
@ -218,7 +231,7 @@ public class DefaultTbRuleEngineConsumerService extends AbstractConsumerService<
tbTopicWithConsumerPerPartition.getLock().unlock();
}
} else {
scheduleTopicRepartition(queueName); //reschedule later
scheduleTopicRepartition(queueKey); //reschedule later
}
}
@ -231,7 +244,7 @@ public class DefaultTbRuleEngineConsumerService extends AbstractConsumerService<
@Override
protected void launchMainConsumers() {
consumers.forEach((queue, consumer) -> launchConsumer(consumer, consumerConfigurations.get(queue), consumerStats.get(queue), queue));
consumers.forEach((queue, consumer) -> launchConsumer(consumer, consumerConfigurations.get(queue), consumerStats.get(queue), queue.getQueueName()));
}
@Override
@ -241,11 +254,11 @@ public class DefaultTbRuleEngineConsumerService extends AbstractConsumerService<
.forEach((tpi) -> removeConsumerForTopicByTpi(tbTopicWithConsumerPerPartition.getTopic(), tbTopicWithConsumerPerPartition.getConsumers(), tpi)));
}
void launchConsumer(TbQueueConsumer<TbProtoQueueMsg<ToRuleEngineMsg>> consumer, TbRuleEngineQueueConfiguration configuration, TbRuleEngineConsumerStats stats, String threadSuffix) {
void launchConsumer(TbQueueConsumer<TbProtoQueueMsg<ToRuleEngineMsg>> consumer, Queue configuration, TbRuleEngineConsumerStats stats, String threadSuffix) {
consumersExecutor.execute(() -> consumerLoop(consumer, configuration, stats, threadSuffix));
}
void consumerLoop(TbQueueConsumer<TbProtoQueueMsg<ToRuleEngineMsg>> consumer, TbRuleEngineQueueConfiguration configuration, TbRuleEngineConsumerStats stats, String threadSuffix) {
void consumerLoop(TbQueueConsumer<TbProtoQueueMsg<ToRuleEngineMsg>> consumer, org.thingsboard.server.common.data.queue.Queue configuration, TbRuleEngineConsumerStats stats, String threadSuffix) {
updateCurrentThreadName(threadSuffix);
while (!stopped && !consumer.isStopped()) {
try {
@ -256,13 +269,13 @@ public class DefaultTbRuleEngineConsumerService extends AbstractConsumerService<
final TbRuleEngineSubmitStrategy submitStrategy = getSubmitStrategy(configuration);
final TbRuleEngineProcessingStrategy ackStrategy = getAckStrategy(configuration);
submitStrategy.init(msgs);
while (!stopped) {
while (!stopped && !consumer.isStopped()) {
TbMsgPackProcessingContext ctx = new TbMsgPackProcessingContext(configuration.getName(), submitStrategy, ackStrategy.isSkipTimeoutMsgs());
submitStrategy.submitAttempt((id, msg) -> submitExecutor.submit(() -> submitMessage(configuration, stats, ctx, id, msg)));
final boolean timeout = !ctx.await(configuration.getPackProcessingTimeout(), TimeUnit.MILLISECONDS);
TbRuleEngineProcessingResult result = new TbRuleEngineProcessingResult(configuration.getName(), timeout, ctx);
TbRuleEngineProcessingResult result = new TbRuleEngineProcessingResult(configuration.getId(), timeout, ctx);
if (timeout) {
printFirstOrAll(configuration, ctx, ctx.getPendingMap(), "Timeout");
}
@ -310,15 +323,15 @@ public class DefaultTbRuleEngineConsumerService extends AbstractConsumerService<
Thread.currentThread().setName(name);
}
TbRuleEngineProcessingStrategy getAckStrategy(TbRuleEngineQueueConfiguration configuration) {
TbRuleEngineProcessingStrategy getAckStrategy(Queue configuration) {
return processingStrategyFactory.newInstance(configuration.getName(), configuration.getProcessingStrategy());
}
TbRuleEngineSubmitStrategy getSubmitStrategy(TbRuleEngineQueueConfiguration configuration) {
TbRuleEngineSubmitStrategy getSubmitStrategy(Queue configuration) {
return submitStrategyFactory.newInstance(configuration.getName(), configuration.getSubmitStrategy());
}
void submitMessage(TbRuleEngineQueueConfiguration configuration, TbRuleEngineConsumerStats stats, TbMsgPackProcessingContext ctx, UUID id, TbProtoQueueMsg<ToRuleEngineMsg> msg) {
void submitMessage(Queue configuration, TbRuleEngineConsumerStats stats, TbMsgPackProcessingContext ctx, UUID id, TbProtoQueueMsg<ToRuleEngineMsg> msg) {
log.trace("[{}] Creating callback for topic {} message: {}", id, configuration.getName(), msg.getValue());
ToRuleEngineMsg toRuleEngineMsg = msg.getValue();
TenantId tenantId = TenantId.fromUUID(new UUID(toRuleEngineMsg.getTenantIdMSB(), toRuleEngineMsg.getTenantIdLSB()));
@ -327,7 +340,7 @@ public class DefaultTbRuleEngineConsumerService extends AbstractConsumerService<
new TbMsgPackCallback(id, tenantId, ctx);
try {
if (toRuleEngineMsg.getTbMsg() != null && !toRuleEngineMsg.getTbMsg().isEmpty()) {
forwardToRuleEngineActor(configuration.getName(), tenantId, toRuleEngineMsg, callback);
forwardToRuleEngineActor(configuration.getId(), tenantId, toRuleEngineMsg, callback);
} else {
callback.onSuccess();
}
@ -336,12 +349,12 @@ public class DefaultTbRuleEngineConsumerService extends AbstractConsumerService<
}
}
private void printFirstOrAll(TbRuleEngineQueueConfiguration configuration, TbMsgPackProcessingContext ctx, Map<UUID, TbProtoQueueMsg<ToRuleEngineMsg>> map, String prefix) {
private void printFirstOrAll(Queue configuration, TbMsgPackProcessingContext ctx, Map<UUID, TbProtoQueueMsg<ToRuleEngineMsg>> map, String prefix) {
boolean printAll = log.isTraceEnabled();
log.info("{} to process [{}] messages", prefix, map.size());
for (Map.Entry<UUID, TbProtoQueueMsg<ToRuleEngineMsg>> pending : map.entrySet()) {
ToRuleEngineMsg tmp = pending.getValue().getValue();
TbMsg tmpMsg = TbMsg.fromBytes(configuration.getName(), tmp.getTbMsg().toByteArray(), TbMsgCallback.EMPTY);
TbMsg tmpMsg = TbMsg.fromBytes(configuration.getId(), tmp.getTbMsg().toByteArray(), TbMsgCallback.EMPTY);
RuleNodeInfo ruleNodeInfo = ctx.getLastVisitedRuleNode(pending.getKey());
if (printAll) {
log.trace("[{}] {} to process message: {}, Last Rule Node: {}", TenantId.fromUUID(new UUID(tmp.getTenantIdMSB(), tmp.getTenantIdLSB())), prefix, tmpMsg, ruleNodeInfo);
@ -380,14 +393,77 @@ public class DefaultTbRuleEngineConsumerService extends AbstractConsumerService<
, proto.getResponse(), error);
tbDeviceRpcService.processRpcResponseFromDevice(response);
callback.onSuccess();
} else if (nfMsg.hasQueueUpdateMsg()) {
repartitionExecutor.execute(() -> updateQueue(nfMsg.getQueueUpdateMsg()));
callback.onSuccess();
} else if (nfMsg.hasQueueDeleteMsg()) {
repartitionExecutor.execute(() -> deleteQueue(nfMsg.getQueueDeleteMsg()));
callback.onSuccess();
} else {
log.trace("Received notification with missing handler");
callback.onSuccess();
}
}
private void forwardToRuleEngineActor(String queueName, TenantId tenantId, ToRuleEngineMsg toRuleEngineMsg, TbMsgCallback callback) {
TbMsg tbMsg = TbMsg.fromBytes(queueName, toRuleEngineMsg.getTbMsg().toByteArray(), callback);
private void updateQueue(TransportProtos.QueueUpdateMsg queueUpdateMsg) {
log.info("Received queue update msg: [{}]", queueUpdateMsg);
String queueName = queueUpdateMsg.getQueueName();
TenantId tenantId = new TenantId(new UUID(queueUpdateMsg.getTenantIdMSB(), queueUpdateMsg.getTenantIdLSB()));
QueueId queueId = new QueueId(new UUID(queueUpdateMsg.getQueueIdMSB(), queueUpdateMsg.getQueueIdLSB()));
QueueKey queueKey = new QueueKey(ServiceType.TB_RULE_ENGINE, queueUpdateMsg.getQueueName(), tenantId);
Queue queue = queueService.findQueueById(tenantId, queueId);
Queue oldQueue = consumerConfigurations.remove(queueKey);
if (oldQueue != null) {
if (oldQueue.isConsumerPerPartition()) {
TbTopicWithConsumerPerPartition consumerPerPartition = topicsConsumerPerPartition.remove(queueKey);
ReentrantLock lock = consumerPerPartition.getLock();
try {
lock.lock();
consumerPerPartition.getConsumers().values().forEach(TbQueueConsumer::unsubscribe);
} finally {
lock.unlock();
}
} else {
TbQueueConsumer<TbProtoQueueMsg<ToRuleEngineMsg>> consumer = consumers.remove(queueKey);
consumer.unsubscribe();
}
}
initConsumer(queue);
if (!queue.isConsumerPerPartition()) {
launchConsumer(consumers.get(queueKey), consumerConfigurations.get(queueKey), consumerStats.get(queueKey), queueName);
}
partitionService.updateQueue(queueUpdateMsg);
partitionService.recalculatePartitions(serviceInfoProvider.getServiceInfo(), new ArrayList<>(partitionService.getOtherServices(ServiceType.TB_RULE_ENGINE)));
}
private void deleteQueue(TransportProtos.QueueDeleteMsg queueDeleteMsg) {
log.info("Received queue delete msg: [{}]", queueDeleteMsg);
TenantId tenantId = new TenantId(new UUID(queueDeleteMsg.getTenantIdMSB(), queueDeleteMsg.getTenantIdLSB()));
QueueKey queueKey = new QueueKey(ServiceType.TB_RULE_ENGINE, queueDeleteMsg.getQueueName(), tenantId);
Queue queue = consumerConfigurations.remove(queueKey);
if (queue != null) {
if (queue.isConsumerPerPartition()) {
TbTopicWithConsumerPerPartition tbTopicWithConsumerPerPartition = topicsConsumerPerPartition.remove(queueKey);
if (tbTopicWithConsumerPerPartition != null) {
tbTopicWithConsumerPerPartition.getConsumers().values().forEach(TbQueueConsumer::unsubscribe);
tbTopicWithConsumerPerPartition.getConsumers().clear();
}
} else {
TbQueueConsumer<TbProtoQueueMsg<ToRuleEngineMsg>> consumer = consumers.remove(queueKey);
if (consumer != null) {
consumer.unsubscribe();
}
}
}
partitionService.removeQueue(queueDeleteMsg);
}
private void forwardToRuleEngineActor(QueueId queueId, TenantId tenantId, ToRuleEngineMsg toRuleEngineMsg, TbMsgCallback callback) {
TbMsg tbMsg = TbMsg.fromBytes(queueId, toRuleEngineMsg.getTbMsg().toByteArray(), callback);
QueueToRuleEngineMsg msg;
ProtocolStringList relationTypesList = toRuleEngineMsg.getRelationTypesList();
Set<String> relationTypes = null;

2
application/src/main/java/org/thingsboard/server/service/queue/TbTopicWithConsumerPerPartition.java

@ -15,7 +15,6 @@
*/
package org.thingsboard.server.service.queue;
import lombok.Builder;
import lombok.Data;
import lombok.Getter;
import lombok.RequiredArgsConstructor;
@ -25,7 +24,6 @@ import org.thingsboard.server.queue.TbQueueConsumer;
import org.thingsboard.server.queue.common.TbProtoQueueMsg;
import java.util.Collections;
import java.util.Map;
import java.util.Queue;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;

7
application/src/main/java/org/thingsboard/server/service/queue/processing/AbstractConsumerService.java

@ -35,6 +35,7 @@ import org.thingsboard.server.common.msg.queue.ServiceType;
import org.thingsboard.server.common.msg.queue.TbCallback;
import org.thingsboard.server.queue.TbQueueConsumer;
import org.thingsboard.server.queue.common.TbProtoQueueMsg;
import org.thingsboard.server.queue.discovery.PartitionService;
import org.thingsboard.server.queue.discovery.event.PartitionChangeEvent;
import org.thingsboard.server.common.transport.util.DataDecodingEncodingService;
import org.thingsboard.server.queue.discovery.TbApplicationEventListener;
@ -69,17 +70,20 @@ public abstract class AbstractConsumerService<N extends com.google.protobuf.Gene
protected final TbTenantProfileCache tenantProfileCache;
protected final TbDeviceProfileCache deviceProfileCache;
protected final TbApiUsageStateService apiUsageStateService;
protected final PartitionService partitionService;
protected final TbQueueConsumer<TbProtoQueueMsg<N>> nfConsumer;
public AbstractConsumerService(ActorSystemContext actorContext, DataDecodingEncodingService encodingService,
TbTenantProfileCache tenantProfileCache, TbDeviceProfileCache deviceProfileCache,
TbApiUsageStateService apiUsageStateService, TbQueueConsumer<TbProtoQueueMsg<N>> nfConsumer) {
TbApiUsageStateService apiUsageStateService, PartitionService partitionService,
TbQueueConsumer<TbProtoQueueMsg<N>> nfConsumer) {
this.actorContext = actorContext;
this.encodingService = encodingService;
this.tenantProfileCache = tenantProfileCache;
this.deviceProfileCache = deviceProfileCache;
this.apiUsageStateService = apiUsageStateService;
this.partitionService = partitionService;
this.nfConsumer = nfConsumer;
}
@ -166,6 +170,7 @@ public abstract class AbstractConsumerService<N extends com.google.protobuf.Gene
}
} else if (EntityType.TENANT.equals(componentLifecycleMsg.getEntityId().getEntityType())) {
tenantProfileCache.evict(componentLifecycleMsg.getTenantId());
partitionService.removeTenant(componentLifecycleMsg.getTenantId());
if (componentLifecycleMsg.getEvent().equals(ComponentLifecycleEvent.UPDATED)) {
apiUsageStateService.onTenantUpdate(componentLifecycleMsg.getTenantId());
} else if (componentLifecycleMsg.getEvent().equals(ComponentLifecycleEvent.DELETED)) {

7
application/src/main/java/org/thingsboard/server/service/queue/processing/TbRuleEngineProcessingResult.java

@ -16,6 +16,7 @@
package org.thingsboard.server.service.queue.processing;
import lombok.Getter;
import org.thingsboard.server.common.data.id.QueueId;
import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.msg.queue.RuleEngineException;
import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg;
@ -28,7 +29,7 @@ import java.util.concurrent.ConcurrentMap;
public class TbRuleEngineProcessingResult {
@Getter
private final String queueName;
private final QueueId queueId;
@Getter
private final boolean success;
@Getter
@ -36,8 +37,8 @@ public class TbRuleEngineProcessingResult {
@Getter
private final TbMsgPackProcessingContext ctx;
public TbRuleEngineProcessingResult(String queueName, boolean timeout, TbMsgPackProcessingContext ctx) {
this.queueName = queueName;
public TbRuleEngineProcessingResult(QueueId queueId, boolean timeout, TbMsgPackProcessingContext ctx) {
this.queueId = queueId;
this.timeout = timeout;
this.ctx = ctx;
this.success = !timeout && ctx.getPendingMap().isEmpty() && ctx.getFailedMap().isEmpty();

44
application/src/main/java/org/thingsboard/server/service/queue/processing/TbRuleEngineProcessingStrategyFactory.java

@ -18,11 +18,11 @@ package org.thingsboard.server.service.queue.processing;
import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Component;
import org.springframework.util.CollectionUtils;
import org.thingsboard.server.common.data.queue.ProcessingStrategy;
import org.thingsboard.server.common.msg.TbMsg;
import org.thingsboard.server.common.msg.queue.TbMsgCallback;
import org.thingsboard.server.gen.transport.TransportProtos;
import org.thingsboard.server.queue.common.TbProtoQueueMsg;
import org.thingsboard.server.queue.settings.TbRuleEngineQueueAckStrategyConfiguration;
import java.util.UUID;
import java.util.concurrent.ConcurrentHashMap;
@ -33,22 +33,22 @@ import java.util.concurrent.TimeUnit;
@Slf4j
public class TbRuleEngineProcessingStrategyFactory {
public TbRuleEngineProcessingStrategy newInstance(String name, TbRuleEngineQueueAckStrategyConfiguration configuration) {
switch (configuration.getType()) {
case "SKIP_ALL_FAILURES":
public TbRuleEngineProcessingStrategy newInstance(String name, ProcessingStrategy processingStrategy) {
switch (processingStrategy.getType()) {
case SKIP_ALL_FAILURES:
return new SkipStrategy(name, false);
case "SKIP_ALL_FAILURES_AND_TIMED_OUT":
case SKIP_ALL_FAILURES_AND_TIMED_OUT:
return new SkipStrategy(name, true);
case "RETRY_ALL":
return new RetryStrategy(name, true, true, true, configuration);
case "RETRY_FAILED":
return new RetryStrategy(name, false, true, false, configuration);
case "RETRY_TIMED_OUT":
return new RetryStrategy(name, false, false, true, configuration);
case "RETRY_FAILED_AND_TIMED_OUT":
return new RetryStrategy(name, false, true, true, configuration);
case RETRY_ALL:
return new RetryStrategy(name, true, true, true, processingStrategy);
case RETRY_FAILED:
return new RetryStrategy(name, false, true, false, processingStrategy);
case RETRY_TIMED_OUT:
return new RetryStrategy(name, false, false, true, processingStrategy);
case RETRY_FAILED_AND_TIMED_OUT:
return new RetryStrategy(name, false, true, true, processingStrategy);
default:
throw new RuntimeException("TbRuleEngineProcessingStrategy with type " + configuration.getType() + " is not supported!");
throw new RuntimeException("TbRuleEngineProcessingStrategy with type " + processingStrategy.getType() + " is not supported!");
}
}
@ -66,15 +66,15 @@ public class TbRuleEngineProcessingStrategyFactory {
private int initialTotalCount;
private int retryCount;
public RetryStrategy(String queueName, boolean retrySuccessful, boolean retryFailed, boolean retryTimeout, TbRuleEngineQueueAckStrategyConfiguration configuration) {
public RetryStrategy(String queueName, boolean retrySuccessful, boolean retryFailed, boolean retryTimeout, ProcessingStrategy processingStrategy) {
this.queueName = queueName;
this.retrySuccessful = retrySuccessful;
this.retryFailed = retryFailed;
this.retryTimeout = retryTimeout;
this.maxRetries = configuration.getRetries();
this.maxAllowedFailurePercentage = configuration.getFailurePercentage();
this.pauseBetweenRetries = configuration.getPauseBetweenRetries();
this.maxPauseBetweenRetries = configuration.getMaxPauseBetweenRetries();
this.maxRetries = processingStrategy.getRetries();
this.maxAllowedFailurePercentage = processingStrategy.getFailurePercentage();
this.pauseBetweenRetries = processingStrategy.getPauseBetweenRetries();
this.maxPauseBetweenRetries = processingStrategy.getMaxPauseBetweenRetries();
}
@Override
@ -125,7 +125,7 @@ public class TbRuleEngineProcessingStrategyFactory {
}
log.debug("[{}] Going to reprocess {} messages", queueName, toReprocess.size());
if (log.isTraceEnabled()) {
toReprocess.forEach((id, msg) -> log.trace("Going to reprocess [{}]: {}", id, TbMsg.fromBytes(result.getQueueName(), msg.getValue().getTbMsg().toByteArray(), TbMsgCallback.EMPTY)));
toReprocess.forEach((id, msg) -> log.trace("Going to reprocess [{}]: {}", id, TbMsg.fromBytes(result.getQueueId(), msg.getValue().getTbMsg().toByteArray(), TbMsgCallback.EMPTY)));
}
if (pauseBetweenRetries > 0) {
try {
@ -164,10 +164,10 @@ public class TbRuleEngineProcessingStrategyFactory {
log.debug("[{}] Reprocessing skipped for {} failed and {} timeout messages", queueName, result.getFailedMap().size(), result.getPendingMap().size());
}
if (log.isTraceEnabled()) {
result.getFailedMap().forEach((id, msg) -> log.trace("Failed messages [{}]: {}", id, TbMsg.fromBytes(result.getQueueName(), msg.getValue().getTbMsg().toByteArray(), TbMsgCallback.EMPTY)));
result.getFailedMap().forEach((id, msg) -> log.trace("Failed messages [{}]: {}", id, TbMsg.fromBytes(result.getQueueId(), msg.getValue().getTbMsg().toByteArray(), TbMsgCallback.EMPTY)));
}
if (log.isTraceEnabled()) {
result.getPendingMap().forEach((id, msg) -> log.trace("Timeout messages [{}]: {}", id, TbMsg.fromBytes(result.getQueueName(), msg.getValue().getTbMsg().toByteArray(), TbMsgCallback.EMPTY)));
result.getPendingMap().forEach((id, msg) -> log.trace("Timeout messages [{}]: {}", id, TbMsg.fromBytes(result.getQueueId(), msg.getValue().getTbMsg().toByteArray(), TbMsgCallback.EMPTY)));
}
return new TbRuleEngineProcessingDecision(true, null);
}

19
application/src/main/java/org/thingsboard/server/service/queue/processing/TbRuleEngineSubmitStrategyFactory.java

@ -17,26 +17,27 @@ package org.thingsboard.server.service.queue.processing;
import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Component;
import org.thingsboard.server.common.data.queue.SubmitStrategy;
import org.thingsboard.server.queue.settings.TbRuleEngineQueueSubmitStrategyConfiguration;
@Component
@Slf4j
public class TbRuleEngineSubmitStrategyFactory {
public TbRuleEngineSubmitStrategy newInstance(String name, TbRuleEngineQueueSubmitStrategyConfiguration configuration) {
switch (configuration.getType()) {
case "BURST":
public TbRuleEngineSubmitStrategy newInstance(String name, SubmitStrategy submitStrategy) {
switch (submitStrategy.getType()) {
case BURST:
return new BurstTbRuleEngineSubmitStrategy(name);
case "BATCH":
return new BatchTbRuleEngineSubmitStrategy(name, configuration.getBatchSize());
case "SEQUENTIAL_BY_ORIGINATOR":
case BATCH:
return new BatchTbRuleEngineSubmitStrategy(name, submitStrategy.getBatchSize());
case SEQUENTIAL_BY_ORIGINATOR:
return new SequentialByOriginatorIdTbRuleEngineSubmitStrategy(name);
case "SEQUENTIAL_BY_TENANT":
case SEQUENTIAL_BY_TENANT:
return new SequentialByTenantIdTbRuleEngineSubmitStrategy(name);
case "SEQUENTIAL":
case SEQUENTIAL:
return new SequentialTbRuleEngineSubmitStrategy(name);
default:
throw new RuntimeException("TbRuleEngineProcessingStrategy with type " + configuration.getType() + " is not supported!");
throw new RuntimeException("TbRuleEngineProcessingStrategy with type " + submitStrategy.getType() + " is not supported!");
}
}

3
application/src/main/java/org/thingsboard/server/service/security/permission/Resource.java

@ -40,7 +40,8 @@ public enum Resource {
TB_RESOURCE(EntityType.TB_RESOURCE),
OTA_PACKAGE(EntityType.OTA_PACKAGE),
EDGE(EntityType.EDGE),
RPC(EntityType.RPC);
RPC(EntityType.RPC),
QUEUE(EntityType.QUEUE);
private final EntityType entityType;

1
application/src/main/java/org/thingsboard/server/service/security/permission/SysAdminPermissions.java

@ -39,6 +39,7 @@ public class SysAdminPermissions extends AbstractPermissions {
put(Resource.OAUTH2_CONFIGURATION_TEMPLATE, PermissionChecker.allowAllPermissionChecker);
put(Resource.TENANT_PROFILE, PermissionChecker.allowAllPermissionChecker);
put(Resource.TB_RESOURCE, systemEntityPermissionChecker);
put(Resource.QUEUE, systemEntityPermissionChecker);
}
private static final PermissionChecker systemEntityPermissionChecker = new PermissionChecker() {

17
application/src/main/java/org/thingsboard/server/service/security/permission/TenantAdminPermissions.java

@ -46,6 +46,7 @@ public class TenantAdminPermissions extends AbstractPermissions {
put(Resource.OTA_PACKAGE, tenantEntityPermissionChecker);
put(Resource.EDGE, tenantEntityPermissionChecker);
put(Resource.RPC, tenantEntityPermissionChecker);
put(Resource.QUEUE, queuePermissionChecker);
}
public static final PermissionChecker tenantEntityPermissionChecker = new PermissionChecker() {
@ -121,4 +122,20 @@ public class TenantAdminPermissions extends AbstractPermissions {
}
};
private static final PermissionChecker queuePermissionChecker = new PermissionChecker() {
@Override
public boolean hasPermission(SecurityUser user, Operation operation, EntityId entityId, HasTenantId entity) {
if (entity.getTenantId() == null || entity.getTenantId().isNullUid()) {
return operation == Operation.READ;
}
if (!user.getTenantId().equals(entity.getTenantId())) {
return false;
}
return true;
}
};
}

22
application/src/main/java/org/thingsboard/server/service/subscription/DefaultSubscriptionManagerService.java

@ -50,6 +50,7 @@ import org.thingsboard.server.gen.transport.TransportProtos.TbSubscriptionUpdate
import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMsg;
import org.thingsboard.server.queue.TbQueueProducer;
import org.thingsboard.server.queue.common.TbProtoQueueMsg;
import org.thingsboard.server.queue.discovery.NotificationsTopicService;
import org.thingsboard.server.queue.discovery.PartitionService;
import org.thingsboard.server.queue.discovery.TbApplicationEventListener;
import org.thingsboard.server.queue.discovery.TbServiceInfoProvider;
@ -88,6 +89,9 @@ public class DefaultSubscriptionManagerService extends TbApplicationEventListene
@Autowired
private TimeseriesService tsService;
@Autowired
private NotificationsTopicService notificationsTopicService;
@Autowired
private PartitionService partitionService;
@ -264,7 +268,7 @@ public class DefaultSubscriptionManagerService extends TbApplicationEventListene
updateDeviceInactivityTimeout(tenantId, entityId, attributes);
} else if (TbAttributeSubscriptionScope.SHARED_SCOPE.name().equalsIgnoreCase(scope) && notifyDevice) {
clusterService.pushMsgToCore(DeviceAttributesEventNotificationMsg.onUpdate(tenantId,
new DeviceId(entityId.getId()), DataConstants.SHARED_SCOPE, new ArrayList<>(attributes))
new DeviceId(entityId.getId()), DataConstants.SHARED_SCOPE, new ArrayList<>(attributes))
, null);
}
}
@ -377,7 +381,7 @@ public class DefaultSubscriptionManagerService extends TbApplicationEventListene
TelemetrySubscriptionUpdate update = new TelemetrySubscriptionUpdate(s.getSubscriptionId(), subscriptionUpdate);
localSubscriptionService.onSubscriptionUpdate(s.getSessionId(), update, TbCallback.EMPTY);
} else {
TopicPartitionInfo tpi = partitionService.getNotificationsTopic(ServiceType.TB_CORE, s.getServiceId());
TopicPartitionInfo tpi = notificationsTopicService.getNotificationsTopic(ServiceType.TB_CORE, s.getServiceId());
toCoreNotificationsProducer.send(tpi, toProto(s, subscriptionUpdate, ignoreEmptyUpdates), null);
}
}
@ -400,7 +404,7 @@ public class DefaultSubscriptionManagerService extends TbApplicationEventListene
AlarmSubscriptionUpdate update = new AlarmSubscriptionUpdate(s.getSubscriptionId(), alarm, deleted);
localSubscriptionService.onSubscriptionUpdate(s.getSessionId(), update, TbCallback.EMPTY);
} else {
TopicPartitionInfo tpi = partitionService.getNotificationsTopic(ServiceType.TB_CORE, s.getServiceId());
TopicPartitionInfo tpi = notificationsTopicService.getNotificationsTopic(ServiceType.TB_CORE, s.getServiceId());
toCoreNotificationsProducer.send(tpi, toProto(s, alarm, deleted), null);
}
}
@ -441,7 +445,7 @@ public class DefaultSubscriptionManagerService extends TbApplicationEventListene
}
});
if (!missedUpdates.isEmpty()) {
TopicPartitionInfo tpi = partitionService.getNotificationsTopic(ServiceType.TB_CORE, subscription.getServiceId());
TopicPartitionInfo tpi = notificationsTopicService.getNotificationsTopic(ServiceType.TB_CORE, subscription.getServiceId());
toCoreNotificationsProducer.send(tpi, toProto(subscription, missedUpdates), null);
}
},
@ -464,7 +468,7 @@ public class DefaultSubscriptionManagerService extends TbApplicationEventListene
DonAsynchron.withCallback(tsService.findLatest(subscription.getTenantId(), subscription.getEntityId(), subscription.getKeyStates().keySet()),
missedUpdates -> {
if (missedUpdates != null && !missedUpdates.isEmpty()) {
TopicPartitionInfo tpi = partitionService.getNotificationsTopic(ServiceType.TB_CORE, subscription.getServiceId());
TopicPartitionInfo tpi = notificationsTopicService.getNotificationsTopic(ServiceType.TB_CORE, subscription.getServiceId());
toCoreNotificationsProducer.send(tpi, toProto(subscription, missedUpdates), null);
}
},
@ -483,7 +487,7 @@ public class DefaultSubscriptionManagerService extends TbApplicationEventListene
DonAsynchron.withCallback(tsService.findAll(subscription.getTenantId(), subscription.getEntityId(), queries),
missedUpdates -> {
if (missedUpdates != null && !missedUpdates.isEmpty()) {
TopicPartitionInfo tpi = partitionService.getNotificationsTopic(ServiceType.TB_CORE, subscription.getServiceId());
TopicPartitionInfo tpi = notificationsTopicService.getNotificationsTopic(ServiceType.TB_CORE, subscription.getServiceId());
toCoreNotificationsProducer.send(tpi, toProto(subscription, missedUpdates), null);
}
},
@ -533,7 +537,7 @@ public class DefaultSubscriptionManagerService extends TbApplicationEventListene
});
ToCoreNotificationMsg toCoreMsg = ToCoreNotificationMsg.newBuilder().setToLocalSubscriptionServiceMsg(
LocalSubscriptionServiceMsgProto.newBuilder().setSubUpdate(builder.build()).build())
LocalSubscriptionServiceMsgProto.newBuilder().setSubUpdate(builder.build()).build())
.build();
return new TbProtoQueueMsg<>(subscription.getEntityId().getId(), toCoreMsg);
}
@ -547,8 +551,8 @@ public class DefaultSubscriptionManagerService extends TbApplicationEventListene
builder.setDeleted(deleted);
ToCoreNotificationMsg toCoreMsg = ToCoreNotificationMsg.newBuilder().setToLocalSubscriptionServiceMsg(
LocalSubscriptionServiceMsgProto.newBuilder()
.setAlarmSubUpdate(builder.build()).build())
LocalSubscriptionServiceMsgProto.newBuilder()
.setAlarmSubUpdate(builder.build()).build())
.build();
return new TbProtoQueueMsg<>(subscription.getEntityId().getId(), toCoreMsg);
}

2
application/src/main/java/org/thingsboard/server/service/subscription/DefaultTbLocalSubscriptionService.java

@ -76,7 +76,7 @@ public class DefaultTbLocalSubscriptionService implements TbLocalSubscriptionSer
private TbApplicationEventListener<ClusterTopologyChangeEvent> clusterTopologyChangeListener = new TbApplicationEventListener<>() {
@Override
protected void onTbApplicationEvent(ClusterTopologyChangeEvent event) {
if (event.getServiceQueueKeys().stream().anyMatch(key -> ServiceType.TB_CORE.equals(key.getServiceType()))) {
if (event.getQueueKeys().stream().anyMatch(key -> ServiceType.TB_CORE.equals(key.getType()))) {
/*
* If the cluster topology has changed, we need to push all current subscriptions to SubscriptionManagerService again.
* Otherwise, the SubscriptionManagerService may "forget" those subscriptions in case of restart.

13
application/src/main/java/org/thingsboard/server/service/transport/DefaultTbCoreToTransportService.java

@ -16,7 +16,6 @@
package org.thingsboard.server.service.transport;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Service;
import org.thingsboard.server.common.msg.queue.ServiceType;
import org.thingsboard.server.common.msg.queue.TopicPartitionInfo;
@ -25,7 +24,7 @@ import org.thingsboard.server.queue.TbQueueCallback;
import org.thingsboard.server.queue.TbQueueMsgMetadata;
import org.thingsboard.server.queue.TbQueueProducer;
import org.thingsboard.server.queue.common.TbProtoQueueMsg;
import org.thingsboard.server.queue.discovery.PartitionService;
import org.thingsboard.server.queue.discovery.NotificationsTopicService;
import org.thingsboard.server.queue.provider.TbQueueProducerProvider;
import org.thingsboard.server.queue.util.TbCoreComponent;
@ -39,11 +38,11 @@ import static org.thingsboard.server.dao.model.ModelConstants.NULL_UUID;
@TbCoreComponent
public class DefaultTbCoreToTransportService implements TbCoreToTransportService {
private final PartitionService partitionService;
private final NotificationsTopicService notificationsTopicService;
private final TbQueueProducer<TbProtoQueueMsg<ToTransportMsg>> tbTransportProducer;
public DefaultTbCoreToTransportService(PartitionService partitionService, TbQueueProducerProvider tbQueueProducerProvider) {
this.partitionService = partitionService;
public DefaultTbCoreToTransportService(NotificationsTopicService notificationsTopicService, TbQueueProducerProvider tbQueueProducerProvider) {
this.notificationsTopicService = notificationsTopicService;
this.tbTransportProducer = tbQueueProducerProvider.getTransportNotificationsMsgProducer();
}
@ -54,14 +53,14 @@ public class DefaultTbCoreToTransportService implements TbCoreToTransportService
@Override
public void process(String nodeId, ToTransportMsg msg, Runnable onSuccess, Consumer<Throwable> onFailure) {
if (nodeId == null || nodeId.isEmpty()){
if (nodeId == null || nodeId.isEmpty()) {
log.trace("process: skipping message without nodeId [{}], (ToTransportMsg) msg [{}]", nodeId, msg);
if (onSuccess != null) {
onSuccess.run();
}
return;
}
TopicPartitionInfo tpi = partitionService.getNotificationsTopic(ServiceType.TB_TRANSPORT, nodeId);
TopicPartitionInfo tpi = notificationsTopicService.getNotificationsTopic(ServiceType.TB_TRANSPORT, nodeId);
UUID sessionId = new UUID(msg.getSessionIdMSB(), msg.getSessionIdLSB());
log.trace("[{}][{}] Pushing session data to topic: {}", tpi.getFullTopicName(), sessionId, msg);
TbProtoQueueMsg<ToTransportMsg> queueMsg = new TbProtoQueueMsg<>(NULL_UUID, msg);

25
application/src/main/java/org/thingsboard/server/service/transport/DefaultTransportApiService.java

@ -57,6 +57,7 @@ import org.thingsboard.server.common.data.ota.OtaPackageType;
import org.thingsboard.server.common.data.ota.OtaPackageUtil;
import org.thingsboard.server.common.data.page.PageData;
import org.thingsboard.server.common.data.page.PageLink;
import org.thingsboard.server.common.data.queue.Queue;
import org.thingsboard.server.common.data.relation.EntityRelation;
import org.thingsboard.server.common.data.security.DeviceCredentials;
import org.thingsboard.server.common.data.security.DeviceCredentialsType;
@ -72,6 +73,7 @@ import org.thingsboard.server.dao.device.provision.ProvisionFailedException;
import org.thingsboard.server.dao.device.provision.ProvisionRequest;
import org.thingsboard.server.dao.device.provision.ProvisionResponse;
import org.thingsboard.server.dao.ota.OtaPackageService;
import org.thingsboard.server.dao.queue.QueueService;
import org.thingsboard.server.dao.relation.RelationService;
import org.thingsboard.server.dao.tenant.TbTenantProfileCache;
import org.thingsboard.server.gen.transport.TransportProtos;
@ -99,6 +101,7 @@ import org.thingsboard.server.service.executors.DbCallbackExecutorService;
import org.thingsboard.server.service.profile.TbDeviceProfileCache;
import org.thingsboard.server.service.resource.TbResourceService;
import java.util.List;
import java.util.Optional;
import java.util.UUID;
import java.util.concurrent.ConcurrentHashMap;
@ -134,6 +137,7 @@ public class DefaultTransportApiService implements TransportApiService {
private final TbResourceService resourceService;
private final OtaPackageService otaPackageService;
private final OtaPackageDataCache otaPackageDataCache;
private final QueueService queueService;
private final ConcurrentMap<String, ReentrantLock> deviceCreationLocks = new ConcurrentHashMap<>();
@ -176,6 +180,8 @@ public class DefaultTransportApiService implements TransportApiService {
result = handle(transportApiRequestMsg.getDeviceCredentialsRequestMsg());
} else if (transportApiRequestMsg.hasOtaPackageRequestMsg()) {
result = handle(transportApiRequestMsg.getOtaPackageRequestMsg());
} else if (transportApiRequestMsg.hasGetAllQueueRoutingInfoRequestMsg()) {
return Futures.transform(handle(transportApiRequestMsg.getGetAllQueueRoutingInfoRequestMsg()), value -> new TbProtoQueueMsg<>(tbProtoQueueMsg.getKey(), value, tbProtoQueueMsg.getHeaders()), MoreExecutors.directExecutor());
}
return Futures.transform(Optional.ofNullable(result).orElseGet(this::getEmptyTransportApiResponseFuture),
@ -636,6 +642,25 @@ public class DefaultTransportApiService implements TransportApiService {
}
}
private ListenableFuture<TransportApiResponseMsg> handle(TransportProtos.GetAllQueueRoutingInfoRequestMsg requestMsg) {
return queuesToTransportApiResponseMsg(queueService.findAllQueues());
}
private ListenableFuture<TransportApiResponseMsg> queuesToTransportApiResponseMsg(List<Queue> queues) {
return Futures.immediateFuture(TransportApiResponseMsg.newBuilder()
.addAllGetQueueRoutingInfoResponseMsgs(queues.stream()
.map(queue -> TransportProtos.GetQueueRoutingInfoResponseMsg.newBuilder()
.setTenantIdMSB(queue.getTenantId().getId().getMostSignificantBits())
.setTenantIdLSB(queue.getTenantId().getId().getLeastSignificantBits())
.setQueueIdMSB(queue.getId().getId().getMostSignificantBits())
.setQueueIdLSB(queue.getId().getId().getLeastSignificantBits())
.setQueueName(queue.getName())
.setQueueTopic(queue.getTopic())
.setPartitions(queue.getPartitions())
.build()).collect(Collectors.toList())).build());
}
private Long checkLong(Long l) {
return l != null ? l : 0;
}

1
application/src/main/resources/thingsboard.yml

@ -1106,7 +1106,6 @@ service:
type: "${TB_SERVICE_TYPE:monolith}" # monolith or tb-core or tb-rule-engine
# Unique id for this service (autogenerated if empty)
id: "${TB_SERVICE_ID:}"
tenant_id: "${TB_SERVICE_TENANT_ID:}" # empty or specific tenant id.
metrics:
# Enable/disable actuator metrics.

5
application/src/test/java/org/thingsboard/server/controller/AbstractWebTest.java

@ -74,6 +74,11 @@ import org.thingsboard.server.common.data.id.UUIDBased;
import org.thingsboard.server.common.data.page.PageData;
import org.thingsboard.server.common.data.page.PageLink;
import org.thingsboard.server.common.data.page.TimePageLink;
import org.thingsboard.server.common.data.queue.ProcessingStrategy;
import org.thingsboard.server.common.data.queue.ProcessingStrategyType;
import org.thingsboard.server.common.data.queue.Queue;
import org.thingsboard.server.common.data.queue.SubmitStrategy;
import org.thingsboard.server.common.data.queue.SubmitStrategyType;
import org.thingsboard.server.common.data.security.Authority;
import org.thingsboard.server.config.ThingsboardSecurityConfiguration;
import org.thingsboard.server.dao.tenant.TenantProfileService;

33
application/src/test/java/org/thingsboard/server/controller/BaseTenantProfileControllerTest.java

@ -24,11 +24,16 @@ import org.springframework.beans.factory.annotation.Autowired;
import org.thingsboard.server.common.data.EntityInfo;
import org.thingsboard.server.common.data.Tenant;
import org.thingsboard.server.common.data.TenantProfile;
import org.thingsboard.server.common.data.queue.ProcessingStrategy;
import org.thingsboard.server.common.data.queue.ProcessingStrategyType;
import org.thingsboard.server.common.data.queue.SubmitStrategy;
import org.thingsboard.server.common.data.queue.SubmitStrategyType;
import org.thingsboard.server.common.data.tenant.profile.DefaultTenantProfileConfiguration;
import org.thingsboard.server.common.data.tenant.profile.TenantProfileData;
import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.data.page.PageData;
import org.thingsboard.server.common.data.page.PageLink;
import org.thingsboard.server.common.data.tenant.profile.TenantProfileQueueConfiguration;
import org.thingsboard.server.dao.tenant.TenantProfileService;
import java.util.ArrayList;
@ -44,9 +49,6 @@ public abstract class BaseTenantProfileControllerTest extends AbstractController
private IdComparator<TenantProfile> idComparator = new IdComparator<>();
private IdComparator<EntityInfo> tenantProfileInfoIdComparator = new IdComparator<>();
@Autowired
private TenantProfileService tenantProfileService;
@Test
public void testSaveTenantProfile() throws Exception {
loginSysAdmin();
@ -141,6 +143,7 @@ public abstract class BaseTenantProfileControllerTest extends AbstractController
TenantProfile tenantProfile = this.createTenantProfile("Tenant Profile");
TenantProfile savedTenantProfile = doPost("/api/tenantProfile", tenantProfile, TenantProfile.class);
savedTenantProfile.setIsolatedTbRuleEngine(true);
addMainQueueConfig(savedTenantProfile);
doPost("/api/tenantProfile", savedTenantProfile).andExpect(status().isBadRequest())
.andExpect(statusReason(containsString("Can't update isolatedTbRuleEngine property")));
}
@ -295,4 +298,28 @@ public abstract class BaseTenantProfileControllerTest extends AbstractController
tenantProfile.setIsolatedTbRuleEngine(false);
return tenantProfile;
}
private void addMainQueueConfig(TenantProfile tenantProfile) {
TenantProfileQueueConfiguration mainQueueConfiguration = new TenantProfileQueueConfiguration();
mainQueueConfiguration.setName("Main");
mainQueueConfiguration.setTopic("tb_rule_engine.main");
mainQueueConfiguration.setPollInterval(25);
mainQueueConfiguration.setPartitions(10);
mainQueueConfiguration.setConsumerPerPartition(true);
mainQueueConfiguration.setPackProcessingTimeout(2000);
SubmitStrategy mainQueueSubmitStrategy = new SubmitStrategy();
mainQueueSubmitStrategy.setType(SubmitStrategyType.BURST);
mainQueueSubmitStrategy.setBatchSize(1000);
mainQueueConfiguration.setSubmitStrategy(mainQueueSubmitStrategy);
ProcessingStrategy mainQueueProcessingStrategy = new ProcessingStrategy();
mainQueueProcessingStrategy.setType(ProcessingStrategyType.SKIP_ALL_FAILURES);
mainQueueProcessingStrategy.setRetries(3);
mainQueueProcessingStrategy.setFailurePercentage(0);
mainQueueProcessingStrategy.setPauseBetweenRetries(3);
mainQueueProcessingStrategy.setMaxPauseBetweenRetries(3);
mainQueueConfiguration.setProcessingStrategy(mainQueueProcessingStrategy);
TenantProfileData profileData = tenantProfile.getProfileData();
profileData.setQueueConfiguration(Collections.singletonList(mainQueueConfiguration));
tenantProfile.setProfileData(profileData);
}
}

25
application/src/test/java/org/thingsboard/server/service/cluster/routing/HashPartitionServiceTest.java

@ -21,20 +21,18 @@ import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.mockito.Mockito;
import org.mockito.junit.MockitoJUnitRunner;
import org.springframework.context.ApplicationEventPublisher;
import org.springframework.test.util.ReflectionTestUtils;
import org.thingsboard.server.common.data.id.DeviceId;
import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.queue.QueueService;
import org.thingsboard.server.queue.discovery.HashPartitionService;
import org.thingsboard.server.common.msg.queue.ServiceType;
import org.thingsboard.server.queue.discovery.TbServiceInfoProvider;
import org.thingsboard.server.common.msg.queue.TopicPartitionInfo;
import org.thingsboard.server.gen.transport.TransportProtos;
import org.thingsboard.server.queue.discovery.HashPartitionService;
import org.thingsboard.server.queue.discovery.QueueRoutingInfoService;
import org.thingsboard.server.queue.discovery.TbServiceInfoProvider;
import org.thingsboard.server.queue.discovery.TenantRoutingInfoService;
import org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings;
import java.util.ArrayList;
import java.util.Collections;
@ -45,7 +43,6 @@ import java.util.Map;
import java.util.stream.Collectors;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
@Slf4j
@RunWith(MockitoJUnitRunner.class)
@ -58,33 +55,25 @@ public class HashPartitionServiceTest {
private TbServiceInfoProvider discoveryService;
private TenantRoutingInfoService routingInfoService;
private ApplicationEventPublisher applicationEventPublisher;
private TbQueueRuleEngineSettings ruleEngineSettings;
private QueueService queueService;
private QueueRoutingInfoService queueRoutingInfoService;
private String hashFunctionName = "sha256";
@Before
public void setup() throws Exception {
discoveryService = mock(TbServiceInfoProvider.class);
applicationEventPublisher = mock(ApplicationEventPublisher.class);
routingInfoService = mock(TenantRoutingInfoService.class);
ruleEngineSettings = mock(TbQueueRuleEngineSettings.class);
queueService = mock(QueueService.class);
queueRoutingInfoService = mock(QueueRoutingInfoService.class);
clusterRoutingService = new HashPartitionService(discoveryService,
routingInfoService,
applicationEventPublisher,
ruleEngineSettings,
queueService
);
when(ruleEngineSettings.getQueues()).thenReturn(Collections.emptyList());
queueRoutingInfoService);
ReflectionTestUtils.setField(clusterRoutingService, "coreTopic", "tb.core");
ReflectionTestUtils.setField(clusterRoutingService, "corePartitions", 10);
ReflectionTestUtils.setField(clusterRoutingService, "hashFunctionName", hashFunctionName);
TransportProtos.ServiceInfo currentServer = TransportProtos.ServiceInfo.newBuilder()
.setServiceId("tb-core-0")
.setTenantIdMSB(TenantId.NULL_UUID.getMostSignificantBits())
.setTenantIdLSB(TenantId.NULL_UUID.getLeastSignificantBits())
.addAllServiceTypes(Collections.singletonList(ServiceType.TB_CORE.name()))
.build();
// when(queueService.resolve(Mockito.any(), Mockito.anyString())).thenAnswer(i -> i.getArguments()[1]);
@ -93,8 +82,6 @@ public class HashPartitionServiceTest {
for (int i = 1; i < SERVER_COUNT; i++) {
otherServers.add(TransportProtos.ServiceInfo.newBuilder()
.setServiceId("tb-rule-" + i)
.setTenantIdMSB(TenantId.NULL_UUID.getMostSignificantBits())
.setTenantIdLSB(TenantId.NULL_UUID.getLeastSignificantBits())
.addAllServiceTypes(Collections.singletonList(ServiceType.TB_CORE.name()))
.build());
}

10
common/cluster-api/src/main/java/org/thingsboard/server/cluster/TbClusterService.java

@ -15,31 +15,31 @@
*/
package org.thingsboard.server.cluster;
import org.thingsboard.server.common.data.edge.EdgeEventActionType;
import org.thingsboard.server.common.data.edge.EdgeEventType;
import org.thingsboard.server.common.msg.ToDeviceActorNotificationMsg;
import org.thingsboard.server.common.data.ApiUsageState;
import org.thingsboard.server.common.data.Device;
import org.thingsboard.server.common.data.DeviceProfile;
import org.thingsboard.server.common.data.TbResource;
import org.thingsboard.server.common.data.Tenant;
import org.thingsboard.server.common.data.TenantProfile;
import org.thingsboard.server.common.data.id.DeviceId;
import org.thingsboard.server.common.data.edge.EdgeEventActionType;
import org.thingsboard.server.common.data.edge.EdgeEventType;
import org.thingsboard.server.common.data.id.EdgeId;
import org.thingsboard.server.common.data.id.EntityId;
import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.data.plugin.ComponentLifecycleEvent;
import org.thingsboard.server.common.msg.TbMsg;
import org.thingsboard.server.common.msg.ToDeviceActorNotificationMsg;
import org.thingsboard.server.common.msg.queue.TopicPartitionInfo;
import org.thingsboard.server.common.msg.rpc.FromDeviceRpcResponse;
import org.thingsboard.server.gen.transport.TransportProtos;
import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg;
import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg;
import org.thingsboard.server.queue.TbQueueCallback;
import org.thingsboard.server.queue.TbQueueClusterService;
import java.util.UUID;
public interface TbClusterService {
public interface TbClusterService extends TbQueueClusterService {
void pushMsgToCore(TopicPartitionInfo tpi, UUID msgKey, ToCoreMsg msg, TbQueueCallback callback);

2
common/cluster-api/src/main/java/org/thingsboard/server/queue/TbQueueAdmin.java

@ -20,4 +20,6 @@ public interface TbQueueAdmin {
void createTopicIfNotExists(String topic);
void destroy();
void deleteTopic(String topic);
}

12
common/cluster-api/src/main/java/org/thingsboard/server/queue/QueueService.java → common/cluster-api/src/main/java/org/thingsboard/server/queue/TbQueueClusterService.java

@ -15,14 +15,10 @@
*/
package org.thingsboard.server.queue;
import org.thingsboard.server.common.msg.queue.ServiceType;
import org.thingsboard.server.common.data.queue.Queue;
import java.util.Set;
public interface QueueService {
Set<String> getQueuesByServiceType(ServiceType serviceType);
String resolve(ServiceType serviceType, String queueName);
public interface TbQueueClusterService {
void onQueueChange(Queue queue);
void onQueueDelete(Queue queue);
}

48
common/cluster-api/src/main/proto/queue.proto

@ -20,21 +20,12 @@ package transport;
option java_package = "org.thingsboard.server.gen.transport";
option java_outer_classname = "TransportProtos";
message QueueInfo {
string name = 1;
string topic = 2;
int32 partitions = 3;
}
/**
* Service Discovery Data Structures;
*/
message ServiceInfo {
string serviceId = 1;
repeated string serviceTypes = 2;
int64 tenantIdMSB = 3;
int64 tenantIdLSB = 4;
repeated QueueInfo ruleEngineQueues = 5;
repeated string transports = 6;
}
@ -197,6 +188,37 @@ message GetEntityProfileRequestMsg {
int64 entityIdLSB = 3;
}
message GetAllQueueRoutingInfoRequestMsg {
}
message GetQueueRoutingInfoResponseMsg {
int64 tenantIdMSB = 1;
int64 tenantIdLSB = 2;
int64 queueIdMSB = 3;
int64 queueIdLSB = 4;
string queueName = 5;
string queueTopic = 6;
int32 partitions = 7;
}
message QueueUpdateMsg {
int64 tenantIdMSB = 1;
int64 tenantIdLSB = 2;
int64 queueIdMSB = 3;
int64 queueIdLSB = 4;
string queueName = 5;
string queueTopic = 6;
int32 partitions = 7;
}
message QueueDeleteMsg {
int64 tenantIdMSB = 1;
int64 tenantIdLSB = 2;
int64 queueIdMSB = 3;
int64 queueIdLSB = 4;
string queueName = 5;
}
message LwM2MRegistrationRequestMsg {
string tenantId = 1;
string endpoint = 2;
@ -673,6 +695,7 @@ message TransportApiRequestMsg {
GetSnmpDevicesRequestMsg snmpDevicesRequestMsg = 11;
GetDeviceRequestMsg deviceRequestMsg = 12;
GetDeviceCredentialsRequestMsg deviceCredentialsRequestMsg = 13;
GetAllQueueRoutingInfoRequestMsg getAllQueueRoutingInfoRequestMsg = 14;
}
/* Response from ThingsBoard Core Service to Transport Service */
@ -687,6 +710,7 @@ message TransportApiResponseMsg {
GetOtaPackageResponseMsg otaPackageResponseMsg = 8;
GetDeviceResponseMsg deviceResponseMsg = 9;
GetDeviceCredentialsResponseMsg deviceCredentialsResponseMsg = 10;
repeated GetQueueRoutingInfoResponseMsg getQueueRoutingInfoResponseMsgs = 11;
}
/* Messages that are handled by ThingsBoard Core Service */
@ -704,6 +728,8 @@ message ToCoreNotificationMsg {
FromDeviceRPCResponseProto fromDeviceRpcResponse = 2;
bytes componentLifecycleMsg = 3;
bytes edgeEventUpdateMsg = 4;
QueueUpdateMsg queueUpdateMsg = 5;
QueueDeleteMsg queueDeleteMsg = 6;
}
/* Messages that are handled by ThingsBoard RuleEngine Service */
@ -718,6 +744,8 @@ message ToRuleEngineMsg {
message ToRuleEngineNotificationMsg {
bytes componentLifecycleMsg = 1;
FromDeviceRPCResponseProto fromDeviceRpcResponse = 2;
QueueUpdateMsg queueUpdateMsg = 3;
QueueDeleteMsg queueDeleteMsg = 4;
}
/* Messages that are handled by ThingsBoard Transport Service */
@ -736,6 +764,8 @@ message ToTransportMsg {
ResourceUpdateMsg resourceUpdateMsg = 12;
ResourceDeleteMsg resourceDeleteMsg = 13;
UplinkNotificationMsg uplinkNotificationMsg = 14;
QueueUpdateMsg queueUpdateMsg = 15;
QueueDeleteMsg queueDeleteMsg = 16;
}
message UsageStatsKVProto{

45
common/dao-api/src/main/java/org/thingsboard/server/dao/queue/QueueService.java

@ -0,0 +1,45 @@
/**
* Copyright © 2016-2022 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.dao.queue;
import org.thingsboard.server.common.data.id.QueueId;
import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.data.page.PageData;
import org.thingsboard.server.common.data.page.PageLink;
import org.thingsboard.server.common.data.queue.Queue;
import java.util.List;
public interface QueueService {
Queue saveQueue(Queue queue);
void deleteQueue(TenantId tenantId, QueueId queueId);
List<Queue> findQueuesByTenantId(TenantId tenantId);
PageData<Queue> findQueuesByTenantId(TenantId tenantId, PageLink pageLink);
List<Queue> findAllQueues();
Queue findQueueById(TenantId tenantId, QueueId queueId);
Queue findQueueByTenantIdAndName(TenantId tenantId, String name);
Queue findQueueByTenantIdAndNameInternal(TenantId tenantId, String queueName);
void deleteQueuesByTenantId(TenantId tenantId);
}

2
common/dao-api/src/main/java/org/thingsboard/server/dao/rule/RuleChainService.java

@ -94,6 +94,8 @@ public interface RuleChainService {
List<RuleNode> findRuleNodesByTenantIdAndType(TenantId tenantId, String name, String toString);
List<RuleNode> findRuleNodesByTenantIdAndType(TenantId tenantId, String type);
RuleNode saveRuleNode(TenantId tenantId, RuleNode ruleNode);
void deleteRuleNodes(TenantId tenantId, RuleChainId ruleChainId);

13
common/dao-api/src/main/java/org/thingsboard/server/dao/tenant/TenantService.java

@ -19,9 +19,12 @@ import com.google.common.util.concurrent.ListenableFuture;
import org.thingsboard.server.common.data.Tenant;
import org.thingsboard.server.common.data.TenantInfo;
import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.data.id.TenantProfileId;
import org.thingsboard.server.common.data.page.PageData;
import org.thingsboard.server.common.data.page.PageLink;
import java.util.List;
public interface TenantService {
Tenant findTenantById(TenantId tenantId);
@ -29,14 +32,16 @@ public interface TenantService {
TenantInfo findTenantInfoById(TenantId tenantId);
ListenableFuture<Tenant> findTenantByIdAsync(TenantId callerId, TenantId tenantId);
Tenant saveTenant(Tenant tenant);
void deleteTenant(TenantId tenantId);
PageData<Tenant> findTenants(PageLink pageLink);
PageData<TenantInfo> findTenantInfos(PageLink pageLink);
List<TenantId> findTenantIdsByTenantProfileId(TenantProfileId tenantProfileId);
void deleteTenants();
}

5
common/data/src/main/java/org/thingsboard/server/common/data/DeviceProfile.java

@ -27,6 +27,7 @@ import org.thingsboard.server.common.data.device.profile.DeviceProfileData;
import org.thingsboard.server.common.data.id.DashboardId;
import org.thingsboard.server.common.data.id.DeviceProfileId;
import org.thingsboard.server.common.data.id.OtaPackageId;
import org.thingsboard.server.common.data.id.QueueId;
import org.thingsboard.server.common.data.id.RuleChainId;
import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.data.validation.Length;
@ -76,7 +77,7 @@ public class DeviceProfile extends SearchTextBased<DeviceProfileId> implements H
@ApiModelProperty(position = 8, value = "Reference to the rule engine queue. " +
"If present, the specified queue will be used to store all unprocessed messages related to device, including telemetry, attribute updates, etc. " +
"Otherwise, the 'Main' queue will be used to store those messages.")
private String defaultQueueName;
private QueueId defaultQueueId;
@Valid
private transient DeviceProfileData profileData;
@JsonIgnore
@ -109,7 +110,7 @@ public class DeviceProfile extends SearchTextBased<DeviceProfileId> implements H
this.isDefault = deviceProfile.isDefault();
this.defaultRuleChainId = deviceProfile.getDefaultRuleChainId();
this.defaultDashboardId = deviceProfile.getDefaultDashboardId();
this.defaultQueueName = deviceProfile.getDefaultQueueName();
this.defaultQueueId = deviceProfile.getDefaultQueueId();
this.setProfileData(deviceProfile.getProfileData());
this.provisionDeviceKey = deviceProfile.getProvisionDeviceKey();
this.firmwareId = deviceProfile.getFirmwareId();

2
common/data/src/main/java/org/thingsboard/server/common/data/EntityType.java

@ -19,5 +19,5 @@ package org.thingsboard.server.common.data;
* @author Andrew Shvayka
*/
public enum EntityType {
TENANT, CUSTOMER, USER, DASHBOARD, ASSET, DEVICE, ALARM, RULE_CHAIN, RULE_NODE, ENTITY_VIEW, WIDGETS_BUNDLE, WIDGET_TYPE, TENANT_PROFILE, DEVICE_PROFILE, API_USAGE_STATE, TB_RESOURCE, OTA_PACKAGE, EDGE, RPC;
TENANT, CUSTOMER, USER, DASHBOARD, ASSET, DEVICE, ALARM, RULE_CHAIN, RULE_NODE, ENTITY_VIEW, WIDGETS_BUNDLE, WIDGET_TYPE, TENANT_PROFILE, DEVICE_PROFILE, API_USAGE_STATE, TB_RESOURCE, OTA_PACKAGE, EDGE, RPC, QUEUE;
}

2
common/data/src/main/java/org/thingsboard/server/common/data/TenantProfile.java

@ -57,7 +57,7 @@ public class TenantProfile extends SearchTextBased<TenantProfileId> implements H
@ApiModelProperty(position = 7, value = "If enabled, will push all messages related to this tenant and processed by the rule engine into separate queue. " +
"Useful for complex microservices deployments, to isolate processing of the data for specific tenants", example = "true")
private boolean isolatedTbRuleEngine;
@ApiModelProperty(position = 8, value = "Complex JSON object that contains profile settings: max devices, max assets, rate limits, etc.")
@ApiModelProperty(position = 8, value = "Complex JSON object that contains profile settings: queue configs, max devices, max assets, rate limits, etc.")
private transient TenantProfileData profileData;
@JsonIgnore
private byte[] profileDataBytes;

2
common/data/src/main/java/org/thingsboard/server/common/data/id/EntityIdFactory.java

@ -77,6 +77,8 @@ public class EntityIdFactory {
return new EdgeId(uuid);
case RPC:
return new RpcId(uuid);
case QUEUE:
return new QueueId(uuid);
}
throw new IllegalArgumentException("EntityType " + type + " is not supported!");
}

41
common/data/src/main/java/org/thingsboard/server/common/data/id/QueueId.java

@ -0,0 +1,41 @@
/**
* Copyright © 2016-2022 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.common.data.id;
import com.fasterxml.jackson.annotation.JsonCreator;
import com.fasterxml.jackson.annotation.JsonProperty;
import org.thingsboard.server.common.data.EntityType;
import java.util.UUID;
public class QueueId extends UUIDBased implements EntityId {
private static final long serialVersionUID = 1L;
@JsonCreator
public QueueId(@JsonProperty("id") UUID id) {
super(id);
}
public static QueueId fromString(String queueId) {
return new QueueId(UUID.fromString(queueId));
}
@Override
public EntityType getEntityType() {
return EntityType.QUEUE;
}
}

27
common/data/src/main/java/org/thingsboard/server/common/data/queue/ProcessingStrategy.java

@ -0,0 +1,27 @@
/**
* Copyright © 2016-2022 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.common.data.queue;
import lombok.Data;
@Data
public class ProcessingStrategy {
private ProcessingStrategyType type;
private int retries;
private double failurePercentage;
private long pauseBetweenRetries;
private long maxPauseBetweenRetries;
}

36
common/data/src/main/java/org/thingsboard/server/common/data/queue/ProcessingStrategyType.java

@ -0,0 +1,36 @@
/**
* Copyright © 2016-2022 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
/**
* Copyright © 2016-2020 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.common.data.queue;
public enum ProcessingStrategyType {
SKIP_ALL_FAILURES, SKIP_ALL_FAILURES_AND_TIMED_OUT, RETRY_ALL, RETRY_FAILED, RETRY_TIMED_OUT, RETRY_FAILED_AND_TIMED_OUT
}

61
common/data/src/main/java/org/thingsboard/server/common/data/queue/Queue.java

@ -0,0 +1,61 @@
/**
* Copyright © 2016-2022 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.common.data.queue;
import lombok.Data;
import org.thingsboard.server.common.data.HasName;
import org.thingsboard.server.common.data.HasTenantId;
import org.thingsboard.server.common.data.SearchTextBasedWithAdditionalInfo;
import org.thingsboard.server.common.data.id.QueueId;
import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.data.tenant.profile.TenantProfileQueueConfiguration;
@Data
public class Queue extends SearchTextBasedWithAdditionalInfo<QueueId> implements HasName, HasTenantId {
private TenantId tenantId;
private String name;
private String topic;
private int pollInterval;
private int partitions;
private boolean consumerPerPartition;
private long packProcessingTimeout;
private SubmitStrategy submitStrategy;
private ProcessingStrategy processingStrategy;
public Queue() {
}
public Queue(QueueId id) {
super(id);
}
public Queue(TenantId tenantId, TenantProfileQueueConfiguration queueConfiguration) {
this.tenantId = tenantId;
this.name = queueConfiguration.getName();
this.topic = queueConfiguration.getTopic();
this.pollInterval = queueConfiguration.getPollInterval();
this.partitions = queueConfiguration.getPartitions();
this.consumerPerPartition = queueConfiguration.isConsumerPerPartition();
this.packProcessingTimeout = queueConfiguration.getPackProcessingTimeout();
this.submitStrategy = queueConfiguration.getSubmitStrategy();
this.processingStrategy = queueConfiguration.getProcessingStrategy();
}
@Override
public String getSearchText() {
return getName();
}
}

24
common/data/src/main/java/org/thingsboard/server/common/data/queue/SubmitStrategy.java

@ -0,0 +1,24 @@
/**
* Copyright © 2016-2022 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.common.data.queue;
import lombok.Data;
@Data
public class SubmitStrategy {
private SubmitStrategyType type;
private int batchSize;
}

20
common/data/src/main/java/org/thingsboard/server/common/data/queue/SubmitStrategyType.java

@ -0,0 +1,20 @@
/**
* Copyright © 2016-2022 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.common.data.queue;
public enum SubmitStrategyType {
BURST, BATCH, SEQUENTIAL_BY_ORIGINATOR, SEQUENTIAL_BY_TENANT, SEQUENTIAL
}

5
common/data/src/main/java/org/thingsboard/server/common/data/tenant/profile/TenantProfileData.java

@ -19,6 +19,8 @@ import io.swagger.annotations.ApiModel;
import io.swagger.annotations.ApiModelProperty;
import lombok.Data;
import java.util.List;
@ApiModel
@Data
public class TenantProfileData {
@ -26,4 +28,7 @@ public class TenantProfileData {
@ApiModelProperty(position = 1, value = "Complex JSON object that contains profile settings: max devices, max assets, rate limits, etc.")
private TenantProfileConfiguration configuration;
@ApiModelProperty(position = 2, value = "JSON array of queue configuration per tenant profile")
private List<TenantProfileQueueConfiguration> queueConfiguration;
}

32
common/data/src/main/java/org/thingsboard/server/common/data/tenant/profile/TenantProfileQueueConfiguration.java

@ -0,0 +1,32 @@
/**
* Copyright © 2016-2022 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.common.data.tenant.profile;
import lombok.Data;
import org.thingsboard.server.common.data.queue.ProcessingStrategy;
import org.thingsboard.server.common.data.queue.SubmitStrategy;
@Data
public class TenantProfileQueueConfiguration {
private String name;
private String topic;
private int pollInterval;
private int partitions;
private boolean consumerPerPartition;
private long packProcessingTimeout;
private SubmitStrategy submitStrategy;
private ProcessingStrategy processingStrategy;
}

63
common/message/src/main/java/org/thingsboard/server/common/msg/TbMsg.java

@ -27,15 +27,14 @@ import org.thingsboard.server.common.data.EntityType;
import org.thingsboard.server.common.data.id.CustomerId;
import org.thingsboard.server.common.data.id.EntityId;
import org.thingsboard.server.common.data.id.EntityIdFactory;
import org.thingsboard.server.common.data.id.QueueId;
import org.thingsboard.server.common.data.id.RuleChainId;
import org.thingsboard.server.common.data.id.RuleNodeId;
import org.thingsboard.server.common.msg.gen.MsgProtos;
import org.thingsboard.server.common.msg.queue.ServiceQueue;
import org.thingsboard.server.common.msg.queue.TbMsgCallback;
import java.io.Serializable;
import java.util.UUID;
import java.util.concurrent.atomic.AtomicInteger;
/**
* Created by ashvayka on 13.01.18.
@ -44,7 +43,7 @@ import java.util.concurrent.atomic.AtomicInteger;
@Slf4j
public final class TbMsg implements Serializable {
private final String queueName;
private final QueueId queueId;
private final UUID id;
private final long ts;
private final String type;
@ -68,12 +67,12 @@ public final class TbMsg implements Serializable {
return ctx.getAndIncrementRuleNodeCounter();
}
public static TbMsg newMsg(String queueName, String type, EntityId originator, TbMsgMetaData metaData, String data, RuleChainId ruleChainId, RuleNodeId ruleNodeId) {
return newMsg(queueName, type, originator, null, metaData, data, ruleChainId, ruleNodeId);
public static TbMsg newMsg(QueueId queueId, String type, EntityId originator, TbMsgMetaData metaData, String data, RuleChainId ruleChainId, RuleNodeId ruleNodeId) {
return newMsg(queueId, type, originator, null, metaData, data, ruleChainId, ruleNodeId);
}
public static TbMsg newMsg(String queueName, String type, EntityId originator, CustomerId customerId, TbMsgMetaData metaData, String data, RuleChainId ruleChainId, RuleNodeId ruleNodeId) {
return new TbMsg(queueName, UUID.randomUUID(), System.currentTimeMillis(), type, originator, customerId,
public static TbMsg newMsg(QueueId queueId, String type, EntityId originator, CustomerId customerId, TbMsgMetaData metaData, String data, RuleChainId ruleChainId, RuleNodeId ruleNodeId) {
return new TbMsg(queueId, UUID.randomUUID(), System.currentTimeMillis(), type, originator, customerId,
metaData.copy(), TbMsgDataType.JSON, data, ruleChainId, ruleNodeId, null, TbMsgCallback.EMPTY);
}
@ -82,23 +81,23 @@ public final class TbMsg implements Serializable {
}
public static TbMsg newMsg(String type, EntityId originator, CustomerId customerId, TbMsgMetaData metaData, String data) {
return new TbMsg(ServiceQueue.MAIN, UUID.randomUUID(), System.currentTimeMillis(), type, originator, customerId,
return new TbMsg(null, UUID.randomUUID(), System.currentTimeMillis(), type, originator, customerId,
metaData.copy(), TbMsgDataType.JSON, data, null, null, null, TbMsgCallback.EMPTY);
}
// REALLY NEW MSG
public static TbMsg newMsg(String queueName, String type, EntityId originator, TbMsgMetaData metaData, String data) {
return newMsg(queueName, type, originator, null, metaData, data);
public static TbMsg newMsg(QueueId queueId, String type, EntityId originator, TbMsgMetaData metaData, String data) {
return newMsg(queueId, type, originator, null, metaData, data);
}
public static TbMsg newMsg(String queueName, String type, EntityId originator, CustomerId customerId, TbMsgMetaData metaData, String data) {
return new TbMsg(queueName, UUID.randomUUID(), System.currentTimeMillis(), type, originator, customerId,
public static TbMsg newMsg(QueueId queueId, String type, EntityId originator, CustomerId customerId, TbMsgMetaData metaData, String data) {
return new TbMsg(queueId, UUID.randomUUID(), System.currentTimeMillis(), type, originator, customerId,
metaData.copy(), TbMsgDataType.JSON, data, null, null, null, TbMsgCallback.EMPTY);
}
public static TbMsg newMsg(String type, EntityId originator, CustomerId customerId, TbMsgMetaData metaData, TbMsgDataType dataType, String data) {
return new TbMsg(ServiceQueue.MAIN, UUID.randomUUID(), System.currentTimeMillis(), type, originator, customerId,
return new TbMsg(null, UUID.randomUUID(), System.currentTimeMillis(), type, originator, customerId,
metaData.copy(), dataType, data, null, null, null, TbMsgCallback.EMPTY);
}
@ -109,50 +108,50 @@ public final class TbMsg implements Serializable {
// For Tests only
public static TbMsg newMsg(String type, EntityId originator, TbMsgMetaData metaData, TbMsgDataType dataType, String data, RuleChainId ruleChainId, RuleNodeId ruleNodeId) {
return new TbMsg(ServiceQueue.MAIN, UUID.randomUUID(), System.currentTimeMillis(), type, originator, null,
return new TbMsg(null, UUID.randomUUID(), System.currentTimeMillis(), type, originator, null,
metaData.copy(), dataType, data, ruleChainId, ruleNodeId, null, TbMsgCallback.EMPTY);
}
public static TbMsg newMsg(String type, EntityId originator, TbMsgMetaData metaData, String data, TbMsgCallback callback) {
return new TbMsg(ServiceQueue.MAIN, UUID.randomUUID(), System.currentTimeMillis(), type, originator, null,
return new TbMsg(null, UUID.randomUUID(), System.currentTimeMillis(), type, originator, null,
metaData.copy(), TbMsgDataType.JSON, data, null, null, null, callback);
}
public static TbMsg transformMsg(TbMsg tbMsg, String type, EntityId originator, TbMsgMetaData metaData, String data) {
return new TbMsg(tbMsg.queueName, tbMsg.id, tbMsg.ts, type, originator, tbMsg.customerId, metaData.copy(), tbMsg.dataType,
return new TbMsg(tbMsg.queueId, tbMsg.id, tbMsg.ts, type, originator, tbMsg.customerId, metaData.copy(), tbMsg.dataType,
data, tbMsg.ruleChainId, tbMsg.ruleNodeId, tbMsg.ctx.copy(), tbMsg.callback);
}
public static TbMsg transformMsg(TbMsg tbMsg, CustomerId customerId) {
return new TbMsg(tbMsg.queueName, tbMsg.id, tbMsg.ts, tbMsg.type, tbMsg.originator, customerId, tbMsg.metaData, tbMsg.dataType,
return new TbMsg(tbMsg.queueId, tbMsg.id, tbMsg.ts, tbMsg.type, tbMsg.originator, customerId, tbMsg.metaData, tbMsg.dataType,
tbMsg.data, tbMsg.ruleChainId, tbMsg.ruleNodeId, tbMsg.ctx.copy(), tbMsg.getCallback());
}
public static TbMsg transformMsg(TbMsg tbMsg, RuleChainId ruleChainId) {
return new TbMsg(tbMsg.queueName, tbMsg.id, tbMsg.ts, tbMsg.type, tbMsg.originator, tbMsg.customerId, tbMsg.metaData, tbMsg.dataType,
return new TbMsg(tbMsg.queueId, tbMsg.id, tbMsg.ts, tbMsg.type, tbMsg.originator, tbMsg.customerId, tbMsg.metaData, tbMsg.dataType,
tbMsg.data, ruleChainId, null, tbMsg.ctx.copy(), tbMsg.getCallback());
}
public static TbMsg transformMsg(TbMsg tbMsg, String queueName) {
return new TbMsg(queueName, tbMsg.id, tbMsg.ts, tbMsg.type, tbMsg.originator, tbMsg.customerId, tbMsg.metaData, tbMsg.dataType,
public static TbMsg transformMsg(TbMsg tbMsg, QueueId queueId) {
return new TbMsg(queueId, tbMsg.id, tbMsg.ts, tbMsg.type, tbMsg.originator, tbMsg.customerId, tbMsg.metaData, tbMsg.dataType,
tbMsg.data, tbMsg.getRuleChainId(), null, tbMsg.ctx.copy(), tbMsg.getCallback());
}
public static TbMsg transformMsg(TbMsg tbMsg, RuleChainId ruleChainId, String queueName) {
return new TbMsg(queueName, tbMsg.id, tbMsg.ts, tbMsg.type, tbMsg.originator, tbMsg.customerId, tbMsg.metaData, tbMsg.dataType,
public static TbMsg transformMsg(TbMsg tbMsg, RuleChainId ruleChainId, QueueId queueId) {
return new TbMsg(queueId, tbMsg.id, tbMsg.ts, tbMsg.type, tbMsg.originator, tbMsg.customerId, tbMsg.metaData, tbMsg.dataType,
tbMsg.data, ruleChainId, null, tbMsg.ctx.copy(), tbMsg.getCallback());
}
//used for enqueueForTellNext
public static TbMsg newMsg(TbMsg tbMsg, String queueName, RuleChainId ruleChainId, RuleNodeId ruleNodeId) {
return new TbMsg(queueName, UUID.randomUUID(), tbMsg.getTs(), tbMsg.getType(), tbMsg.getOriginator(), tbMsg.customerId, tbMsg.getMetaData().copy(),
public static TbMsg newMsg(TbMsg tbMsg, QueueId queueId, RuleChainId ruleChainId, RuleNodeId ruleNodeId) {
return new TbMsg(queueId, UUID.randomUUID(), tbMsg.getTs(), tbMsg.getType(), tbMsg.getOriginator(), tbMsg.customerId, tbMsg.getMetaData().copy(),
tbMsg.getDataType(), tbMsg.getData(), ruleChainId, ruleNodeId, tbMsg.ctx.copy(), TbMsgCallback.EMPTY);
}
private TbMsg(String queueName, UUID id, long ts, String type, EntityId originator, CustomerId customerId, TbMsgMetaData metaData, TbMsgDataType dataType, String data,
private TbMsg(QueueId queueId, UUID id, long ts, String type, EntityId originator, CustomerId customerId, TbMsgMetaData metaData, TbMsgDataType dataType, String data,
RuleChainId ruleChainId, RuleNodeId ruleNodeId, TbMsgProcessingCtx ctx, TbMsgCallback callback) {
this.id = id;
this.queueName = queueName != null ? queueName : ServiceQueue.MAIN;
this.queueId = queueId;
if (ts > 0) {
this.ts = ts;
} else {
@ -221,7 +220,7 @@ public final class TbMsg implements Serializable {
return builder.build().toByteArray();
}
public static TbMsg fromBytes(String queueName, byte[] data, TbMsgCallback callback) {
public static TbMsg fromBytes(QueueId queueId, byte[] data, TbMsgCallback callback) {
try {
MsgProtos.TbMsgProto proto = MsgProtos.TbMsgProto.parseFrom(data);
TbMsgMetaData metaData = new TbMsgMetaData(proto.getMetaData().getDataMap());
@ -248,7 +247,7 @@ public final class TbMsg implements Serializable {
}
TbMsgDataType dataType = TbMsgDataType.values()[proto.getDataType()];
return new TbMsg(queueName, UUID.fromString(proto.getId()), proto.getTs(), proto.getType(), entityId, customerId,
return new TbMsg(queueId, UUID.fromString(proto.getId()), proto.getTs(), proto.getType(), entityId, customerId,
metaData, dataType, proto.getData(), ruleChainId, ruleNodeId, ctx, callback);
} catch (InvalidProtocolBufferException e) {
throw new IllegalStateException("Could not parse protobuf for TbMsg", e);
@ -260,12 +259,12 @@ public final class TbMsg implements Serializable {
}
public TbMsg copyWithRuleChainId(RuleChainId ruleChainId, UUID msgId) {
return new TbMsg(this.queueName, msgId, this.ts, this.type, this.originator, this.customerId,
return new TbMsg(this.queueId, msgId, this.ts, this.type, this.originator, this.customerId,
this.metaData, this.dataType, this.data, ruleChainId, null, this.ctx, callback);
}
public TbMsg copyWithRuleNodeId(RuleChainId ruleChainId, RuleNodeId ruleNodeId, UUID msgId) {
return new TbMsg(this.queueName, msgId, this.ts, this.type, this.originator, this.customerId,
return new TbMsg(this.queueId, msgId, this.ts, this.type, this.originator, this.customerId,
this.metaData, this.dataType, this.data, ruleChainId, ruleNodeId, this.ctx, callback);
}
@ -278,10 +277,6 @@ public final class TbMsg implements Serializable {
}
}
public String getQueueName() {
return queueName != null ? queueName : ServiceQueue.MAIN;
}
public void pushToStack(RuleChainId ruleChainId, RuleNodeId ruleNodeId) {
ctx.push(ruleChainId, ruleNodeId);
}

2
common/message/src/main/java/org/thingsboard/server/common/msg/queue/PartitionChangeMsg.java

@ -29,7 +29,7 @@ import java.util.Set;
public final class PartitionChangeMsg implements TbActorMsg {
@Getter
private final ServiceQueueKey serviceQueueKey;
private final ServiceType serviceType;
@Getter
private final Set<TopicPartitionInfo> partitions;

62
common/message/src/main/java/org/thingsboard/server/common/msg/queue/ServiceQueue.java

@ -1,62 +0,0 @@
/**
* Copyright © 2016-2022 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.common.msg.queue;
import lombok.ToString;
import java.util.Objects;
@ToString
public class ServiceQueue {
public static final String MAIN = "Main";
private final ServiceType type;
private final String queue;
public ServiceQueue(ServiceType type) {
this.type = type;
this.queue = MAIN;
}
public ServiceQueue(ServiceType type, String queue) {
this.type = type;
this.queue = queue != null ? queue : MAIN;
}
public ServiceType getType() {
return type;
}
public String getQueue() {
return queue;
}
@Override
public boolean equals(Object o) {
if (this == o) return true;
if (o == null || getClass() != o.getClass()) return false;
ServiceQueue that = (ServiceQueue) o;
return type == that.type &&
queue.equals(that.queue);
}
@Override
public int hashCode() {
return Objects.hash(type, queue);
}
}

54
common/message/src/main/java/org/thingsboard/server/common/msg/queue/ServiceQueueKey.java

@ -1,54 +0,0 @@
/**
* Copyright © 2016-2022 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.common.msg.queue;
import lombok.Getter;
import lombok.ToString;
import org.thingsboard.server.common.data.id.TenantId;
import java.util.Objects;
@ToString
public class ServiceQueueKey {
@Getter
private final ServiceQueue serviceQueue;
@Getter
private final TenantId tenantId;
public ServiceQueueKey(ServiceQueue serviceQueue, TenantId tenantId) {
this.serviceQueue = serviceQueue;
this.tenantId = tenantId;
}
@Override
public boolean equals(Object o) {
if (this == o) return true;
if (o == null || getClass() != o.getClass()) return false;
ServiceQueueKey that = (ServiceQueueKey) o;
return serviceQueue.equals(that.serviceQueue) &&
Objects.equals(tenantId, that.tenantId);
}
@Override
public int hashCode() {
return Objects.hash(serviceQueue, tenantId);
}
public ServiceType getServiceType() {
return serviceQueue.getType();
}
}

2
common/message/src/main/java/org/thingsboard/server/common/msg/queue/TopicPartitionInfo.java

@ -41,7 +41,7 @@ public class TopicPartitionInfo {
this.partition = partition;
this.myPartition = myPartition;
String tmp = topic;
if (tenantId != null) {
if (tenantId != null && !tenantId.isNullUid()) {
tmp += "." + tenantId.getId().toString();
}
if (partition != null) {

62
common/queue/src/main/java/org/thingsboard/server/queue/DefaultQueueService.java

@ -1,62 +0,0 @@
/**
* Copyright © 2016-2022 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.queue;
import lombok.RequiredArgsConstructor;
import org.springframework.stereotype.Service;
import org.springframework.util.StringUtils;
import org.thingsboard.server.common.msg.queue.ServiceQueue;
import org.thingsboard.server.common.msg.queue.ServiceType;
import org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings;
import org.thingsboard.server.queue.settings.TbRuleEngineQueueConfiguration;
import javax.annotation.PostConstruct;
import java.util.Collections;
import java.util.LinkedHashSet;
import java.util.Set;
import java.util.stream.Collectors;
@Service
@RequiredArgsConstructor
public class DefaultQueueService implements QueueService {
private final TbQueueRuleEngineSettings ruleEngineSettings;
private Set<String> ruleEngineQueues;
@PostConstruct
public void init() {
ruleEngineQueues = ruleEngineSettings.getQueues().stream()
.map(TbRuleEngineQueueConfiguration::getName).collect(Collectors.toCollection(LinkedHashSet::new));
}
@Override
public Set<String> getQueuesByServiceType(ServiceType type) {
if (type == ServiceType.TB_RULE_ENGINE) {
return ruleEngineQueues;
} else {
return Collections.emptySet();
}
}
@Override
public String resolve(ServiceType serviceType, String queueName) {
if (StringUtils.isEmpty(queueName) || !getQueuesByServiceType(serviceType).contains(queueName)) {
return ServiceQueue.MAIN;
} else {
return queueName;
}
}
}

114
common/queue/src/main/java/org/thingsboard/server/queue/RuleEngineTbQueueAdminFactory.java

@ -0,0 +1,114 @@
/**
* Copyright © 2016-2022 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.queue;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.thingsboard.server.queue.azure.servicebus.TbServiceBusAdmin;
import org.thingsboard.server.queue.azure.servicebus.TbServiceBusQueueConfigs;
import org.thingsboard.server.queue.azure.servicebus.TbServiceBusSettings;
import org.thingsboard.server.queue.kafka.TbKafkaAdmin;
import org.thingsboard.server.queue.kafka.TbKafkaSettings;
import org.thingsboard.server.queue.kafka.TbKafkaTopicConfigs;
import org.thingsboard.server.queue.pubsub.TbPubSubAdmin;
import org.thingsboard.server.queue.pubsub.TbPubSubSettings;
import org.thingsboard.server.queue.pubsub.TbPubSubSubscriptionSettings;
import org.thingsboard.server.queue.rabbitmq.TbRabbitMqAdmin;
import org.thingsboard.server.queue.rabbitmq.TbRabbitMqQueueArguments;
import org.thingsboard.server.queue.rabbitmq.TbRabbitMqSettings;
import org.thingsboard.server.queue.sqs.TbAwsSqsAdmin;
import org.thingsboard.server.queue.sqs.TbAwsSqsQueueAttributes;
import org.thingsboard.server.queue.sqs.TbAwsSqsSettings;
@Configuration
public class RuleEngineTbQueueAdminFactory {
@Autowired(required = false)
private TbKafkaTopicConfigs kafkaTopicConfigs;
@Autowired(required = false)
private TbKafkaSettings kafkaSettings;
@Autowired(required = false)
private TbAwsSqsQueueAttributes awsSqsQueueAttributes;
@Autowired(required = false)
private TbAwsSqsSettings awsSqsSettings;
@Autowired(required = false)
private TbPubSubSubscriptionSettings pubSubSubscriptionSettings;
@Autowired(required = false)
private TbPubSubSettings pubSubSettings;
@Autowired(required = false)
private TbRabbitMqQueueArguments rabbitMqQueueArguments;
@Autowired(required = false)
private TbRabbitMqSettings rabbitMqSettings;
@Autowired(required = false)
private TbServiceBusQueueConfigs serviceBusQueueConfigs;
@Autowired(required = false)
private TbServiceBusSettings serviceBusSettings;
@ConditionalOnExpression("'${queue.type:null}'=='kafka'")
@Bean
public TbQueueAdmin createKafkaAdmin() {
return new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getRuleEngineConfigs());
}
@ConditionalOnExpression("'${queue.type:null}'=='aws-sqs'")
@Bean
public TbQueueAdmin createAwsSqsAdmin() {
return new TbAwsSqsAdmin(awsSqsSettings, awsSqsQueueAttributes.getRuleEngineAttributes());
}
@ConditionalOnExpression("'${queue.type:null}'=='pubsub'")
@Bean
public TbQueueAdmin createPubSubAdmin() {
return new TbPubSubAdmin(pubSubSettings, pubSubSubscriptionSettings.getRuleEngineSettings());
}
@ConditionalOnExpression("'${queue.type:null}'=='rabbitmq'")
@Bean
public TbQueueAdmin createRabbitMqAdmin() {
return new TbRabbitMqAdmin(rabbitMqSettings, rabbitMqQueueArguments.getRuleEngineArgs());
}
@ConditionalOnExpression("'${queue.type:null}'=='service-bus'")
@Bean
public TbQueueAdmin createServiceBusAdmin() {
return new TbServiceBusAdmin(serviceBusSettings, serviceBusQueueConfigs.getRuleEngineConfigs());
}
@ConditionalOnExpression("'${queue.type:null}'=='in-memory'")
@Bean
public TbQueueAdmin createInMemoryAdmin() {
return new TbQueueAdmin() {
@Override
public void createTopicIfNotExists(String topic) {
}
@Override
public void deleteTopic(String topic) {
}
@Override
public void destroy() {
}
};
}
}

25
common/queue/src/main/java/org/thingsboard/server/queue/azure/servicebus/TbServiceBusAdmin.java

@ -82,6 +82,31 @@ public class TbServiceBusAdmin implements TbQueueAdmin {
}
}
@Override
public void deleteTopic(String topic) {
if (queues.contains(topic)) {
doDelete(topic);
} else {
try {
if (client.getQueue(topic) != null) {
doDelete(topic);
} else {
log.warn("Azure Service Bus Queue [{}] is not exist.", topic);
}
} catch (ServiceBusException | InterruptedException e) {
log.error("Failed to delete Azure Service Bus queue [{}]", topic, e);
}
}
}
private void doDelete(String topic) {
try {
client.deleteTopic(topic);
} catch (ServiceBusException | InterruptedException e) {
log.error("Failed to delete Azure Service Bus queue [{}]", topic, e);
}
}
private void setQueueConfigs(QueueDescription queueDescription) {
queueConfigs.forEach((confKey, confValue) -> {
switch (confKey) {

36
common/queue/src/main/java/org/thingsboard/server/queue/discovery/DefaultTbServiceInfoProvider.java

@ -23,12 +23,8 @@ import org.springframework.context.ApplicationContext;
import org.springframework.stereotype.Component;
import org.springframework.util.StringUtils;
import org.thingsboard.server.common.data.TbTransportService;
import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.msg.queue.ServiceType;
import org.thingsboard.server.gen.transport.TransportProtos;
import org.thingsboard.server.gen.transport.TransportProtos.ServiceInfo;
import org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings;
import org.thingsboard.server.queue.settings.TbRuleEngineQueueConfiguration;
import org.thingsboard.server.queue.util.AfterContextReady;
import javax.annotation.PostConstruct;
@ -38,8 +34,6 @@ import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.List;
import java.util.Optional;
import java.util.UUID;
import java.util.stream.Collectors;
@Component
@ -54,18 +48,11 @@ public class DefaultTbServiceInfoProvider implements TbServiceInfoProvider {
@Value("${service.type:monolith}")
private String serviceType;
@Getter
@Value("${service.tenant_id:}")
private String tenantIdStr;
@Autowired(required = false)
private TbQueueRuleEngineSettings ruleEngineSettings;
@Autowired
private ApplicationContext applicationContext;
private List<ServiceType> serviceTypes;
private ServiceInfo serviceInfo;
private TenantId isolatedTenant;
@PostConstruct
public void init() {
@ -85,25 +72,6 @@ public class DefaultTbServiceInfoProvider implements TbServiceInfoProvider {
ServiceInfo.Builder builder = ServiceInfo.newBuilder()
.setServiceId(serviceId)
.addAllServiceTypes(serviceTypes.stream().map(ServiceType::name).collect(Collectors.toList()));
UUID tenantId;
if (!StringUtils.isEmpty(tenantIdStr)) {
tenantId = UUID.fromString(tenantIdStr);
isolatedTenant = TenantId.fromUUID(tenantId);
} else {
tenantId = TenantId.NULL_UUID;
}
builder.setTenantIdMSB(tenantId.getMostSignificantBits());
builder.setTenantIdLSB(tenantId.getLeastSignificantBits());
if (serviceTypes.contains(ServiceType.TB_RULE_ENGINE) && ruleEngineSettings != null) {
for (TbRuleEngineQueueConfiguration queue : ruleEngineSettings.getQueues()) {
TransportProtos.QueueInfo queueInfo = TransportProtos.QueueInfo.newBuilder()
.setName(queue.getName())
.setTopic(queue.getTopic())
.setPartitions(queue.getPartitions()).build();
builder.addRuleEngineQueues(queueInfo);
}
}
serviceInfo = builder.build();
}
@ -131,8 +99,4 @@ public class DefaultTbServiceInfoProvider implements TbServiceInfoProvider {
return serviceTypes.contains(serviceType);
}
@Override
public Optional<TenantId> getIsolatedTenant() {
return Optional.ofNullable(isolatedTenant);
}
}

311
common/queue/src/main/java/org/thingsboard/server/queue/discovery/HashPartitionService.java

@ -16,26 +16,21 @@
package org.thingsboard.server.queue.discovery;
import com.google.common.hash.HashFunction;
import com.google.common.hash.Hasher;
import com.google.common.hash.Hashing;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.context.ApplicationEventPublisher;
import org.springframework.stereotype.Service;
import org.springframework.util.StringUtils;
import org.thingsboard.server.common.data.id.EntityId;
import org.thingsboard.server.common.data.id.QueueId;
import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.msg.queue.ServiceQueue;
import org.thingsboard.server.common.msg.queue.ServiceQueueKey;
import org.thingsboard.server.common.msg.queue.ServiceType;
import org.thingsboard.server.common.msg.queue.TopicPartitionInfo;
import org.thingsboard.server.gen.transport.TransportProtos;
import org.thingsboard.server.gen.transport.TransportProtos.ServiceInfo;
import org.thingsboard.server.queue.QueueService;
import org.thingsboard.server.queue.discovery.event.ClusterTopologyChangeEvent;
import org.thingsboard.server.queue.discovery.event.PartitionChangeEvent;
import org.thingsboard.server.queue.discovery.event.ServiceListChangedEvent;
import org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings;
import javax.annotation.PostConstruct;
import java.util.ArrayList;
@ -65,17 +60,17 @@ public class HashPartitionService implements PartitionService {
private final ApplicationEventPublisher applicationEventPublisher;
private final TbServiceInfoProvider serviceInfoProvider;
private final TenantRoutingInfoService tenantRoutingInfoService;
private final TbQueueRuleEngineSettings tbQueueRuleEngineSettings;
private final QueueService queueService;
private final ConcurrentMap<ServiceQueue, String> partitionTopics = new ConcurrentHashMap<>();
private final ConcurrentMap<ServiceQueue, Integer> partitionSizes = new ConcurrentHashMap<>();
private final ConcurrentMap<TenantId, TenantRoutingInfo> tenantRoutingInfoMap = new ConcurrentHashMap<>();
private final QueueRoutingInfoService queueRoutingInfoService;
private final ConcurrentMap<QueueId, QueueRoutingInfo> queuesById = new ConcurrentHashMap<>();
private ConcurrentMap<QueueKey, List<Integer>> myPartitions = new ConcurrentHashMap<>();
private ConcurrentMap<ServiceQueueKey, List<Integer>> myPartitions = new ConcurrentHashMap<>();
private ConcurrentMap<TopicPartitionInfoKey, TopicPartitionInfo> tpiCache = new ConcurrentHashMap<>();
private final ConcurrentMap<QueueKey, String> partitionTopicsMap = new ConcurrentHashMap<>();
private final ConcurrentMap<QueueKey, Integer> partitionSizesMap = new ConcurrentHashMap<>();
private final ConcurrentMap<TenantId, TenantRoutingInfo> tenantRoutingInfoMap = new ConcurrentHashMap<>();
private Map<String, TopicPartitionInfo> tbCoreNotificationTopics = new HashMap<>();
private Map<String, TopicPartitionInfo> tbRuleEngineNotificationTopics = new HashMap<>();
private Map<String, List<ServiceInfo>> tbTransportServicesByType = new HashMap<>();
private List<ServiceInfo> currentOtherServices;
@ -84,105 +79,188 @@ public class HashPartitionService implements PartitionService {
public HashPartitionService(TbServiceInfoProvider serviceInfoProvider,
TenantRoutingInfoService tenantRoutingInfoService,
ApplicationEventPublisher applicationEventPublisher,
TbQueueRuleEngineSettings tbQueueRuleEngineSettings,
QueueService queueService) {
QueueRoutingInfoService queueRoutingInfoService) {
this.serviceInfoProvider = serviceInfoProvider;
this.tenantRoutingInfoService = tenantRoutingInfoService;
this.applicationEventPublisher = applicationEventPublisher;
this.tbQueueRuleEngineSettings = tbQueueRuleEngineSettings;
this.queueService = queueService;
this.queueRoutingInfoService = queueRoutingInfoService;
}
@PostConstruct
public void init() {
this.hashFunction = forName(hashFunctionName);
partitionSizes.put(new ServiceQueue(ServiceType.TB_CORE), corePartitions);
partitionTopics.put(new ServiceQueue(ServiceType.TB_CORE), coreTopic);
tbQueueRuleEngineSettings.getQueues().forEach(queueConfiguration -> {
partitionTopics.put(new ServiceQueue(ServiceType.TB_RULE_ENGINE, queueConfiguration.getName()), queueConfiguration.getTopic());
partitionSizes.put(new ServiceQueue(ServiceType.TB_RULE_ENGINE, queueConfiguration.getName()), queueConfiguration.getPartitions());
partitionsInit();
}
private void partitionsInit() {
QueueKey coreKey = new QueueKey(ServiceType.TB_CORE);
partitionSizesMap.put(coreKey, corePartitions);
partitionTopicsMap.put(coreKey, coreTopic);
List<QueueRoutingInfo> queueRoutingInfoList;
String serviceType = serviceInfoProvider.getServiceType();
if ("tb-transport".equals(serviceType)) {
//If transport started earlier than tb-core
int getQueuesRetries = 10;
while (true) {
if (getQueuesRetries > 0) {
log.info("Try to get queue routing info.");
try {
queueRoutingInfoList = queueRoutingInfoService.getAllQueuesRoutingInfo();
break;
} catch (Exception e) {
log.info("Failed to get queues routing info!");
getQueuesRetries--;
}
try {
Thread.sleep(10000);
} catch (InterruptedException e) {
log.info("Failed to await queues routing info!", e);
}
} else {
throw new RuntimeException("Failed to await queues routing info!");
}
}
} else {
queueRoutingInfoList = queueRoutingInfoService.getAllQueuesRoutingInfo();
}
queueRoutingInfoList.forEach(queue -> {
QueueKey queueKey = new QueueKey(ServiceType.TB_RULE_ENGINE, queue);
partitionTopicsMap.put(queueKey, queue.getQueueTopic());
partitionSizesMap.put(queueKey, queue.getPartitions());
queuesById.put(queue.getQueueId(), queue);
});
}
@Override
public void updateQueue(TransportProtos.QueueUpdateMsg queueUpdateMsg) {
TenantId tenantId = new TenantId(new UUID(queueUpdateMsg.getTenantIdMSB(), queueUpdateMsg.getTenantIdLSB()));
QueueKey queueKey = new QueueKey(ServiceType.TB_RULE_ENGINE, queueUpdateMsg.getQueueName(), tenantId);
QueueRoutingInfo queue = new QueueRoutingInfo(queueUpdateMsg);
queuesById.put(queue.getQueueId(), queue);
partitionTopicsMap.put(queueKey, queueUpdateMsg.getQueueTopic());
partitionSizesMap.put(queueKey, queueUpdateMsg.getPartitions());
myPartitions.remove(queueKey);
}
@Override
public void removeQueue(TransportProtos.QueueDeleteMsg queueDeleteMsg) {
TenantId tenantId = new TenantId(new UUID(queueDeleteMsg.getTenantIdMSB(), queueDeleteMsg.getTenantIdLSB()));
QueueKey queueKey = new QueueKey(ServiceType.TB_RULE_ENGINE, queueDeleteMsg.getQueueName(), tenantId);
queuesById.remove(new QueueId(new UUID(queueDeleteMsg.getQueueIdMSB(), queueDeleteMsg.getQueueIdLSB())));
myPartitions.remove(queueKey);
partitionTopicsMap.remove(queueKey);
partitionSizesMap.remove(queueKey);
//TODO: remove after merging tb entity services
removeTenant(tenantId);
}
@Override
@Deprecated
public TopicPartitionInfo resolve(ServiceType serviceType, TenantId tenantId, EntityId entityId, String queueName) {
log.warn("This method is deprecated and will be removed!!!");
TenantId isolatedOrSystemTenantId = getIsolatedOrSystemTenantId(serviceType, tenantId);
QueueKey queueKey = new QueueKey(serviceType, queueName, isolatedOrSystemTenantId);
if (!partitionSizesMap.containsKey(queueKey)) {
queueKey = new QueueKey(serviceType, isolatedOrSystemTenantId);
}
return resolve(queueKey, entityId);
}
@Override
public TopicPartitionInfo resolve(ServiceType serviceType, TenantId tenantId, EntityId entityId) {
return resolve(new ServiceQueue(serviceType), tenantId, entityId);
return resolve(serviceType, null, tenantId, entityId);
}
@Override
public TopicPartitionInfo resolve(ServiceType serviceType, String queueName, TenantId tenantId, EntityId entityId) {
queueName = queueService.resolve(serviceType, queueName);
return resolve(new ServiceQueue(serviceType, queueName), tenantId, entityId);
public TopicPartitionInfo resolve(ServiceType serviceType, QueueId queueId, TenantId tenantId, EntityId entityId) {
QueueKey queueKey;
if (queueId == null) {
queueKey = getMainQueueKey(serviceType, tenantId);
} else {
QueueRoutingInfo queueRoutingInfo = queuesById.get(queueId);
if (queueRoutingInfo == null) {
log.debug("Queue was removed but still used in CheckPoint rule node. [{}][{}]", tenantId, entityId);
queueKey = getMainQueueKey(serviceType, tenantId);
} else if (!queueRoutingInfo.getTenantId().equals(getIsolatedOrSystemTenantId(serviceType, tenantId))) {
log.debug("Tenant profile was changed but CheckPoint rule node still uses the queue from system level. [{}][{}]", tenantId, entityId);
queueKey = getMainQueueKey(serviceType, tenantId);
} else {
queueKey = new QueueKey(serviceType, queueRoutingInfo);
}
}
return resolve(queueKey, entityId);
}
private TopicPartitionInfo resolve(ServiceQueue serviceQueue, TenantId tenantId, EntityId entityId) {
private QueueKey getMainQueueKey(ServiceType serviceType, TenantId tenantId) {
return new QueueKey(serviceType, getIsolatedOrSystemTenantId(serviceType, tenantId));
}
private TopicPartitionInfo resolve(QueueKey queueKey, EntityId entityId) {
int hash = hashFunction.newHasher()
.putLong(entityId.getId().getMostSignificantBits())
.putLong(entityId.getId().getLeastSignificantBits()).hash().asInt();
Integer partitionSize = partitionSizes.get(serviceQueue);
int partition;
if (partitionSize != null) {
partition = Math.abs(hash % partitionSize);
} else {
//TODO: In 2.6/3.1 this should not happen because all Rule Engine Queues will be in the DB and we always know their partition sizes.
partition = 0;
}
boolean isolatedTenant = isIsolated(serviceQueue, tenantId);
TopicPartitionInfoKey cacheKey = new TopicPartitionInfoKey(serviceQueue, isolatedTenant ? tenantId : null, partition);
return tpiCache.computeIfAbsent(cacheKey, key -> buildTopicPartitionInfo(serviceQueue, tenantId, partition));
Integer partitionSize = partitionSizesMap.get(queueKey);
int partition = Math.abs(hash % partitionSize);
return buildTopicPartitionInfo(queueKey, partition);
}
@Override
public synchronized void recalculatePartitions(ServiceInfo currentService, List<ServiceInfo> otherServices) {
partitionsInit();
tbTransportServicesByType.clear();
logServiceInfo(currentService);
otherServices.forEach(this::logServiceInfo);
Map<ServiceQueueKey, List<ServiceInfo>> queueServicesMap = new HashMap<>();
Map<QueueKey, List<ServiceInfo>> queueServicesMap = new HashMap<>();
addNode(queueServicesMap, currentService);
for (ServiceInfo other : otherServices) {
addNode(queueServicesMap, other);
}
queueServicesMap.values().forEach(list -> list.sort(Comparator.comparing(ServiceInfo::getServiceId)));
ConcurrentMap<ServiceQueueKey, List<Integer>> oldPartitions = myPartitions;
TenantId myIsolatedOrSystemTenantId = getSystemOrIsolatedTenantId(currentService);
ConcurrentMap<QueueKey, List<Integer>> oldPartitions = myPartitions;
myPartitions = new ConcurrentHashMap<>();
partitionSizes.forEach((serviceQueue, size) -> {
ServiceQueueKey myServiceQueueKey = new ServiceQueueKey(serviceQueue, myIsolatedOrSystemTenantId);
partitionSizesMap.forEach((queueKey, size) -> {
for (int i = 0; i < size; i++) {
ServiceInfo serviceInfo = resolveByPartitionIdx(queueServicesMap.get(myServiceQueueKey), i);
ServiceInfo serviceInfo = resolveByPartitionIdx(queueServicesMap.get(queueKey), i);
if (currentService.equals(serviceInfo)) {
ServiceQueueKey serviceQueueKey = new ServiceQueueKey(serviceQueue, getSystemOrIsolatedTenantId(serviceInfo));
myPartitions.computeIfAbsent(serviceQueueKey, key -> new ArrayList<>()).add(i);
myPartitions.computeIfAbsent(queueKey, key -> new ArrayList<>()).add(i);
}
}
});
tpiCache.clear();
oldPartitions.forEach((serviceQueueKey, partitions) -> {
if (!myPartitions.containsKey(serviceQueueKey)) {
log.info("[{}] NO MORE PARTITIONS FOR CURRENT KEY", serviceQueueKey);
applicationEventPublisher.publishEvent(new PartitionChangeEvent(this, serviceQueueKey, Collections.emptySet()));
oldPartitions.forEach((queueKey, partitions) -> {
if (!myPartitions.containsKey(queueKey)) {
log.info("[{}] NO MORE PARTITIONS FOR CURRENT KEY", queueKey);
applicationEventPublisher.publishEvent(new PartitionChangeEvent(this, queueKey, Collections.emptySet()));
}
});
myPartitions.forEach((serviceQueueKey, partitions) -> {
if (!partitions.equals(oldPartitions.get(serviceQueueKey))) {
log.info("[{}] NEW PARTITIONS: {}", serviceQueueKey, partitions);
myPartitions.forEach((queueKey, partitions) -> {
if (!partitions.equals(oldPartitions.get(queueKey))) {
log.info("[{}] NEW PARTITIONS: {}", queueKey, partitions);
Set<TopicPartitionInfo> tpiList = partitions.stream()
.map(partition -> buildTopicPartitionInfo(serviceQueueKey, partition))
.map(partition -> buildTopicPartitionInfo(queueKey, partition))
.collect(Collectors.toSet());
applicationEventPublisher.publishEvent(new PartitionChangeEvent(this, serviceQueueKey, tpiList));
applicationEventPublisher.publishEvent(new PartitionChangeEvent(this, queueKey, tpiList));
}
});
if (currentOtherServices == null) {
currentOtherServices = new ArrayList<>(otherServices);
} else {
Set<ServiceQueueKey> changes = new HashSet<>();
Map<ServiceQueueKey, List<ServiceInfo>> currentMap = getServiceKeyListMap(currentOtherServices);
Map<ServiceQueueKey, List<ServiceInfo>> newMap = getServiceKeyListMap(otherServices);
Set<QueueKey> changes = new HashSet<>();
Map<QueueKey, List<ServiceInfo>> currentMap = getServiceKeyListMap(currentOtherServices);
Map<QueueKey, List<ServiceInfo>> newMap = getServiceKeyListMap(otherServices);
currentOtherServices = otherServices;
currentMap.forEach((key, list) -> {
if (!list.equals(newMap.get(key))) {
@ -201,34 +279,32 @@ public class HashPartitionService implements PartitionService {
@Override
public Set<String> getAllServiceIds(ServiceType serviceType) {
Set<String> result = new HashSet<>();
return getAllServices(serviceType).stream().map(ServiceInfo::getServiceId).collect(Collectors.toSet());
}
@Override
public Set<ServiceInfo> getAllServices(ServiceType serviceType) {
Set<ServiceInfo> result = getOtherServices(serviceType);
ServiceInfo current = serviceInfoProvider.getServiceInfo();
if (current.getServiceTypesList().contains(serviceType.name())) {
result.add(current.getServiceId());
result.add(current);
}
return result;
}
@Override
public Set<ServiceInfo> getOtherServices(ServiceType serviceType) {
Set<ServiceInfo> result = new HashSet<>();
if (currentOtherServices != null) {
for (ServiceInfo serviceInfo : currentOtherServices) {
if (serviceInfo.getServiceTypesList().contains(serviceType.name())) {
result.add(serviceInfo.getServiceId());
result.add(serviceInfo);
}
}
}
return result;
}
@Override
public TopicPartitionInfo getNotificationsTopic(ServiceType serviceType, String serviceId) {
switch (serviceType) {
case TB_CORE:
return tbCoreNotificationTopics.computeIfAbsent(serviceId,
id -> buildNotificationsTopicPartitionInfo(serviceType, serviceId));
case TB_RULE_ENGINE:
return tbRuleEngineNotificationTopics.computeIfAbsent(serviceId,
id -> buildNotificationsTopicPartitionInfo(serviceType, serviceId));
default:
return buildNotificationsTopicPartitionInfo(serviceType, serviceId);
}
}
@Override
public int resolvePartitionIndex(UUID entityId, int partitions) {
@ -238,51 +314,41 @@ public class HashPartitionService implements PartitionService {
return Math.abs(hash % partitions);
}
@Override
public void removeTenant(TenantId tenantId) {
tenantRoutingInfoMap.remove(tenantId);
}
@Override
public int countTransportsByType(String type) {
var list = tbTransportServicesByType.get(type);
return list == null ? 0 : list.size();
}
private Map<ServiceQueueKey, List<ServiceInfo>> getServiceKeyListMap(List<ServiceInfo> services) {
final Map<ServiceQueueKey, List<ServiceInfo>> currentMap = new HashMap<>();
private Map<QueueKey, List<ServiceInfo>> getServiceKeyListMap(List<ServiceInfo> services) {
final Map<QueueKey, List<ServiceInfo>> currentMap = new HashMap<>();
services.forEach(serviceInfo -> {
for (String serviceTypeStr : serviceInfo.getServiceTypesList()) {
ServiceType serviceType = ServiceType.valueOf(serviceTypeStr.toUpperCase());
if (ServiceType.TB_RULE_ENGINE.equals(serviceType)) {
for (TransportProtos.QueueInfo queue : serviceInfo.getRuleEngineQueuesList()) {
ServiceQueueKey serviceQueueKey = new ServiceQueueKey(new ServiceQueue(serviceType, queue.getName()), getSystemOrIsolatedTenantId(serviceInfo));
currentMap.computeIfAbsent(serviceQueueKey, key -> new ArrayList<>()).add(serviceInfo);
}
partitionTopicsMap.keySet().forEach(queueKey ->
currentMap.computeIfAbsent(queueKey, key -> new ArrayList<>()).add(serviceInfo));
} else {
ServiceQueueKey serviceQueueKey = new ServiceQueueKey(new ServiceQueue(serviceType), getSystemOrIsolatedTenantId(serviceInfo));
currentMap.computeIfAbsent(serviceQueueKey, key -> new ArrayList<>()).add(serviceInfo);
QueueKey queueKey = new QueueKey(serviceType);
currentMap.computeIfAbsent(queueKey, key -> new ArrayList<>()).add(serviceInfo);
}
}
});
return currentMap;
}
private TopicPartitionInfo buildNotificationsTopicPartitionInfo(ServiceType serviceType, String serviceId) {
return new TopicPartitionInfo(serviceType.name().toLowerCase() + ".notifications." + serviceId, null, null, false);
}
private TopicPartitionInfo buildTopicPartitionInfo(ServiceQueueKey serviceQueueKey, int partition) {
return buildTopicPartitionInfo(serviceQueueKey.getServiceQueue(), serviceQueueKey.getTenantId(), partition);
}
private TopicPartitionInfo buildTopicPartitionInfo(ServiceQueue serviceQueue, TenantId tenantId, int partition) {
private TopicPartitionInfo buildTopicPartitionInfo(QueueKey queueKey, int partition) {
TopicPartitionInfo.TopicPartitionInfoBuilder tpi = TopicPartitionInfo.builder();
tpi.topic(partitionTopics.get(serviceQueue));
tpi.topic(partitionTopicsMap.get(queueKey));
tpi.partition(partition);
ServiceQueueKey myPartitionsSearchKey;
if (isIsolated(serviceQueue, tenantId)) {
tpi.tenantId(tenantId);
myPartitionsSearchKey = new ServiceQueueKey(serviceQueue, tenantId);
} else {
myPartitionsSearchKey = new ServiceQueueKey(serviceQueue, TenantId.SYS_TENANT_ID);
}
List<Integer> partitions = myPartitions.get(myPartitionsSearchKey);
tpi.tenantId(queueKey.getTenantId());
List<Integer> partitions = myPartitions.get(queueKey);
if (partitions != null) {
tpi.myPartition(partitions.contains(partition));
} else {
@ -291,7 +357,7 @@ public class HashPartitionService implements PartitionService {
return tpi.build();
}
private boolean isIsolated(ServiceQueue serviceQueue, TenantId tenantId) {
private boolean isIsolated(ServiceType serviceType, TenantId tenantId) {
if (TenantId.SYS_TENANT_ID.equals(tenantId)) {
return false;
}
@ -308,7 +374,7 @@ public class HashPartitionService implements PartitionService {
if (routingInfo == null) {
throw new RuntimeException("Tenant not found!");
}
switch (serviceQueue.getType()) {
switch (serviceType) {
case TB_CORE:
return routingInfo.isIsolatedTbCore();
case TB_RULE_ENGINE:
@ -318,35 +384,28 @@ public class HashPartitionService implements PartitionService {
}
}
private void logServiceInfo(TransportProtos.ServiceInfo server) {
TenantId tenantId = getSystemOrIsolatedTenantId(server);
if (tenantId.isNullUid()) {
log.info("[{}] Found common server: [{}]", server.getServiceId(), server.getServiceTypesList());
} else {
log.info("[{}][{}] Found specific server: [{}]", server.getServiceId(), tenantId, server.getServiceTypesList());
}
private TenantId getIsolatedOrSystemTenantId(ServiceType serviceType, TenantId tenantId) {
return isIsolated(serviceType, tenantId) ? tenantId : TenantId.SYS_TENANT_ID;
}
private TenantId getSystemOrIsolatedTenantId(TransportProtos.ServiceInfo serviceInfo) {
return TenantId.fromUUID(new UUID(serviceInfo.getTenantIdMSB(), serviceInfo.getTenantIdLSB()));
private void logServiceInfo(TransportProtos.ServiceInfo server) {
log.info("[{}] Found common server: [{}]", server.getServiceId(), server.getServiceTypesList());
}
private void addNode(Map<ServiceQueueKey, List<ServiceInfo>> queueServiceList, ServiceInfo instance) {
TenantId tenantId = getSystemOrIsolatedTenantId(instance);
private void addNode(Map<QueueKey, List<ServiceInfo>> queueServiceList, ServiceInfo instance) {
for (String serviceTypeStr : instance.getServiceTypesList()) {
ServiceType serviceType = ServiceType.valueOf(serviceTypeStr.toUpperCase());
if (ServiceType.TB_RULE_ENGINE.equals(serviceType)) {
for (TransportProtos.QueueInfo queue : instance.getRuleEngineQueuesList()) {
ServiceQueueKey serviceQueueKey = new ServiceQueueKey(new ServiceQueue(serviceType, queue.getName()), tenantId);
partitionSizes.put(new ServiceQueue(ServiceType.TB_RULE_ENGINE, queue.getName()), queue.getPartitions());
partitionTopics.put(new ServiceQueue(ServiceType.TB_RULE_ENGINE, queue.getName()), queue.getTopic());
queueServiceList.computeIfAbsent(serviceQueueKey, key -> new ArrayList<>()).add(instance);
}
} else {
ServiceQueueKey serviceQueueKey = new ServiceQueueKey(new ServiceQueue(serviceType), tenantId);
queueServiceList.computeIfAbsent(serviceQueueKey, key -> new ArrayList<>()).add(instance);
partitionTopicsMap.keySet().forEach(key -> {
if (key.getType().equals(ServiceType.TB_RULE_ENGINE)) {
queueServiceList.computeIfAbsent(key, k -> new ArrayList<>()).add(instance);
}
});
} else if (ServiceType.TB_CORE.equals(serviceType)) {
queueServiceList.computeIfAbsent(new QueueKey(serviceType), key -> new ArrayList<>()).add(instance);
}
}
for (String transportType : instance.getTransportsList()) {
tbTransportServicesByType.computeIfAbsent(transportType, t -> new ArrayList<>()).add(instance);
}

54
common/queue/src/main/java/org/thingsboard/server/queue/discovery/NotificationsTopicService.java

@ -0,0 +1,54 @@
/**
* Copyright © 2016-2022 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.queue.discovery;
import org.springframework.stereotype.Service;
import org.thingsboard.server.common.msg.queue.ServiceType;
import org.thingsboard.server.common.msg.queue.TopicPartitionInfo;
import java.util.HashMap;
import java.util.Map;
@Service
public class NotificationsTopicService {
private Map<String, TopicPartitionInfo> tbCoreNotificationTopics = new HashMap<>();
private Map<String, TopicPartitionInfo> tbRuleEngineNotificationTopics = new HashMap<>();
/**
* Each Service should start a consumer for messages that target individual service instance based on serviceId.
* This topic is likely to have single partition, and is always assigned to the service.
* @param serviceType
* @param serviceId
* @return
*/
public TopicPartitionInfo getNotificationsTopic(ServiceType serviceType, String serviceId) {
switch (serviceType) {
case TB_CORE:
return tbCoreNotificationTopics.computeIfAbsent(serviceId,
id -> buildNotificationsTopicPartitionInfo(serviceType, serviceId));
case TB_RULE_ENGINE:
return tbRuleEngineNotificationTopics.computeIfAbsent(serviceId,
id -> buildNotificationsTopicPartitionInfo(serviceType, serviceId));
default:
return buildNotificationsTopicPartitionInfo(serviceType, serviceId);
}
}
private TopicPartitionInfo buildNotificationsTopicPartitionInfo(ServiceType serviceType, String serviceId) {
return new TopicPartitionInfo(serviceType.name().toLowerCase() + ".notifications." + serviceId, null, null, false);
}
}

23
common/queue/src/main/java/org/thingsboard/server/queue/discovery/PartitionService.java

@ -16,6 +16,7 @@
package org.thingsboard.server.queue.discovery;
import org.thingsboard.server.common.data.id.EntityId;
import org.thingsboard.server.common.data.id.QueueId;
import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.msg.queue.ServiceType;
import org.thingsboard.server.common.msg.queue.TopicPartitionInfo;
@ -31,9 +32,12 @@ import java.util.UUID;
*/
public interface PartitionService {
@Deprecated
TopicPartitionInfo resolve(ServiceType serviceType, TenantId tenantId, EntityId entityId, String queueName);
TopicPartitionInfo resolve(ServiceType serviceType, TenantId tenantId, EntityId entityId);
TopicPartitionInfo resolve(ServiceType serviceType, String queueName, TenantId tenantId, EntityId entityId);
TopicPartitionInfo resolve(ServiceType serviceType, QueueId queueId, TenantId tenantId, EntityId entityId);
/**
* Received from the Discovery service when network topology is changed.
@ -49,16 +53,17 @@ public interface PartitionService {
*/
Set<String> getAllServiceIds(ServiceType serviceType);
/**
* Each Service should start a consumer for messages that target individual service instance based on serviceId.
* This topic is likely to have single partition, and is always assigned to the service.
* @param serviceType
* @param serviceId
* @return
*/
TopicPartitionInfo getNotificationsTopic(ServiceType serviceType, String serviceId);
Set<TransportProtos.ServiceInfo> getAllServices(ServiceType serviceType);
Set<TransportProtos.ServiceInfo> getOtherServices(ServiceType serviceType);
int resolvePartitionIndex(UUID entityId, int partitions);
void removeTenant(TenantId tenantId);
int countTransportsByType(String type);
void updateQueue(TransportProtos.QueueUpdateMsg queueUpdateMsg);
void removeQueue(TransportProtos.QueueDeleteMsg queueDeleteMsg);
}

56
common/queue/src/main/java/org/thingsboard/server/queue/discovery/QueueKey.java

@ -0,0 +1,56 @@
/**
* Copyright © 2016-2022 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.queue.discovery;
import lombok.AllArgsConstructor;
import lombok.Data;
import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.data.queue.Queue;
import org.thingsboard.server.common.msg.queue.ServiceType;
@Data
@AllArgsConstructor
public class QueueKey {
private static final String MAIN = "Main";
private final ServiceType type;
private final String queueName;
private final TenantId tenantId;
public QueueKey(ServiceType type, Queue queue) {
this.type = type;
this.queueName = queue.getName();
this.tenantId = queue.getTenantId();
}
public QueueKey(ServiceType type, QueueRoutingInfo queueRoutingInfo) {
this.type = type;
this.queueName = queueRoutingInfo.getQueueName();
this.tenantId = queueRoutingInfo.getTenantId();
}
public QueueKey(ServiceType type, TenantId tenantId) {
this.type = type;
this.queueName = MAIN;
this.tenantId = tenantId != null ? tenantId : TenantId.SYS_TENANT_ID;
}
public QueueKey(ServiceType type) {
this.type = type;
this.queueName = MAIN;
this.tenantId = TenantId.SYS_TENANT_ID;
}
}

60
common/queue/src/main/java/org/thingsboard/server/queue/discovery/QueueRoutingInfo.java

@ -0,0 +1,60 @@
/**
* Copyright © 2016-2022 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.queue.discovery;
import lombok.Data;
import org.thingsboard.server.common.data.id.QueueId;
import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.data.queue.Queue;
import org.thingsboard.server.gen.transport.TransportProtos.GetQueueRoutingInfoResponseMsg;
import org.thingsboard.server.gen.transport.TransportProtos.QueueUpdateMsg;
import java.util.UUID;
@Data
public class QueueRoutingInfo {
private final TenantId tenantId;
private final QueueId queueId;
private final String queueName;
private final String queueTopic;
private final int partitions;
public QueueRoutingInfo(Queue queue) {
this.tenantId = queue.getTenantId();
this.queueId = queue.getId();
this.queueName = queue.getName();
this.queueTopic = queue.getTopic();
this.partitions = queue.getPartitions();
}
public QueueRoutingInfo(GetQueueRoutingInfoResponseMsg routingInfo) {
this.tenantId = new TenantId(new UUID(routingInfo.getTenantIdMSB(), routingInfo.getTenantIdLSB()));
this.queueId = new QueueId(new UUID(routingInfo.getQueueIdMSB(), routingInfo.getQueueIdLSB()));
this.queueName = routingInfo.getQueueName();
this.queueTopic = routingInfo.getQueueTopic();
this.partitions = routingInfo.getPartitions();
}
public QueueRoutingInfo(QueueUpdateMsg queueUpdateMsg) {
this.tenantId = new TenantId(new UUID(queueUpdateMsg.getTenantIdMSB(), queueUpdateMsg.getTenantIdLSB()));
this.queueId = new QueueId(new UUID(queueUpdateMsg.getQueueIdMSB(), queueUpdateMsg.getQueueIdLSB()));
this.queueName = queueUpdateMsg.getQueueName();
this.queueTopic = queueUpdateMsg.getQueueTopic();
this.partitions = queueUpdateMsg.getPartitions();
}
}

24
common/queue/src/main/java/org/thingsboard/server/queue/discovery/QueueRoutingInfoService.java

@ -0,0 +1,24 @@
/**
* Copyright © 2016-2022 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.queue.discovery;
import java.util.List;
public interface QueueRoutingInfoService {
List<QueueRoutingInfo> getAllQueuesRoutingInfo();
}

7
common/queue/src/main/java/org/thingsboard/server/queue/discovery/TbServiceInfoProvider.java

@ -15,20 +15,17 @@
*/
package org.thingsboard.server.queue.discovery;
import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.msg.queue.ServiceType;
import org.thingsboard.server.gen.transport.TransportProtos.ServiceInfo;
import java.util.Optional;
public interface TbServiceInfoProvider {
String getServiceId();
String getServiceType();
ServiceInfo getServiceInfo();
boolean isService(ServiceType serviceType);
Optional<TenantId> getIsolatedTenant();
}

44
common/queue/src/main/java/org/thingsboard/server/queue/discovery/TopicPartitionInfoKey.java

@ -1,44 +0,0 @@
/**
* Copyright © 2016-2022 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.queue.discovery;
import lombok.AllArgsConstructor;
import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.msg.queue.ServiceQueue;
import java.util.Objects;
@AllArgsConstructor
public class TopicPartitionInfoKey {
private ServiceQueue serviceQueue;
private TenantId isolatedTenantId;
private int partition;
@Override
public boolean equals(Object o) {
if (this == o) return true;
if (o == null || getClass() != o.getClass()) return false;
TopicPartitionInfoKey that = (TopicPartitionInfoKey) o;
return partition == that.partition &&
serviceQueue.equals(that.serviceQueue) &&
Objects.equals(isolatedTenantId, that.isolatedTenantId);
}
@Override
public int hashCode() {
return Objects.hash(serviceQueue, isolatedTenantId, partition);
}
}

Some files were not shown because too many files changed in this diff

Loading…
Cancel
Save