From 18ce96fd10580d04912a589012fa1249c6824542 Mon Sep 17 00:00:00 2001 From: ViacheslavKlimov Date: Fri, 14 Jul 2023 13:14:01 +0300 Subject: [PATCH 01/23] Ability to update isolated processing option in tenant profile --- .../DefaultTbTenantProfileService.java | 9 +- .../server/controller/AbstractWebTest.java | 21 +- .../service/queue/QueueServiceTest.java | 225 ++++++++++++++++++ .../service/ttl/AlarmsCleanUpServiceTest.java | 2 +- .../server/actors/TbActorMailbox.java | 6 +- .../queue/discovery/HashPartitionService.java | 16 +- .../validator/TenantProfileDataValidator.java | 2 - .../profile/tenant-profile.component.ts | 49 +++- 8 files changed, 300 insertions(+), 30 deletions(-) create mode 100644 application/src/test/java/org/thingsboard/server/service/queue/QueueServiceTest.java diff --git a/application/src/main/java/org/thingsboard/server/service/entitiy/tenant/profile/DefaultTbTenantProfileService.java b/application/src/main/java/org/thingsboard/server/service/entitiy/tenant/profile/DefaultTbTenantProfileService.java index 1219a5b929..98385ef8f7 100644 --- a/application/src/main/java/org/thingsboard/server/service/entitiy/tenant/profile/DefaultTbTenantProfileService.java +++ b/application/src/main/java/org/thingsboard/server/service/entitiy/tenant/profile/DefaultTbTenantProfileService.java @@ -44,12 +44,11 @@ public class DefaultTbTenantProfileService extends AbstractTbEntityService imple @Override public TenantProfile save(TenantId tenantId, TenantProfile tenantProfile, TenantProfile oldTenantProfile) throws ThingsboardException { TenantProfile savedTenantProfile = checkNotNull(tenantProfileService.saveTenantProfile(tenantId, tenantProfile)); - if (oldTenantProfile != null && savedTenantProfile.isIsolatedTbRuleEngine()) { - List tenantIds = tenantService.findTenantIdsByTenantProfileId(savedTenantProfile.getId()); - tbQueueService.updateQueuesByTenants(tenantIds, savedTenantProfile, oldTenantProfile); - } - tenantProfileCache.put(savedTenantProfile); + + List tenantIds = tenantService.findTenantIdsByTenantProfileId(savedTenantProfile.getId()); + tbQueueService.updateQueuesByTenants(tenantIds, savedTenantProfile, oldTenantProfile); + tbClusterService.onTenantProfileChange(savedTenantProfile, null); tbClusterService.broadcastEntityStateChangeEvent(TenantId.SYS_TENANT_ID, savedTenantProfile.getId(), tenantProfile.getId() == null ? ComponentLifecycleEvent.CREATED : ComponentLifecycleEvent.UPDATED); diff --git a/application/src/test/java/org/thingsboard/server/controller/AbstractWebTest.java b/application/src/test/java/org/thingsboard/server/controller/AbstractWebTest.java index c7580867e0..449164776a 100644 --- a/application/src/test/java/org/thingsboard/server/controller/AbstractWebTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/AbstractWebTest.java @@ -969,13 +969,20 @@ public abstract class AbstractWebTest extends AbstractInMemoryStorageTest { return (DeviceActorMessageProcessor) ReflectionTestUtils.getField(actor, "processor"); } - protected void updateDefaultTenantProfile(Consumer updater) throws ThingsboardException { - TenantProfile tenantProfile = tenantProfileService.findDefaultTenantProfile(TenantId.SYS_TENANT_ID); - TenantProfileData profileData = tenantProfile.getProfileData(); - DefaultTenantProfileConfiguration profileConfiguration = (DefaultTenantProfileConfiguration) profileData.getConfiguration(); - updater.accept(profileConfiguration); - tenantProfile.setProfileData(profileData); - tbTenantProfileService.save(TenantId.SYS_TENANT_ID, tenantProfile, null); + protected void updateDefaultTenantProfileConfig(Consumer updater) throws ThingsboardException { + updateDefaultTenantProfile(tenantProfile -> { + TenantProfileData profileData = tenantProfile.getProfileData(); + DefaultTenantProfileConfiguration profileConfiguration = (DefaultTenantProfileConfiguration) profileData.getConfiguration(); + updater.accept(profileConfiguration); + tenantProfile.setProfileData(profileData); + }); + } + + protected void updateDefaultTenantProfile(Consumer updater) throws ThingsboardException { + TenantProfile oldTenantProfile = tenantProfileService.findDefaultTenantProfile(TenantId.SYS_TENANT_ID); + TenantProfile tenantProfile = JacksonUtil.clone(oldTenantProfile); + updater.accept(tenantProfile); + tbTenantProfileService.save(TenantId.SYS_TENANT_ID, tenantProfile, oldTenantProfile); } } diff --git a/application/src/test/java/org/thingsboard/server/service/queue/QueueServiceTest.java b/application/src/test/java/org/thingsboard/server/service/queue/QueueServiceTest.java new file mode 100644 index 0000000000..4d2c50add3 --- /dev/null +++ b/application/src/test/java/org/thingsboard/server/service/queue/QueueServiceTest.java @@ -0,0 +1,225 @@ +/** + * Copyright © 2016-2023 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.queue; + +import org.junit.Before; +import org.junit.Test; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.mock.mockito.SpyBean; +import org.thingsboard.server.actors.ActorSystemContext; +import org.thingsboard.server.common.data.DataConstants; +import org.thingsboard.server.common.data.Device; +import org.thingsboard.server.common.data.DeviceProfile; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.queue.ProcessingStrategy; +import org.thingsboard.server.common.data.queue.ProcessingStrategyType; +import org.thingsboard.server.common.data.queue.Queue; +import org.thingsboard.server.common.data.queue.SubmitStrategy; +import org.thingsboard.server.common.data.queue.SubmitStrategyType; +import org.thingsboard.server.common.data.tenant.profile.TenantProfileQueueConfiguration; +import org.thingsboard.server.common.msg.TbMsg; +import org.thingsboard.server.common.msg.queue.QueueToRuleEngineMsg; +import org.thingsboard.server.common.msg.queue.ServiceType; +import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; +import org.thingsboard.server.controller.AbstractControllerTest; +import org.thingsboard.server.dao.queue.QueueService; +import org.thingsboard.server.dao.service.DaoSqlTest; +import org.thingsboard.server.queue.discovery.PartitionService; +import org.thingsboard.server.service.entitiy.queue.TbQueueService; +import org.thingsboard.server.service.profile.TbDeviceProfileCache; + +import java.util.List; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicReference; +import java.util.function.Consumer; +import java.util.function.Predicate; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.awaitility.Awaitility.await; +import static org.mockito.ArgumentMatchers.argThat; +import static org.mockito.Mockito.timeout; +import static org.mockito.Mockito.verify; + +@DaoSqlTest +public class QueueServiceTest extends AbstractControllerTest { + + @SpyBean + private ActorSystemContext actorContext; + @Autowired + private TbQueueService tbQueueService; + @Autowired + private QueueService queueService; + @SpyBean + private PartitionService partitionService; + @Autowired + private TbDeviceProfileCache deviceProfileCache; + + @Before + public void beforeEach() { + Queue mainQueue = queueService.findQueueByTenantIdAndName(TenantId.SYS_TENANT_ID, DataConstants.MAIN_QUEUE_NAME); + if (mainQueue == null) { + mainQueue = new Queue(TenantId.SYS_TENANT_ID, getMainQueueConfig()); + tbQueueService.saveQueue(mainQueue); + } + + Queue hpQueue = queueService.findQueueByTenantIdAndName(TenantId.SYS_TENANT_ID, DataConstants.HP_QUEUE_NAME); + if (hpQueue == null) { + hpQueue = new Queue(TenantId.SYS_TENANT_ID, getHighPriorityQueueConfig()); + tbQueueService.saveQueue(hpQueue); + } + } + + @Test + public void testQueuesUpdateOnTenantProfileUpdate() throws Exception { + loginTenantAdmin(); + DeviceProfile hpQueueProfile = createDeviceProfile("HighPriority profile"); + hpQueueProfile.setDefaultQueueName(DataConstants.HP_QUEUE_NAME); + hpQueueProfile = doPost("/api/deviceProfile", hpQueueProfile, DeviceProfile.class); + Device hpQueueDevice = createDevice("HP", hpQueueProfile.getName(), "HP"); + deviceProfileCache.evict(tenantId, hpQueueProfile.getId()); + + DeviceProfile mainQueueProfile = createDeviceProfile("Main profile"); + mainQueueProfile.setDefaultQueueName(DataConstants.MAIN_QUEUE_NAME); + mainQueueProfile = doPost("/api/deviceProfile", mainQueueProfile, DeviceProfile.class); + Device mainQueueDevice = createDevice("Main", mainQueueProfile.getName(), "Main"); + + verifyUsedQueueAndMessage(DataConstants.HP_QUEUE_NAME, hpQueueDevice.getId(), DataConstants.ATTRIBUTES_UPDATED, () -> { + doPost("/api/plugins/telemetry/DEVICE/" + hpQueueDevice.getId() + "/attributes/SERVER_SCOPE", "{\"test\":123}", String.class); + }, usedTpi -> { + assertThat(usedTpi.getTenantId()).get().isEqualTo(TenantId.SYS_TENANT_ID); + }); + verifyUsedQueueAndMessage(DataConstants.MAIN_QUEUE_NAME, mainQueueDevice.getId(), DataConstants.ATTRIBUTES_UPDATED, () -> { + doPost("/api/plugins/telemetry/DEVICE/" + mainQueueDevice.getId() + "/attributes/SERVER_SCOPE", "{\"test\":123}", String.class); + }, usedTpi -> { + assertThat(usedTpi.getTenantId()).get().isEqualTo(TenantId.SYS_TENANT_ID); + }); + + updateDefaultTenantProfile(tenantProfile -> { + tenantProfile.setIsolatedTbRuleEngine(true); + tenantProfile.getProfileData().setQueueConfiguration(List.of( + getMainQueueConfig() + )); + }); + + verifyUsedQueueAndMessage(DataConstants.MAIN_QUEUE_NAME, mainQueueDevice.getId(), DataConstants.ATTRIBUTES_UPDATED, () -> { + doPost("/api/plugins/telemetry/DEVICE/" + mainQueueDevice.getId() + "/attributes/SERVER_SCOPE", "{\"test\":123}", String.class); + }, usedTpi -> { + assertThat(usedTpi.getTenantId()).get().isEqualTo(tenantId); + }); + verifyUsedQueueAndMessage(DataConstants.HP_QUEUE_NAME, hpQueueDevice.getId(), DataConstants.ATTRIBUTES_UPDATED, () -> { + doPost("/api/plugins/telemetry/DEVICE/" + hpQueueDevice.getId() + "/attributes/SERVER_SCOPE", "{\"test\":123}", String.class); + }, usedTpi -> { + assertThat(usedTpi.getTopic()).endsWith("main"); + assertThat(usedTpi.getTenantId()).get().isEqualTo(tenantId); + }); + + updateDefaultTenantProfile(tenantProfile -> { + tenantProfile.setIsolatedTbRuleEngine(true); + tenantProfile.getProfileData().setQueueConfiguration(List.of( + getMainQueueConfig(), getHighPriorityQueueConfig() + )); + }); + + verifyUsedQueueAndMessage(DataConstants.HP_QUEUE_NAME, hpQueueDevice.getId(), DataConstants.ATTRIBUTES_UPDATED, () -> { + doPost("/api/plugins/telemetry/DEVICE/" + hpQueueDevice.getId() + "/attributes/SERVER_SCOPE", "{\"test\":123}", String.class); + }, usedTpi -> { + assertThat(usedTpi.getTopic()).endsWith("hp"); + assertThat(usedTpi.getTenantId()).get().isEqualTo(tenantId); + }); + verifyUsedQueueAndMessage(DataConstants.MAIN_QUEUE_NAME, mainQueueDevice.getId(), DataConstants.ATTRIBUTES_UPDATED, () -> { + doPost("/api/plugins/telemetry/DEVICE/" + mainQueueDevice.getId() + "/attributes/SERVER_SCOPE", "{\"test\":123}", String.class); + }, usedTpi -> { + assertThat(usedTpi.getTenantId()).get().isEqualTo(tenantId); + }); + } + + private void verifyUsedQueueAndMessage(String queue, EntityId entityId, String msgType, Runnable action, Consumer tpiAssert) { + await().atMost(15, TimeUnit.SECONDS) + .untilAsserted(() -> { + TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_RULE_ENGINE, queue, tenantId, entityId); + tpiAssert.accept(tpi); + }); + action.run(); + TbMsg tbMsg = awaitTbMsg(msg -> msg.getOriginator().equals(entityId) + && msg.getType().equals(msgType), 10000); + assertThat(tbMsg.getQueueName()).isEqualTo(queue); + + TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_RULE_ENGINE, queue, tenantId, entityId); + tpiAssert.accept(tpi); + } + + protected TbMsg awaitTbMsg(Predicate predicate, int timeoutMillis) { + AtomicReference tbMsgCaptor = new AtomicReference<>(); + verify(actorContext, timeout(timeoutMillis).atLeastOnce()).tell(argThat(actorMsg -> { + if (!(actorMsg instanceof QueueToRuleEngineMsg)) { + return false; + } + TbMsg tbMsg = ((QueueToRuleEngineMsg) actorMsg).getMsg(); + if (predicate.test(tbMsg)) { + tbMsgCaptor.set(tbMsg); + return true; + } + return false; + })); + return tbMsgCaptor.get(); + } + + private TenantProfileQueueConfiguration getHighPriorityQueueConfig() { + TenantProfileQueueConfiguration hpQueueConfig = new TenantProfileQueueConfiguration(); + hpQueueConfig.setName(DataConstants.HP_QUEUE_NAME); + hpQueueConfig.setTopic(DataConstants.HP_QUEUE_TOPIC); + hpQueueConfig.setPollInterval(25); + hpQueueConfig.setPartitions(10); + hpQueueConfig.setConsumerPerPartition(true); + hpQueueConfig.setPackProcessingTimeout(2000); + SubmitStrategy highPriorityQueueSubmitStrategy = new SubmitStrategy(); + highPriorityQueueSubmitStrategy.setType(SubmitStrategyType.BURST); + highPriorityQueueSubmitStrategy.setBatchSize(100); + hpQueueConfig.setSubmitStrategy(highPriorityQueueSubmitStrategy); + ProcessingStrategy highPriorityQueueProcessingStrategy = new ProcessingStrategy(); + highPriorityQueueProcessingStrategy.setType(ProcessingStrategyType.RETRY_FAILED_AND_TIMED_OUT); + highPriorityQueueProcessingStrategy.setRetries(0); + highPriorityQueueProcessingStrategy.setFailurePercentage(0); + highPriorityQueueProcessingStrategy.setPauseBetweenRetries(5); + highPriorityQueueProcessingStrategy.setMaxPauseBetweenRetries(5); + hpQueueConfig.setProcessingStrategy(highPriorityQueueProcessingStrategy); + return hpQueueConfig; + } + + private TenantProfileQueueConfiguration getMainQueueConfig() { + TenantProfileQueueConfiguration mainQueue = new TenantProfileQueueConfiguration(); + mainQueue.setName(DataConstants.MAIN_QUEUE_NAME); + mainQueue.setTopic(DataConstants.MAIN_QUEUE_TOPIC); + mainQueue.setPollInterval(25); + mainQueue.setPartitions(10); + mainQueue.setConsumerPerPartition(true); + mainQueue.setPackProcessingTimeout(2000); + SubmitStrategy mainQueueSubmitStrategy = new SubmitStrategy(); + mainQueueSubmitStrategy.setType(SubmitStrategyType.BURST); + mainQueueSubmitStrategy.setBatchSize(1000); + mainQueue.setSubmitStrategy(mainQueueSubmitStrategy); + ProcessingStrategy mainQueueProcessingStrategy = new ProcessingStrategy(); + mainQueueProcessingStrategy.setType(ProcessingStrategyType.SKIP_ALL_FAILURES); + mainQueueProcessingStrategy.setRetries(3); + mainQueueProcessingStrategy.setFailurePercentage(0); + mainQueueProcessingStrategy.setPauseBetweenRetries(3); + mainQueueProcessingStrategy.setMaxPauseBetweenRetries(3); + mainQueue.setProcessingStrategy(mainQueueProcessingStrategy); + return mainQueue; + } + +} diff --git a/application/src/test/java/org/thingsboard/server/service/ttl/AlarmsCleanUpServiceTest.java b/application/src/test/java/org/thingsboard/server/service/ttl/AlarmsCleanUpServiceTest.java index 6cabff788e..09e78df9ff 100644 --- a/application/src/test/java/org/thingsboard/server/service/ttl/AlarmsCleanUpServiceTest.java +++ b/application/src/test/java/org/thingsboard/server/service/ttl/AlarmsCleanUpServiceTest.java @@ -67,7 +67,7 @@ public class AlarmsCleanUpServiceTest extends AbstractControllerTest { @Test public void testAlarmsCleanUp() throws Exception { int ttlDays = 1; - updateDefaultTenantProfile(profileConfiguration -> { + updateDefaultTenantProfileConfig(profileConfiguration -> { profileConfiguration.setAlarmsTtlDays(ttlDays); }); diff --git a/common/actor/src/main/java/org/thingsboard/server/actors/TbActorMailbox.java b/common/actor/src/main/java/org/thingsboard/server/actors/TbActorMailbox.java index ad1604f7b0..857c45ad84 100644 --- a/common/actor/src/main/java/org/thingsboard/server/actors/TbActorMailbox.java +++ b/common/actor/src/main/java/org/thingsboard/server/actors/TbActorMailbox.java @@ -15,7 +15,8 @@ */ package org.thingsboard.server.actors; -import lombok.Data; +import lombok.Getter; +import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.msg.MsgType; @@ -31,7 +32,8 @@ import java.util.function.Predicate; import java.util.function.Supplier; @Slf4j -@Data +@Getter +@RequiredArgsConstructor public final class TbActorMailbox implements TbActorCtx { private static final boolean HIGH_PRIORITY = true; private static final boolean NORMAL_PRIORITY = false; diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/HashPartitionService.java b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/HashPartitionService.java index 14fb0369ee..f56233b144 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/HashPartitionService.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/HashPartitionService.java @@ -165,6 +165,9 @@ public class HashPartitionService implements PartitionService { partitionTopicsMap.put(queueKey, queueUpdateMsg.getQueueTopic()); partitionSizesMap.put(queueKey, queueUpdateMsg.getPartitions()); myPartitions.remove(queueKey); + if (!tenantId.isSysTenantId()) { + tenantRoutingInfoMap.remove(tenantId); + } } @Override @@ -358,16 +361,9 @@ public class HashPartitionService implements PartitionService { if (TenantId.SYS_TENANT_ID.equals(tenantId)) { return false; } - TenantRoutingInfo routingInfo = tenantRoutingInfoMap.get(tenantId); - if (routingInfo == null) { - synchronized (tenantRoutingInfoMap) { - routingInfo = tenantRoutingInfoMap.get(tenantId); - if (routingInfo == null) { - routingInfo = tenantRoutingInfoService.getRoutingInfo(tenantId); - tenantRoutingInfoMap.put(tenantId, routingInfo); - } - } - } + TenantRoutingInfo routingInfo = tenantRoutingInfoMap.computeIfAbsent(tenantId, k -> { + return tenantRoutingInfoService.getRoutingInfo(tenantId); + }); if (routingInfo == null) { throw new TenantNotFoundException(tenantId); } diff --git a/dao/src/main/java/org/thingsboard/server/dao/service/validator/TenantProfileDataValidator.java b/dao/src/main/java/org/thingsboard/server/dao/service/validator/TenantProfileDataValidator.java index 0477ccf32f..aa166ba12e 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/service/validator/TenantProfileDataValidator.java +++ b/dao/src/main/java/org/thingsboard/server/dao/service/validator/TenantProfileDataValidator.java @@ -99,8 +99,6 @@ public class TenantProfileDataValidator extends DataValidator { TenantProfile old = tenantProfileDao.findById(TenantId.SYS_TENANT_ID, tenantProfile.getId().getId()); if (old == null) { throw new DataValidationException("Can't update non existing tenant profile!"); - } else if (old.isIsolatedTbRuleEngine() != tenantProfile.isIsolatedTbRuleEngine()) { - throw new DataValidationException("Can't update isolatedTbRuleEngine property!"); } return old; } diff --git a/ui-ngx/src/app/modules/home/components/profile/tenant-profile.component.ts b/ui-ngx/src/app/modules/home/components/profile/tenant-profile.component.ts index 0257fb35af..2aafa97d99 100644 --- a/ui-ngx/src/app/modules/home/components/profile/tenant-profile.component.ts +++ b/ui-ngx/src/app/modules/home/components/profile/tenant-profile.component.ts @@ -76,6 +76,52 @@ export class TenantProfileComponent extends EntityComponent { additionalInfo: { description: '' } + }, + { + id: guid(), + name: 'HighPriority', + topic: 'tb_rule_engine.hp', + pollInterval: 25, + partitions: 10, + consumerPerPartition: true, + packProcessingTimeout: 2000, + submitStrategy: { + type: 'BURST', + batchSize: 100 + }, + processingStrategy: { + type: 'RETRY_FAILED_AND_TIMED_OUT', + retries: 0, + failurePercentage: 0, + pauseBetweenRetries: 5, + maxPauseBetweenRetries: 5 + }, + additionalInfo: { + description: '' + } + }, + { + id: guid(), + name: 'SequentialByOriginator', + topic: 'tb_rule_engine.sq', + pollInterval: 25, + partitions: 10, + consumerPerPartition: true, + packProcessingTimeout: 2000, + submitStrategy: { + type: 'SEQUENTIAL_BY_ORIGINATOR', + batchSize: 100 + }, + processingStrategy: { + type: 'RETRY_FAILED_AND_TIMED_OUT', + retries: 3, + failurePercentage: 0, + pauseBetweenRetries: 5, + maxPauseBetweenRetries: 5 + }, + additionalInfo: { + description: '' + } } ]; const formGroup = this.fb.group( @@ -118,9 +164,6 @@ export class TenantProfileComponent extends EntityComponent { if (this.entityForm) { if (this.isEditValue) { this.entityForm.enable({emitEvent: false}); - if (!this.isAdd) { - this.entityForm.get('isolatedTbRuleEngine').disable({emitEvent: false}); - } } else { this.entityForm.disable({emitEvent: false}); } From c1a07db64a4534e77410421d60bba7e429689b21 Mon Sep 17 00:00:00 2001 From: ViacheslavKlimov Date: Wed, 19 Jul 2023 13:39:44 +0300 Subject: [PATCH 02/23] Fix tenant profile update handling --- .../tenant/profile/DefaultTbTenantProfileService.java | 7 +++---- .../service/DefaultTransportTenantProfileCache.java | 1 + 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/entitiy/tenant/profile/DefaultTbTenantProfileService.java b/application/src/main/java/org/thingsboard/server/service/entitiy/tenant/profile/DefaultTbTenantProfileService.java index 98385ef8f7..15bcc2e040 100644 --- a/application/src/main/java/org/thingsboard/server/service/entitiy/tenant/profile/DefaultTbTenantProfileService.java +++ b/application/src/main/java/org/thingsboard/server/service/entitiy/tenant/profile/DefaultTbTenantProfileService.java @@ -45,14 +45,13 @@ public class DefaultTbTenantProfileService extends AbstractTbEntityService imple public TenantProfile save(TenantId tenantId, TenantProfile tenantProfile, TenantProfile oldTenantProfile) throws ThingsboardException { TenantProfile savedTenantProfile = checkNotNull(tenantProfileService.saveTenantProfile(tenantId, tenantProfile)); tenantProfileCache.put(savedTenantProfile); - - List tenantIds = tenantService.findTenantIdsByTenantProfileId(savedTenantProfile.getId()); - tbQueueService.updateQueuesByTenants(tenantIds, savedTenantProfile, oldTenantProfile); - tbClusterService.onTenantProfileChange(savedTenantProfile, null); tbClusterService.broadcastEntityStateChangeEvent(TenantId.SYS_TENANT_ID, savedTenantProfile.getId(), tenantProfile.getId() == null ? ComponentLifecycleEvent.CREATED : ComponentLifecycleEvent.UPDATED); + List tenantIds = tenantService.findTenantIdsByTenantProfileId(savedTenantProfile.getId()); + tbQueueService.updateQueuesByTenants(tenantIds, savedTenantProfile, oldTenantProfile); + return savedTenantProfile; } diff --git a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/DefaultTransportTenantProfileCache.java b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/DefaultTransportTenantProfileCache.java index d73e445516..12f89f29e6 100644 --- a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/DefaultTransportTenantProfileCache.java +++ b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/DefaultTransportTenantProfileCache.java @@ -82,6 +82,7 @@ public class DefaultTransportTenantProfileCache implements TransportTenantProfil if (profileOpt.isPresent()) { TenantProfile newProfile = profileOpt.get(); log.trace("[{}] put: {}", newProfile.getId(), newProfile); + profiles.put(newProfile.getId(), newProfile); Set affectedTenants = tenantProfileIds.get(newProfile.getId()); return new TenantProfileUpdateResult(newProfile, affectedTenants != null ? affectedTenants : Collections.emptySet()); } else { From 2fef3858a3bebcfb7bbc550d092c4f73431b05f3 Mon Sep 17 00:00:00 2001 From: ViacheslavKlimov Date: Wed, 19 Jul 2023 13:48:07 +0300 Subject: [PATCH 03/23] Fix queue poll duration config ignored; 1 partition by default for isolated queues --- .../queue/DefaultTbRuleEngineConsumerService.java | 2 +- .../components/profile/tenant-profile.component.ts | 12 ++++++------ 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbRuleEngineConsumerService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbRuleEngineConsumerService.java index f8f6a7d25f..3d02a1f60e 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbRuleEngineConsumerService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbRuleEngineConsumerService.java @@ -266,7 +266,7 @@ public class DefaultTbRuleEngineConsumerService extends AbstractConsumerService< updateCurrentThreadName(threadSuffix); while (!stopped && !consumer.isStopped()) { try { - List> msgs = consumer.poll(pollDuration); + List> msgs = consumer.poll(configuration.getPollInterval()); if (msgs.isEmpty()) { continue; } diff --git a/ui-ngx/src/app/modules/home/components/profile/tenant-profile.component.ts b/ui-ngx/src/app/modules/home/components/profile/tenant-profile.component.ts index 2aafa97d99..e96a237033 100644 --- a/ui-ngx/src/app/modules/home/components/profile/tenant-profile.component.ts +++ b/ui-ngx/src/app/modules/home/components/profile/tenant-profile.component.ts @@ -56,10 +56,10 @@ export class TenantProfileComponent extends EntityComponent { const mainQueue = [ { id: guid(), - consumerPerPartition: true, + consumerPerPartition: false, name: 'Main', packProcessingTimeout: 2000, - partitions: 10, + partitions: 1, pollInterval: 25, processingStrategy: { failurePercentage: 0, @@ -82,8 +82,8 @@ export class TenantProfileComponent extends EntityComponent { name: 'HighPriority', topic: 'tb_rule_engine.hp', pollInterval: 25, - partitions: 10, - consumerPerPartition: true, + partitions: 1, + consumerPerPartition: false, packProcessingTimeout: 2000, submitStrategy: { type: 'BURST', @@ -105,8 +105,8 @@ export class TenantProfileComponent extends EntityComponent { name: 'SequentialByOriginator', topic: 'tb_rule_engine.sq', pollInterval: 25, - partitions: 10, - consumerPerPartition: true, + partitions: 1, + consumerPerPartition: false, packProcessingTimeout: 2000, submitStrategy: { type: 'SEQUENTIAL_BY_ORIGINATOR', From bfd8ff934f8e626947374752d477761fc5e96782 Mon Sep 17 00:00:00 2001 From: ViacheslavKlimov Date: Wed, 19 Jul 2023 16:39:41 +0300 Subject: [PATCH 04/23] Use system queue with same name instead of Main when missing --- .../server/queue/discovery/HashPartitionService.java | 6 +++++- .../dao/service/BaseTenantProfileServiceTest.java | 11 ----------- 2 files changed, 5 insertions(+), 12 deletions(-) diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/HashPartitionService.java b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/HashPartitionService.java index f56233b144..e2312fa7cd 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/HashPartitionService.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/HashPartitionService.java @@ -186,7 +186,8 @@ public class HashPartitionService implements PartitionService { TenantId isolatedOrSystemTenantId = getIsolatedOrSystemTenantId(serviceType, tenantId); QueueKey queueKey = new QueueKey(serviceType, queueName, isolatedOrSystemTenantId); if (!partitionSizesMap.containsKey(queueKey)) { - queueKey = new QueueKey(serviceType, isolatedOrSystemTenantId); + // TODO: fallback to Main in case no system queue + queueKey = new QueueKey(serviceType, queueName, TenantId.SYS_TENANT_ID); } return resolve(queueKey, entityId); } @@ -207,6 +208,9 @@ public class HashPartitionService implements PartitionService { .putLong(entityId.getId().getLeastSignificantBits()).hash().asInt(); Integer partitionSize = partitionSizesMap.get(queueKey); + // if (partitionSize == null) { +// throw new IllegalStateException("Can't get partition ") +// } int partition = Math.abs(hash % partitionSize); return buildTopicPartitionInfo(queueKey, partition); diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/BaseTenantProfileServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/BaseTenantProfileServiceTest.java index 6d41da2965..4129a991cc 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/BaseTenantProfileServiceTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/BaseTenantProfileServiceTest.java @@ -187,17 +187,6 @@ public abstract class BaseTenantProfileServiceTest extends AbstractServiceTest { }); } - @Test - public void testSaveSameTenantProfileWithDifferentIsolatedTbRuleEngine() { - TenantProfile tenantProfile = this.createTenantProfile("Tenant Profile"); - TenantProfile savedTenantProfile = tenantProfileService.saveTenantProfile(TenantId.SYS_TENANT_ID, tenantProfile); - savedTenantProfile.setIsolatedTbRuleEngine(true); - addMainQueueConfig(savedTenantProfile); - Assertions.assertThrows(DataValidationException.class, () -> { - tenantProfileService.saveTenantProfile(TenantId.SYS_TENANT_ID, savedTenantProfile); - }); - } - @Test public void testDeleteTenantProfileWithExistingTenant() { TenantProfile tenantProfile = this.createTenantProfile("Tenant Profile"); From 1ee3c24532f5ee6b3183ae8c6bc8c69373feaaff Mon Sep 17 00:00:00 2001 From: ViacheslavKlimov Date: Wed, 19 Jul 2023 17:37:49 +0300 Subject: [PATCH 05/23] Refactor HashPartitionService.resolve(..) --- .../queue/discovery/HashPartitionService.java | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/HashPartitionService.java b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/HashPartitionService.java index e2312fa7cd..65f750f830 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/HashPartitionService.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/HashPartitionService.java @@ -48,6 +48,8 @@ import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; import java.util.stream.Collectors; +import static org.thingsboard.server.common.data.DataConstants.MAIN_QUEUE_NAME; + @Service @Slf4j public class HashPartitionService implements PartitionService { @@ -186,8 +188,15 @@ public class HashPartitionService implements PartitionService { TenantId isolatedOrSystemTenantId = getIsolatedOrSystemTenantId(serviceType, tenantId); QueueKey queueKey = new QueueKey(serviceType, queueName, isolatedOrSystemTenantId); if (!partitionSizesMap.containsKey(queueKey)) { - // TODO: fallback to Main in case no system queue - queueKey = new QueueKey(serviceType, queueName, TenantId.SYS_TENANT_ID); + if (isolatedOrSystemTenantId.isSysTenantId()) { + queueKey = new QueueKey(serviceType, TenantId.SYS_TENANT_ID); + } else { + queueKey = new QueueKey(serviceType, queueName, TenantId.SYS_TENANT_ID); + if (!MAIN_QUEUE_NAME.equals(queueName) && !partitionSizesMap.containsKey(queueKey)) { + queueKey = new QueueKey(serviceType, TenantId.SYS_TENANT_ID); + } + log.warn("Using queue {} instead of isolated {}", queueKey, queueName); + } } return resolve(queueKey, entityId); } @@ -208,9 +217,9 @@ public class HashPartitionService implements PartitionService { .putLong(entityId.getId().getLeastSignificantBits()).hash().asInt(); Integer partitionSize = partitionSizesMap.get(queueKey); - // if (partitionSize == null) { -// throw new IllegalStateException("Can't get partition ") -// } + if (partitionSize == null) { + throw new IllegalStateException("Partitions info for queue " + queueKey + " is missing"); + } int partition = Math.abs(hash % partitionSize); return buildTopicPartitionInfo(queueKey, partition); From 3b86c8c1f50b631ac8a28f031c3d63962d8f510e Mon Sep 17 00:00:00 2001 From: ViacheslavKlimov Date: Thu, 20 Jul 2023 13:34:06 +0300 Subject: [PATCH 06/23] Refactor HashPartitionService.resolve(..) --- .../server/queue/discovery/HashPartitionService.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/HashPartitionService.java b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/HashPartitionService.java index 65f750f830..31f517c236 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/HashPartitionService.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/HashPartitionService.java @@ -186,6 +186,9 @@ public class HashPartitionService implements PartitionService { @Override public TopicPartitionInfo resolve(ServiceType serviceType, String queueName, TenantId tenantId, EntityId entityId) { TenantId isolatedOrSystemTenantId = getIsolatedOrSystemTenantId(serviceType, tenantId); + if (queueName == null) { + queueName = MAIN_QUEUE_NAME; + } QueueKey queueKey = new QueueKey(serviceType, queueName, isolatedOrSystemTenantId); if (!partitionSizesMap.containsKey(queueKey)) { if (isolatedOrSystemTenantId.isSysTenantId()) { From 829433151207a3aa0cc6d4cd81238f92ce87fd62 Mon Sep 17 00:00:00 2001 From: ViacheslavKlimov Date: Fri, 21 Jul 2023 11:10:39 +0300 Subject: [PATCH 07/23] Configurable topic deletion delay --- .../service/entitiy/queue/DefaultTbQueueService.java | 12 +++++++----- application/src/main/resources/thingsboard.yml | 2 ++ 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/entitiy/queue/DefaultTbQueueService.java b/application/src/main/java/org/thingsboard/server/service/entitiy/queue/DefaultTbQueueService.java index 63e11aeb7a..abf7b694d9 100644 --- a/application/src/main/java/org/thingsboard/server/service/entitiy/queue/DefaultTbQueueService.java +++ b/application/src/main/java/org/thingsboard/server/service/entitiy/queue/DefaultTbQueueService.java @@ -17,6 +17,7 @@ package org.thingsboard.server.service.entitiy.queue; import lombok.AllArgsConstructor; import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Service; import org.thingsboard.server.cluster.TbClusterService; import org.thingsboard.server.common.data.TenantProfile; @@ -44,13 +45,15 @@ import java.util.stream.Collectors; @TbCoreComponent @AllArgsConstructor public class DefaultTbQueueService extends AbstractTbEntityService implements TbQueueService { - private static final long DELETE_DELAY = 30; private final QueueService queueService; private final TbClusterService tbClusterService; private final TbQueueAdmin tbQueueAdmin; private final SchedulerComponent scheduler; + @Value("${queue.rule-engine.topic_deletion_delay:60}") + private int topicDeletionDelay; + @Override public Queue saveQueue(Queue queue) { boolean create = queue.getId() == null; @@ -119,10 +122,9 @@ public class DefaultTbQueueService extends AbstractTbEntityService implements Tb for (int i = currentPartitions; i < oldPartitions; i++) { String fullTopicName = new TopicPartitionInfo(queue.getTopic(), queue.getTenantId(), i, false).getFullTopicName(); log.info("Removed partition [{}]", fullTopicName); - tbQueueAdmin.deleteTopic( - fullTopicName); + tbQueueAdmin.deleteTopic(fullTopicName); } - }, DELETE_DELAY, TimeUnit.SECONDS); + }, topicDeletionDelay, TimeUnit.SECONDS); } } else if (!oldQueue.equals(queue)) { tbClusterService.onQueueChange(queue); @@ -144,7 +146,7 @@ public class DefaultTbQueueService extends AbstractTbEntityService implements Tb log.error("Failed to delete queue [{}]", fullTopicName); } } - }, DELETE_DELAY, TimeUnit.SECONDS); + }, topicDeletionDelay, TimeUnit.SECONDS); notificationEntityService.notifySendMsgToEdgeService(queue.getTenantId(), queue.getId(), EdgeEventActionType.DELETED); } diff --git a/application/src/main/resources/thingsboard.yml b/application/src/main/resources/thingsboard.yml index 589540096c..2c09221148 100644 --- a/application/src/main/resources/thingsboard.yml +++ b/application/src/main/resources/thingsboard.yml @@ -1232,6 +1232,8 @@ queue: failure-percentage: "${TB_QUEUE_RE_SQ_PROCESSING_STRATEGY_FAILURE_PERCENTAGE:0}" # Skip retry if failures or timeouts are less then X percentage of messages; pause-between-retries: "${TB_QUEUE_RE_SQ_PROCESSING_STRATEGY_RETRY_PAUSE:5}" # Time in seconds to wait in consumer thread before retries; max-pause-between-retries: "${TB_QUEUE_RE_SQ_PROCESSING_STRATEGY_MAX_RETRY_PAUSE:5}" # Max allowed time in seconds for pause between retries. + # Delay between Queue update/delete and actual topic deletion. The delay is for Rule Engines to have time to unsubscribe from the topics, and for other services to stop publishing + topic_deletion_delay: "${TB_QUEUE_RULE_ENGINE_TOPIC_DELETION_DELAY_SECS:60}" transport: # For high priority notifications that require minimum latency and processing time notifications_topic: "${TB_QUEUE_TRANSPORT_NOTIFICATIONS_TOPIC:tb_transport.notifications}" From b2fe451bd2cfcdda6158c1c6d8028271d503e505 Mon Sep 17 00:00:00 2001 From: ViacheslavKlimov Date: Fri, 21 Jul 2023 13:20:05 +0300 Subject: [PATCH 08/23] Fix DefaultTbQueueService init --- .../server/service/entitiy/queue/DefaultTbQueueService.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/entitiy/queue/DefaultTbQueueService.java b/application/src/main/java/org/thingsboard/server/service/entitiy/queue/DefaultTbQueueService.java index abf7b694d9..0166a425d3 100644 --- a/application/src/main/java/org/thingsboard/server/service/entitiy/queue/DefaultTbQueueService.java +++ b/application/src/main/java/org/thingsboard/server/service/entitiy/queue/DefaultTbQueueService.java @@ -15,7 +15,7 @@ */ package org.thingsboard.server.service.entitiy.queue; -import lombok.AllArgsConstructor; +import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Service; @@ -43,7 +43,7 @@ import java.util.stream.Collectors; @Slf4j @Service @TbCoreComponent -@AllArgsConstructor +@RequiredArgsConstructor public class DefaultTbQueueService extends AbstractTbEntityService implements TbQueueService { private final QueueService queueService; From fa61783ac64110f4cb3123cb2975dd4e223674a3 Mon Sep 17 00:00:00 2001 From: ViacheslavKlimov Date: Tue, 25 Jul 2023 10:51:22 +0300 Subject: [PATCH 09/23] Log topics list on unsubscribe --- application/src/main/resources/thingsboard.yml | 2 +- .../server/queue/common/AbstractTbQueueConsumerTemplate.java | 4 +++- .../server/queue/kafka/TbKafkaConsumerTemplate.java | 1 - 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/application/src/main/resources/thingsboard.yml b/application/src/main/resources/thingsboard.yml index 2c09221148..163c325f52 100644 --- a/application/src/main/resources/thingsboard.yml +++ b/application/src/main/resources/thingsboard.yml @@ -1233,7 +1233,7 @@ queue: pause-between-retries: "${TB_QUEUE_RE_SQ_PROCESSING_STRATEGY_RETRY_PAUSE:5}" # Time in seconds to wait in consumer thread before retries; max-pause-between-retries: "${TB_QUEUE_RE_SQ_PROCESSING_STRATEGY_MAX_RETRY_PAUSE:5}" # Max allowed time in seconds for pause between retries. # Delay between Queue update/delete and actual topic deletion. The delay is for Rule Engines to have time to unsubscribe from the topics, and for other services to stop publishing - topic_deletion_delay: "${TB_QUEUE_RULE_ENGINE_TOPIC_DELETION_DELAY_SECS:60}" + topic_deletion_delay: "${TB_QUEUE_RULE_ENGINE_TOPIC_DELETION_DELAY_SECS:180}" transport: # For high priority notifications that require minimum latency and processing time notifications_topic: "${TB_QUEUE_TRANSPORT_NOTIFICATIONS_TOPIC:tb_transport.notifications}" diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/common/AbstractTbQueueConsumerTemplate.java b/common/queue/src/main/java/org/thingsboard/server/queue/common/AbstractTbQueueConsumerTemplate.java index 86146dabd1..a152e0c466 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/common/AbstractTbQueueConsumerTemplate.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/common/AbstractTbQueueConsumerTemplate.java @@ -162,7 +162,9 @@ public abstract class AbstractTbQueueConsumerTemplate i @Override public void unsubscribe() { - log.info("unsubscribe topic and stop consumer {}", getTopic()); + log.info("Unsubscribing from topics and stopping consumer for topics {}", partitions.stream() + .map(TopicPartitionInfo::getFullTopicName) + .collect(Collectors.joining(", "))); stopped = true; consumerLock.lock(); try { diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaConsumerTemplate.java b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaConsumerTemplate.java index c17a563d46..00bb7aa541 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaConsumerTemplate.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaConsumerTemplate.java @@ -114,7 +114,6 @@ public class TbKafkaConsumerTemplate extends AbstractTbQue @Override protected void doUnsubscribe() { - log.info("unsubscribe topic and close consumer for topic {}", getTopic()); if (consumer != null) { consumer.unsubscribe(); consumer.close(); From 6db3171ffb499323accd772b42e82ba76446149f Mon Sep 17 00:00:00 2001 From: ViacheslavKlimov Date: Tue, 25 Jul 2023 15:27:54 +0300 Subject: [PATCH 10/23] Refactoring --- .../server/controller/QueueController.java | 1 - .../DefaultTbRuleEngineConsumerService.java | 2 +- .../queue/TbRuleEngineConsumerStats.java | 13 +- .../server/controller/AbstractWebTest.java | 1 + .../controller/BaseTenantControllerTest.java | 170 ++++++++++++- .../BaseTenantProfileControllerTest.java | 17 -- .../notification/NotificationRuleApiTest.java | 4 +- .../service/queue/QueueServiceTest.java | 225 ------------------ .../queue/discovery/HashPartitionService.java | 2 +- .../profile/tenant-profile.component.ts | 12 +- 10 files changed, 181 insertions(+), 266 deletions(-) delete mode 100644 application/src/test/java/org/thingsboard/server/service/queue/QueueServiceTest.java diff --git a/application/src/main/java/org/thingsboard/server/controller/QueueController.java b/application/src/main/java/org/thingsboard/server/controller/QueueController.java index 0e79ae7955..faeea1a4aa 100644 --- a/application/src/main/java/org/thingsboard/server/controller/QueueController.java +++ b/application/src/main/java/org/thingsboard/server/controller/QueueController.java @@ -126,7 +126,6 @@ public class QueueController extends BaseController { @PreAuthorize("hasAnyAuthority('SYS_ADMIN')") @RequestMapping(value = "/queues", params = {"serviceType"}, method = RequestMethod.POST) @ResponseBody - public Queue saveQueue(@ApiParam(value = "A JSON value representing the queue.") @RequestBody Queue queue, @ApiParam(value = QUEUE_SERVICE_TYPE_DESCRIPTION, allowableValues = QUEUE_SERVICE_TYPE_ALLOWABLE_VALUES, required = true) diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbRuleEngineConsumerService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbRuleEngineConsumerService.java index 3d02a1f60e..598b41035f 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbRuleEngineConsumerService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbRuleEngineConsumerService.java @@ -152,7 +152,7 @@ public class DefaultTbRuleEngineConsumerService extends AbstractConsumerService< private void initConsumer(Queue configuration) { QueueKey queueKey = new QueueKey(ServiceType.TB_RULE_ENGINE, configuration); consumerConfigurations.putIfAbsent(queueKey, configuration); - consumerStats.putIfAbsent(queueKey, new TbRuleEngineConsumerStats(configuration.getName(), statsFactory)); + consumerStats.putIfAbsent(queueKey, new TbRuleEngineConsumerStats(configuration, statsFactory)); if (!configuration.isConsumerPerPartition()) { consumers.computeIfAbsent(queueKey, queueName -> tbRuleEngineQueueFactory.createToRuleEngineMsgConsumer(configuration)); } else { diff --git a/application/src/main/java/org/thingsboard/server/service/queue/TbRuleEngineConsumerStats.java b/application/src/main/java/org/thingsboard/server/service/queue/TbRuleEngineConsumerStats.java index 77e805eb62..2904c299ce 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/TbRuleEngineConsumerStats.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/TbRuleEngineConsumerStats.java @@ -18,6 +18,7 @@ package org.thingsboard.server.service.queue; import io.micrometer.core.instrument.Timer; import lombok.extern.slf4j.Slf4j; import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.queue.Queue; import org.thingsboard.server.common.msg.queue.RuleEngineException; import org.thingsboard.server.common.stats.StatsCounter; import org.thingsboard.server.common.stats.StatsFactory; @@ -63,9 +64,11 @@ public class TbRuleEngineConsumerStats { private final ConcurrentMap tenantExceptions = new ConcurrentHashMap<>(); private final String queueName; + private final TenantId tenantId; - public TbRuleEngineConsumerStats(String queueName, StatsFactory statsFactory) { - this.queueName = queueName; + public TbRuleEngineConsumerStats(Queue queue, StatsFactory statsFactory) { + this.queueName = queue.getName(); + this.tenantId = queue.getTenantId(); this.statsFactory = statsFactory; String statsKey = StatsType.RULE_ENGINE.getName() + "." + queueName; @@ -156,7 +159,11 @@ public class TbRuleEngineConsumerStats { counters.forEach(counter -> { stats.append(counter.getName()).append(" = [").append(counter.get()).append("] "); }); - log.info("[{}] Stats: {}", queueName, stats); + if (tenantId.isSysTenantId()) { + log.info("[{}] Stats: {}", queueName, stats); + } else { + log.info("[{}][{}] Stats: {}", queueName, tenantId, stats); + } } } diff --git a/application/src/test/java/org/thingsboard/server/controller/AbstractWebTest.java b/application/src/test/java/org/thingsboard/server/controller/AbstractWebTest.java index 449164776a..d3a77e28d1 100644 --- a/application/src/test/java/org/thingsboard/server/controller/AbstractWebTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/AbstractWebTest.java @@ -57,6 +57,7 @@ import org.springframework.test.web.servlet.request.MockHttpServletRequestBuilde import org.springframework.util.LinkedMultiValueMap; import org.springframework.util.MultiValueMap; import org.springframework.web.context.WebApplicationContext; +import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.rule.engine.api.MailService; import org.thingsboard.server.actors.DefaultTbActorSystem; import org.thingsboard.server.actors.TbActorId; diff --git a/application/src/test/java/org/thingsboard/server/controller/BaseTenantControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/BaseTenantControllerTest.java index 2e42009e5b..85e3fa35bd 100644 --- a/application/src/test/java/org/thingsboard/server/controller/BaseTenantControllerTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/BaseTenantControllerTest.java @@ -27,15 +27,20 @@ import org.junit.Before; import org.junit.Test; import org.mockito.ArgumentMatcher; import org.mockito.Mockito; +import org.springframework.boot.test.mock.mockito.SpyBean; import org.springframework.test.context.TestPropertySource; import org.springframework.test.web.servlet.ResultActions; import org.thingsboard.common.util.ThingsBoardExecutors; +import org.thingsboard.server.actors.ActorSystemContext; import org.thingsboard.server.common.data.DataConstants; +import org.thingsboard.server.common.data.Device; +import org.thingsboard.server.common.data.DeviceProfile; import org.thingsboard.server.common.data.StringUtils; import org.thingsboard.server.common.data.Tenant; import org.thingsboard.server.common.data.TenantInfo; import org.thingsboard.server.common.data.TenantProfile; import org.thingsboard.server.common.data.User; +import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; @@ -49,6 +54,12 @@ import org.thingsboard.server.common.data.security.Authority; import org.thingsboard.server.common.data.tenant.profile.DefaultTenantProfileConfiguration; import org.thingsboard.server.common.data.tenant.profile.TenantProfileData; import org.thingsboard.server.common.data.tenant.profile.TenantProfileQueueConfiguration; +import org.thingsboard.server.common.msg.TbMsg; +import org.thingsboard.server.common.msg.queue.QueueToRuleEngineMsg; +import org.thingsboard.server.common.msg.queue.ServiceType; +import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; +import org.thingsboard.server.dao.service.DaoSqlTest; +import org.thingsboard.server.queue.discovery.PartitionService; import java.util.ArrayList; import java.util.Comparator; @@ -57,12 +68,19 @@ import java.util.List; import java.util.Map; import java.util.Random; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicReference; +import java.util.function.Consumer; +import java.util.function.Predicate; import java.util.stream.Collectors; import static org.assertj.core.api.Assertions.assertThat; +import static org.awaitility.Awaitility.await; import static org.hamcrest.Matchers.containsString; +import static org.mockito.ArgumentMatchers.argThat; import static org.mockito.Mockito.never; +import static org.mockito.Mockito.timeout; import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; @TestPropertySource(properties = { @@ -78,6 +96,11 @@ public abstract class BaseTenantControllerTest extends AbstractControllerTest { ListeningExecutorService executor; + @SpyBean + private PartitionService partitionService; + @SpyBean + private ActorSystemContext actorContext; + @Before public void setUp() throws Exception { executor = MoreExecutors.listeningDecorator(ThingsBoardExecutors.newWorkStealingPool(8, getClass())); @@ -85,6 +108,12 @@ public abstract class BaseTenantControllerTest extends AbstractControllerTest { @After public void tearDown() throws Exception { + loginSysAdmin(); + for (Queue queue : doGetTypedWithPageLink("/api/queues?serviceType=TB_RULE_ENGINE&", new TypeReference>() {}, new PageLink(100)).getData()) { + if (!queue.getName().equals(DataConstants.MAIN_QUEUE_NAME)) { + doDelete("/api/queues/" + queue.getId()).andExpect(status().isOk()); + } + } executor.shutdownNow(); } @@ -518,10 +547,139 @@ public abstract class BaseTenantControllerTest extends AbstractControllerTest { doDelete("/api/tenant/" + tenant.getId().getId().toString()).andExpect(status().isOk()); } + @Test + public void testUpdateTenantProfileToIsolated() throws Exception { + loginSysAdmin(); + doPost("/api/queues?serviceType=TB_RULE_ENGINE", new Queue(TenantId.SYS_TENANT_ID, getQueueConfig(DataConstants.HP_QUEUE_NAME, DataConstants.HP_QUEUE_TOPIC))).andExpect(status().isOk()); + TenantProfile tenantProfile = new TenantProfile(); + tenantProfile.setName("Test profile"); + TenantProfileData tenantProfileData = new TenantProfileData(); + tenantProfileData.setConfiguration(new DefaultTenantProfileConfiguration()); + tenantProfile.setProfileData(tenantProfileData); + tenantProfile.setIsolatedTbRuleEngine(false); + tenantProfile = doPost("/api/tenantProfile", tenantProfile, TenantProfile.class); + createDifferentTenant(); + loginSysAdmin(); + savedDifferentTenant.setTenantProfileId(tenantProfile.getId()); + savedDifferentTenant = doPost("/api/tenant", savedDifferentTenant, Tenant.class); + TenantId tenantId = differentTenantId; + + loginDifferentTenant(); + DeviceProfile hpQueueProfile = createDeviceProfile("HighPriority profile"); + hpQueueProfile.setDefaultQueueName(DataConstants.HP_QUEUE_NAME); + hpQueueProfile = doPost("/api/deviceProfile", hpQueueProfile, DeviceProfile.class); + Device hpQueueDevice = createDevice("HP", hpQueueProfile.getName(), "HP"); + + DeviceProfile mainQueueProfile = createDeviceProfile("Main profile"); + mainQueueProfile.setDefaultQueueName(DataConstants.MAIN_QUEUE_NAME); + mainQueueProfile = doPost("/api/deviceProfile", mainQueueProfile, DeviceProfile.class); + Device mainQueueDevice = createDevice("Main", mainQueueProfile.getName(), "Main"); + + verifyUsedQueueAndMessage(DataConstants.HP_QUEUE_NAME, tenantId, hpQueueDevice.getId(), DataConstants.ATTRIBUTES_UPDATED, () -> { + doPost("/api/plugins/telemetry/DEVICE/" + hpQueueDevice.getId() + "/attributes/SERVER_SCOPE", "{\"test\":123}", String.class); + }, usedTpi -> { + assertThat(usedTpi.getTopic()).isEqualTo(DataConstants.HP_QUEUE_TOPIC); + assertThat(usedTpi.getTenantId()).get().isEqualTo(TenantId.SYS_TENANT_ID); + }); + verifyUsedQueueAndMessage(DataConstants.MAIN_QUEUE_NAME, tenantId, mainQueueDevice.getId(), DataConstants.ATTRIBUTES_UPDATED, () -> { + doPost("/api/plugins/telemetry/DEVICE/" + mainQueueDevice.getId() + "/attributes/SERVER_SCOPE", "{\"test\":123}", String.class); + }, usedTpi -> { + assertThat(usedTpi.getTopic()).isEqualTo(DataConstants.MAIN_QUEUE_TOPIC); + assertThat(usedTpi.getTenantId()).get().isEqualTo(TenantId.SYS_TENANT_ID); + }); + + loginSysAdmin(); + tenantProfile.setIsolatedTbRuleEngine(true); + tenantProfile.getProfileData().setQueueConfiguration(List.of( + getQueueConfig(DataConstants.MAIN_QUEUE_NAME, DataConstants.MAIN_QUEUE_TOPIC) + )); + tenantProfile = doPost("/api/tenantProfile", tenantProfile, TenantProfile.class); + + loginDifferentTenant(); + verifyUsedQueueAndMessage(DataConstants.MAIN_QUEUE_NAME, tenantId, mainQueueDevice.getId(), DataConstants.ATTRIBUTES_UPDATED, () -> { + doPost("/api/plugins/telemetry/DEVICE/" + mainQueueDevice.getId() + "/attributes/SERVER_SCOPE", "{\"test\":123}", String.class); + }, usedTpi -> { + assertThat(usedTpi.getTopic()).isEqualTo(DataConstants.MAIN_QUEUE_TOPIC); + assertThat(usedTpi.getTenantId()).get().isEqualTo(tenantId); + }); + verifyUsedQueueAndMessage(DataConstants.HP_QUEUE_NAME, tenantId, hpQueueDevice.getId(), DataConstants.ATTRIBUTES_UPDATED, () -> { + doPost("/api/plugins/telemetry/DEVICE/" + hpQueueDevice.getId() + "/attributes/SERVER_SCOPE", "{\"test\":123}", String.class); + }, usedTpi -> { + assertThat(usedTpi.getTopic()).isEqualTo(DataConstants.HP_QUEUE_TOPIC); + assertThat(usedTpi.getTenantId()).get().isEqualTo(TenantId.SYS_TENANT_ID); + }); + + loginSysAdmin(); + tenantProfile.setIsolatedTbRuleEngine(true); + tenantProfile.getProfileData().setQueueConfiguration(List.of( + getQueueConfig(DataConstants.MAIN_QUEUE_NAME, DataConstants.MAIN_QUEUE_TOPIC), + getQueueConfig(DataConstants.HP_QUEUE_NAME, DataConstants.HP_QUEUE_TOPIC) + )); + tenantProfile = doPost("/api/tenantProfile", tenantProfile, TenantProfile.class); + + loginDifferentTenant(); + verifyUsedQueueAndMessage(DataConstants.HP_QUEUE_NAME, tenantId, hpQueueDevice.getId(), DataConstants.ATTRIBUTES_UPDATED, () -> { + doPost("/api/plugins/telemetry/DEVICE/" + hpQueueDevice.getId() + "/attributes/SERVER_SCOPE", "{\"test\":123}", String.class); + }, usedTpi -> { + assertThat(usedTpi.getTopic()).isEqualTo(DataConstants.HP_QUEUE_TOPIC); + assertThat(usedTpi.getTenantId()).get().isEqualTo(tenantId); + }); + verifyUsedQueueAndMessage(DataConstants.MAIN_QUEUE_NAME, tenantId, mainQueueDevice.getId(), DataConstants.ATTRIBUTES_UPDATED, () -> { + doPost("/api/plugins/telemetry/DEVICE/" + mainQueueDevice.getId() + "/attributes/SERVER_SCOPE", "{\"test\":123}", String.class); + }, usedTpi -> { + assertThat(usedTpi.getTopic()).isEqualTo(DataConstants.MAIN_QUEUE_TOPIC); + assertThat(usedTpi.getTenantId()).get().isEqualTo(tenantId); + }); + } + + private void verifyUsedQueueAndMessage(String queue, TenantId tenantId, EntityId entityId, String msgType, Runnable action, Consumer tpiAssert) { + await().atMost(15, TimeUnit.SECONDS) + .untilAsserted(() -> { + TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_RULE_ENGINE, queue, tenantId, entityId); + tpiAssert.accept(tpi); + }); + action.run(); + TbMsg tbMsg = awaitTbMsg(msg -> msg.getOriginator().equals(entityId) + && msg.getType().equals(msgType), 10000); + assertThat(tbMsg.getQueueName()).isEqualTo(queue); + + TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_RULE_ENGINE, queue, tenantId, entityId); + tpiAssert.accept(tpi); + } + + protected TbMsg awaitTbMsg(Predicate predicate, int timeoutMillis) { + AtomicReference tbMsgCaptor = new AtomicReference<>(); + verify(actorContext, timeout(timeoutMillis).atLeastOnce()).tell(argThat(actorMsg -> { + if (!(actorMsg instanceof QueueToRuleEngineMsg)) { + return false; + } + TbMsg tbMsg = ((QueueToRuleEngineMsg) actorMsg).getMsg(); + if (predicate.test(tbMsg)) { + tbMsgCaptor.set(tbMsg); + return true; + } + return false; + })); + return tbMsgCaptor.get(); + } + private void addQueueConfig(TenantProfile tenantProfile, String queueName) { + TenantProfileQueueConfiguration queueConfiguration = getQueueConfig(queueName, "tb_rule_engine." + queueName.toLowerCase()); + TenantProfileData profileData = tenantProfile.getProfileData(); + + List configs = profileData.getQueueConfiguration(); + if (configs == null) { + configs = new ArrayList<>(); + } + configs.add(queueConfiguration); + profileData.setQueueConfiguration(configs); + tenantProfile.setProfileData(profileData); + } + + private TenantProfileQueueConfiguration getQueueConfig(String queueName, String topic) { TenantProfileQueueConfiguration queueConfiguration = new TenantProfileQueueConfiguration(); queueConfiguration.setName(queueName); - queueConfiguration.setTopic("tb_rule_engine." + queueName.toLowerCase()); + queueConfiguration.setTopic(topic); queueConfiguration.setPollInterval(25); queueConfiguration.setPartitions(1 + new Random().nextInt(99)); queueConfiguration.setConsumerPerPartition(true); @@ -537,15 +695,7 @@ public abstract class BaseTenantControllerTest extends AbstractControllerTest { processingStrategy.setPauseBetweenRetries(3); processingStrategy.setMaxPauseBetweenRetries(3); queueConfiguration.setProcessingStrategy(processingStrategy); - TenantProfileData profileData = tenantProfile.getProfileData(); - - List configs = profileData.getQueueConfiguration(); - if (configs == null) { - configs = new ArrayList<>(); - } - configs.add(queueConfiguration); - profileData.setQueueConfiguration(configs); - tenantProfile.setProfileData(profileData); + return queueConfiguration; } private List getQueuesFromConfig(List queueConfiguration, List queues) { diff --git a/application/src/test/java/org/thingsboard/server/controller/BaseTenantProfileControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/BaseTenantProfileControllerTest.java index d4116a873d..e8c3baada9 100644 --- a/application/src/test/java/org/thingsboard/server/controller/BaseTenantProfileControllerTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/BaseTenantProfileControllerTest.java @@ -165,23 +165,6 @@ public abstract class BaseTenantProfileControllerTest extends AbstractController testBroadcastEntityStateChangeEventNeverTenantProfile(); } - @Test - public void testSaveSameTenantProfileWithDifferentIsolatedTbRuleEngine() throws Exception { - loginSysAdmin(); - TenantProfile tenantProfile = this.createTenantProfile("Tenant Profile"); - TenantProfile savedTenantProfile = doPost("/api/tenantProfile", tenantProfile, TenantProfile.class); - savedTenantProfile.setIsolatedTbRuleEngine(true); - addMainQueueConfig(savedTenantProfile); - - Mockito.reset(tbClusterService); - - doPost("/api/tenantProfile", savedTenantProfile) - .andExpect(status().isBadRequest()) - .andExpect(statusReason(containsString("Can't update isolatedTbRuleEngine property"))); - - testBroadcastEntityStateChangeEventNeverTenantProfile(); - } - @Test public void testDeleteTenantProfileWithExistingTenant() throws Exception { loginSysAdmin(); diff --git a/application/src/test/java/org/thingsboard/server/service/notification/NotificationRuleApiTest.java b/application/src/test/java/org/thingsboard/server/service/notification/NotificationRuleApiTest.java index b03a914d1f..f0ece07cdc 100644 --- a/application/src/test/java/org/thingsboard/server/service/notification/NotificationRuleApiTest.java +++ b/application/src/test/java/org/thingsboard/server/service/notification/NotificationRuleApiTest.java @@ -301,7 +301,7 @@ public class NotificationRuleApiTest extends AbstractNotificationApiTest { @Test public void testNotificationRuleProcessing_entitiesLimit() throws Exception { int limit = 5; - updateDefaultTenantProfile(profileConfiguration -> { + updateDefaultTenantProfileConfig(profileConfiguration -> { profileConfiguration.setMaxDevices(limit); profileConfiguration.setMaxAssets(limit); profileConfiguration.setMaxCustomers(limit); @@ -396,7 +396,7 @@ public class NotificationRuleApiTest extends AbstractNotificationApiTest { @Test public void testNotificationRequestsPerRuleRateLimits() throws Exception { int notificationRequestsLimit = 10; - updateDefaultTenantProfile(profileConfiguration -> { + updateDefaultTenantProfileConfig(profileConfiguration -> { profileConfiguration.setTenantNotificationRequestsPerRuleRateLimit(notificationRequestsLimit + ":300"); }); diff --git a/application/src/test/java/org/thingsboard/server/service/queue/QueueServiceTest.java b/application/src/test/java/org/thingsboard/server/service/queue/QueueServiceTest.java deleted file mode 100644 index 4d2c50add3..0000000000 --- a/application/src/test/java/org/thingsboard/server/service/queue/QueueServiceTest.java +++ /dev/null @@ -1,225 +0,0 @@ -/** - * Copyright © 2016-2023 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.queue; - -import org.junit.Before; -import org.junit.Test; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.boot.test.mock.mockito.SpyBean; -import org.thingsboard.server.actors.ActorSystemContext; -import org.thingsboard.server.common.data.DataConstants; -import org.thingsboard.server.common.data.Device; -import org.thingsboard.server.common.data.DeviceProfile; -import org.thingsboard.server.common.data.id.EntityId; -import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.common.data.queue.ProcessingStrategy; -import org.thingsboard.server.common.data.queue.ProcessingStrategyType; -import org.thingsboard.server.common.data.queue.Queue; -import org.thingsboard.server.common.data.queue.SubmitStrategy; -import org.thingsboard.server.common.data.queue.SubmitStrategyType; -import org.thingsboard.server.common.data.tenant.profile.TenantProfileQueueConfiguration; -import org.thingsboard.server.common.msg.TbMsg; -import org.thingsboard.server.common.msg.queue.QueueToRuleEngineMsg; -import org.thingsboard.server.common.msg.queue.ServiceType; -import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; -import org.thingsboard.server.controller.AbstractControllerTest; -import org.thingsboard.server.dao.queue.QueueService; -import org.thingsboard.server.dao.service.DaoSqlTest; -import org.thingsboard.server.queue.discovery.PartitionService; -import org.thingsboard.server.service.entitiy.queue.TbQueueService; -import org.thingsboard.server.service.profile.TbDeviceProfileCache; - -import java.util.List; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicReference; -import java.util.function.Consumer; -import java.util.function.Predicate; - -import static org.assertj.core.api.Assertions.assertThat; -import static org.awaitility.Awaitility.await; -import static org.mockito.ArgumentMatchers.argThat; -import static org.mockito.Mockito.timeout; -import static org.mockito.Mockito.verify; - -@DaoSqlTest -public class QueueServiceTest extends AbstractControllerTest { - - @SpyBean - private ActorSystemContext actorContext; - @Autowired - private TbQueueService tbQueueService; - @Autowired - private QueueService queueService; - @SpyBean - private PartitionService partitionService; - @Autowired - private TbDeviceProfileCache deviceProfileCache; - - @Before - public void beforeEach() { - Queue mainQueue = queueService.findQueueByTenantIdAndName(TenantId.SYS_TENANT_ID, DataConstants.MAIN_QUEUE_NAME); - if (mainQueue == null) { - mainQueue = new Queue(TenantId.SYS_TENANT_ID, getMainQueueConfig()); - tbQueueService.saveQueue(mainQueue); - } - - Queue hpQueue = queueService.findQueueByTenantIdAndName(TenantId.SYS_TENANT_ID, DataConstants.HP_QUEUE_NAME); - if (hpQueue == null) { - hpQueue = new Queue(TenantId.SYS_TENANT_ID, getHighPriorityQueueConfig()); - tbQueueService.saveQueue(hpQueue); - } - } - - @Test - public void testQueuesUpdateOnTenantProfileUpdate() throws Exception { - loginTenantAdmin(); - DeviceProfile hpQueueProfile = createDeviceProfile("HighPriority profile"); - hpQueueProfile.setDefaultQueueName(DataConstants.HP_QUEUE_NAME); - hpQueueProfile = doPost("/api/deviceProfile", hpQueueProfile, DeviceProfile.class); - Device hpQueueDevice = createDevice("HP", hpQueueProfile.getName(), "HP"); - deviceProfileCache.evict(tenantId, hpQueueProfile.getId()); - - DeviceProfile mainQueueProfile = createDeviceProfile("Main profile"); - mainQueueProfile.setDefaultQueueName(DataConstants.MAIN_QUEUE_NAME); - mainQueueProfile = doPost("/api/deviceProfile", mainQueueProfile, DeviceProfile.class); - Device mainQueueDevice = createDevice("Main", mainQueueProfile.getName(), "Main"); - - verifyUsedQueueAndMessage(DataConstants.HP_QUEUE_NAME, hpQueueDevice.getId(), DataConstants.ATTRIBUTES_UPDATED, () -> { - doPost("/api/plugins/telemetry/DEVICE/" + hpQueueDevice.getId() + "/attributes/SERVER_SCOPE", "{\"test\":123}", String.class); - }, usedTpi -> { - assertThat(usedTpi.getTenantId()).get().isEqualTo(TenantId.SYS_TENANT_ID); - }); - verifyUsedQueueAndMessage(DataConstants.MAIN_QUEUE_NAME, mainQueueDevice.getId(), DataConstants.ATTRIBUTES_UPDATED, () -> { - doPost("/api/plugins/telemetry/DEVICE/" + mainQueueDevice.getId() + "/attributes/SERVER_SCOPE", "{\"test\":123}", String.class); - }, usedTpi -> { - assertThat(usedTpi.getTenantId()).get().isEqualTo(TenantId.SYS_TENANT_ID); - }); - - updateDefaultTenantProfile(tenantProfile -> { - tenantProfile.setIsolatedTbRuleEngine(true); - tenantProfile.getProfileData().setQueueConfiguration(List.of( - getMainQueueConfig() - )); - }); - - verifyUsedQueueAndMessage(DataConstants.MAIN_QUEUE_NAME, mainQueueDevice.getId(), DataConstants.ATTRIBUTES_UPDATED, () -> { - doPost("/api/plugins/telemetry/DEVICE/" + mainQueueDevice.getId() + "/attributes/SERVER_SCOPE", "{\"test\":123}", String.class); - }, usedTpi -> { - assertThat(usedTpi.getTenantId()).get().isEqualTo(tenantId); - }); - verifyUsedQueueAndMessage(DataConstants.HP_QUEUE_NAME, hpQueueDevice.getId(), DataConstants.ATTRIBUTES_UPDATED, () -> { - doPost("/api/plugins/telemetry/DEVICE/" + hpQueueDevice.getId() + "/attributes/SERVER_SCOPE", "{\"test\":123}", String.class); - }, usedTpi -> { - assertThat(usedTpi.getTopic()).endsWith("main"); - assertThat(usedTpi.getTenantId()).get().isEqualTo(tenantId); - }); - - updateDefaultTenantProfile(tenantProfile -> { - tenantProfile.setIsolatedTbRuleEngine(true); - tenantProfile.getProfileData().setQueueConfiguration(List.of( - getMainQueueConfig(), getHighPriorityQueueConfig() - )); - }); - - verifyUsedQueueAndMessage(DataConstants.HP_QUEUE_NAME, hpQueueDevice.getId(), DataConstants.ATTRIBUTES_UPDATED, () -> { - doPost("/api/plugins/telemetry/DEVICE/" + hpQueueDevice.getId() + "/attributes/SERVER_SCOPE", "{\"test\":123}", String.class); - }, usedTpi -> { - assertThat(usedTpi.getTopic()).endsWith("hp"); - assertThat(usedTpi.getTenantId()).get().isEqualTo(tenantId); - }); - verifyUsedQueueAndMessage(DataConstants.MAIN_QUEUE_NAME, mainQueueDevice.getId(), DataConstants.ATTRIBUTES_UPDATED, () -> { - doPost("/api/plugins/telemetry/DEVICE/" + mainQueueDevice.getId() + "/attributes/SERVER_SCOPE", "{\"test\":123}", String.class); - }, usedTpi -> { - assertThat(usedTpi.getTenantId()).get().isEqualTo(tenantId); - }); - } - - private void verifyUsedQueueAndMessage(String queue, EntityId entityId, String msgType, Runnable action, Consumer tpiAssert) { - await().atMost(15, TimeUnit.SECONDS) - .untilAsserted(() -> { - TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_RULE_ENGINE, queue, tenantId, entityId); - tpiAssert.accept(tpi); - }); - action.run(); - TbMsg tbMsg = awaitTbMsg(msg -> msg.getOriginator().equals(entityId) - && msg.getType().equals(msgType), 10000); - assertThat(tbMsg.getQueueName()).isEqualTo(queue); - - TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_RULE_ENGINE, queue, tenantId, entityId); - tpiAssert.accept(tpi); - } - - protected TbMsg awaitTbMsg(Predicate predicate, int timeoutMillis) { - AtomicReference tbMsgCaptor = new AtomicReference<>(); - verify(actorContext, timeout(timeoutMillis).atLeastOnce()).tell(argThat(actorMsg -> { - if (!(actorMsg instanceof QueueToRuleEngineMsg)) { - return false; - } - TbMsg tbMsg = ((QueueToRuleEngineMsg) actorMsg).getMsg(); - if (predicate.test(tbMsg)) { - tbMsgCaptor.set(tbMsg); - return true; - } - return false; - })); - return tbMsgCaptor.get(); - } - - private TenantProfileQueueConfiguration getHighPriorityQueueConfig() { - TenantProfileQueueConfiguration hpQueueConfig = new TenantProfileQueueConfiguration(); - hpQueueConfig.setName(DataConstants.HP_QUEUE_NAME); - hpQueueConfig.setTopic(DataConstants.HP_QUEUE_TOPIC); - hpQueueConfig.setPollInterval(25); - hpQueueConfig.setPartitions(10); - hpQueueConfig.setConsumerPerPartition(true); - hpQueueConfig.setPackProcessingTimeout(2000); - SubmitStrategy highPriorityQueueSubmitStrategy = new SubmitStrategy(); - highPriorityQueueSubmitStrategy.setType(SubmitStrategyType.BURST); - highPriorityQueueSubmitStrategy.setBatchSize(100); - hpQueueConfig.setSubmitStrategy(highPriorityQueueSubmitStrategy); - ProcessingStrategy highPriorityQueueProcessingStrategy = new ProcessingStrategy(); - highPriorityQueueProcessingStrategy.setType(ProcessingStrategyType.RETRY_FAILED_AND_TIMED_OUT); - highPriorityQueueProcessingStrategy.setRetries(0); - highPriorityQueueProcessingStrategy.setFailurePercentage(0); - highPriorityQueueProcessingStrategy.setPauseBetweenRetries(5); - highPriorityQueueProcessingStrategy.setMaxPauseBetweenRetries(5); - hpQueueConfig.setProcessingStrategy(highPriorityQueueProcessingStrategy); - return hpQueueConfig; - } - - private TenantProfileQueueConfiguration getMainQueueConfig() { - TenantProfileQueueConfiguration mainQueue = new TenantProfileQueueConfiguration(); - mainQueue.setName(DataConstants.MAIN_QUEUE_NAME); - mainQueue.setTopic(DataConstants.MAIN_QUEUE_TOPIC); - mainQueue.setPollInterval(25); - mainQueue.setPartitions(10); - mainQueue.setConsumerPerPartition(true); - mainQueue.setPackProcessingTimeout(2000); - SubmitStrategy mainQueueSubmitStrategy = new SubmitStrategy(); - mainQueueSubmitStrategy.setType(SubmitStrategyType.BURST); - mainQueueSubmitStrategy.setBatchSize(1000); - mainQueue.setSubmitStrategy(mainQueueSubmitStrategy); - ProcessingStrategy mainQueueProcessingStrategy = new ProcessingStrategy(); - mainQueueProcessingStrategy.setType(ProcessingStrategyType.SKIP_ALL_FAILURES); - mainQueueProcessingStrategy.setRetries(3); - mainQueueProcessingStrategy.setFailurePercentage(0); - mainQueueProcessingStrategy.setPauseBetweenRetries(3); - mainQueueProcessingStrategy.setMaxPauseBetweenRetries(3); - mainQueue.setProcessingStrategy(mainQueueProcessingStrategy); - return mainQueue; - } - -} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/HashPartitionService.java b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/HashPartitionService.java index 31f517c236..a8954b2b33 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/HashPartitionService.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/HashPartitionService.java @@ -198,7 +198,7 @@ public class HashPartitionService implements PartitionService { if (!MAIN_QUEUE_NAME.equals(queueName) && !partitionSizesMap.containsKey(queueKey)) { queueKey = new QueueKey(serviceType, TenantId.SYS_TENANT_ID); } - log.warn("Using queue {} instead of isolated {}", queueKey, queueName); + log.warn("Using queue {} instead of isolated {} for tenant {}", queueKey, queueName, isolatedOrSystemTenantId); } } return resolve(queueKey, entityId); diff --git a/ui-ngx/src/app/modules/home/components/profile/tenant-profile.component.ts b/ui-ngx/src/app/modules/home/components/profile/tenant-profile.component.ts index e96a237033..fa8b1636ff 100644 --- a/ui-ngx/src/app/modules/home/components/profile/tenant-profile.component.ts +++ b/ui-ngx/src/app/modules/home/components/profile/tenant-profile.component.ts @@ -56,10 +56,10 @@ export class TenantProfileComponent extends EntityComponent { const mainQueue = [ { id: guid(), - consumerPerPartition: false, + consumerPerPartition: true, name: 'Main', packProcessingTimeout: 2000, - partitions: 1, + partitions: 2, pollInterval: 25, processingStrategy: { failurePercentage: 0, @@ -82,8 +82,8 @@ export class TenantProfileComponent extends EntityComponent { name: 'HighPriority', topic: 'tb_rule_engine.hp', pollInterval: 25, - partitions: 1, - consumerPerPartition: false, + partitions: 2, + consumerPerPartition: true, packProcessingTimeout: 2000, submitStrategy: { type: 'BURST', @@ -105,8 +105,8 @@ export class TenantProfileComponent extends EntityComponent { name: 'SequentialByOriginator', topic: 'tb_rule_engine.sq', pollInterval: 25, - partitions: 1, - consumerPerPartition: false, + partitions: 2, + consumerPerPartition: true, packProcessingTimeout: 2000, submitStrategy: { type: 'SEQUENTIAL_BY_ORIGINATOR', From 55775f2815db89d78596c6a99a74ebb3cffaa830 Mon Sep 17 00:00:00 2001 From: ViacheslavKlimov Date: Thu, 27 Jul 2023 18:33:03 +0300 Subject: [PATCH 11/23] Consumer group per isolated tenant's queue; remove sleeping for Kafka consumer --- .../queue/common/AbstractTbQueueConsumerTemplate.java | 8 +++++++- .../server/queue/kafka/TbKafkaConsumerTemplate.java | 6 ++++++ .../server/queue/provider/KafkaMonolithQueueFactory.java | 2 +- .../queue/provider/KafkaTbRuleEngineQueueFactory.java | 2 +- 4 files changed, 15 insertions(+), 3 deletions(-) diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/common/AbstractTbQueueConsumerTemplate.java b/common/queue/src/main/java/org/thingsboard/server/queue/common/AbstractTbQueueConsumerTemplate.java index a152e0c466..5b0e84c2ed 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/common/AbstractTbQueueConsumerTemplate.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/common/AbstractTbQueueConsumerTemplate.java @@ -103,7 +103,9 @@ public abstract class AbstractTbQueueConsumerTemplate i consumerLock.unlock(); } - if (records.isEmpty()) { return sleepAndReturnEmpty(startNanos, durationInMillis); } + if (records.isEmpty() && !isLongPollingSupported()) { + return sleepAndReturnEmpty(startNanos, durationInMillis); + } return decodeRecords(records); } @@ -189,4 +191,8 @@ public abstract class AbstractTbQueueConsumerTemplate i abstract protected void doUnsubscribe(); + protected boolean isLongPollingSupported() { + return false; + } + } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaConsumerTemplate.java b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaConsumerTemplate.java index 00bb7aa541..9f58446966 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaConsumerTemplate.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaConsumerTemplate.java @@ -122,4 +122,10 @@ public class TbKafkaConsumerTemplate extends AbstractTbQue statsService.unregisterClientGroup(groupId); } } + + @Override + public boolean isLongPollingSupported() { + return true; + } + } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaMonolithQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaMonolithQueueFactory.java index 364abcff4c..779bf9fae3 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaMonolithQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaMonolithQueueFactory.java @@ -187,7 +187,7 @@ public class KafkaMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEngi consumerBuilder.settings(kafkaSettings); consumerBuilder.topic(configuration.getTopic()); consumerBuilder.clientId("re-" + queueName + "-consumer-" + serviceInfoProvider.getServiceId() + "-" + consumerCount.incrementAndGet()); - consumerBuilder.groupId("re-" + queueName + "-consumer"); + consumerBuilder.groupId("re-" + queueName + (!configuration.getTenantId().isSysTenantId() ? "-" + configuration.getTenantId() : "") + "-consumer"); consumerBuilder.decoder(msg -> new TbProtoQueueMsg<>(msg.getKey(), ToRuleEngineMsg.parseFrom(msg.getData()), msg.getHeaders())); consumerBuilder.admin(ruleEngineAdmin); consumerBuilder.statsService(consumerStatsService); diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbRuleEngineQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbRuleEngineQueueFactory.java index b8e07a45f7..eb387a84f4 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbRuleEngineQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbRuleEngineQueueFactory.java @@ -166,7 +166,7 @@ public class KafkaTbRuleEngineQueueFactory implements TbRuleEngineQueueFactory { consumerBuilder.settings(kafkaSettings); consumerBuilder.topic(configuration.getTopic()); consumerBuilder.clientId("re-" + queueName + "-consumer-" + serviceInfoProvider.getServiceId() + "-" + consumerCount.incrementAndGet()); - consumerBuilder.groupId("re-" + queueName + "-consumer"); + consumerBuilder.groupId("re-" + queueName + (!configuration.getTenantId().isSysTenantId() ? "-" + configuration.getTenantId() : "") + "-consumer"); consumerBuilder.decoder(msg -> new TbProtoQueueMsg<>(msg.getKey(), ToRuleEngineMsg.parseFrom(msg.getData()), msg.getHeaders())); consumerBuilder.admin(ruleEngineAdmin); consumerBuilder.statsService(consumerStatsService); From 137cc1809919c4535d0d64b8d8ebda311521f455 Mon Sep 17 00:00:00 2001 From: ViacheslavKlimov Date: Thu, 27 Jul 2023 19:58:31 +0300 Subject: [PATCH 12/23] Auto offset reset config for Kafka --- application/src/main/resources/thingsboard.yml | 1 + .../org/thingsboard/server/queue/kafka/TbKafkaSettings.java | 4 ++++ msa/vc-executor/src/main/resources/tb-vc-executor.yml | 1 + transport/coap/src/main/resources/tb-coap-transport.yml | 1 + transport/http/src/main/resources/tb-http-transport.yml | 1 + transport/lwm2m/src/main/resources/tb-lwm2m-transport.yml | 1 + transport/mqtt/src/main/resources/tb-mqtt-transport.yml | 1 + transport/snmp/src/main/resources/tb-snmp-transport.yml | 1 + 8 files changed, 11 insertions(+) diff --git a/application/src/main/resources/thingsboard.yml b/application/src/main/resources/thingsboard.yml index 163c325f52..96ee2793ed 100644 --- a/application/src/main/resources/thingsboard.yml +++ b/application/src/main/resources/thingsboard.yml @@ -1036,6 +1036,7 @@ queue: fetch_max_bytes: "${TB_QUEUE_KAFKA_FETCH_MAX_BYTES:134217728}" request.timeout.ms: "${TB_QUEUE_KAFKA_REQUEST_TIMEOUT_MS:30000}" # (30 seconds) # refer to https://docs.confluent.io/platform/current/installation/configuration/producer-configs.html#producerconfigs_request.timeout.ms session.timeout.ms: "${TB_QUEUE_KAFKA_SESSION_TIMEOUT_MS:10000}" # (10 seconds) # refer to https://docs.confluent.io/platform/current/installation/configuration/consumer-configs.html#consumerconfigs_session.timeout.ms + auto_offset_reset: "${TB_QUEUE_KAFKA_AUTO_OFFSET_RESET:earliest}" # earliest, latest or none use_confluent_cloud: "${TB_QUEUE_KAFKA_USE_CONFLUENT_CLOUD:false}" confluent: ssl.algorithm: "${TB_QUEUE_KAFKA_CONFLUENT_SSL_ALGORITHM:https}" diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaSettings.java b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaSettings.java index 41ae0d5ea3..d2a54128e3 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaSettings.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaSettings.java @@ -115,6 +115,9 @@ public class TbKafkaSettings { @Value("${queue.kafka.session.timeout.ms:10000}") private int sessionTimeoutMs; + @Value("${queue.kafka.auto_offset_reset:earliest}") + private String autoOffsetReset; + @Value("${queue.kafka.use_confluent_cloud:false}") private boolean useConfluent; @@ -155,6 +158,7 @@ public class TbKafkaSettings { props.put(ConsumerConfig.MAX_PARTITION_FETCH_BYTES_CONFIG, maxPartitionFetchBytes); props.put(ConsumerConfig.FETCH_MAX_BYTES_CONFIG, fetchMaxBytes); props.put(ConsumerConfig.MAX_POLL_INTERVAL_MS_CONFIG, maxPollIntervalMs); + props.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, autoOffsetReset); props.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class); props.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, ByteArrayDeserializer.class); diff --git a/msa/vc-executor/src/main/resources/tb-vc-executor.yml b/msa/vc-executor/src/main/resources/tb-vc-executor.yml index 352f94e091..9aa149280a 100644 --- a/msa/vc-executor/src/main/resources/tb-vc-executor.yml +++ b/msa/vc-executor/src/main/resources/tb-vc-executor.yml @@ -72,6 +72,7 @@ queue: fetch_max_bytes: "${TB_QUEUE_KAFKA_FETCH_MAX_BYTES:134217728}" request.timeout.ms: "${TB_QUEUE_KAFKA_REQUEST_TIMEOUT_MS:30000}" # (30 seconds) # refer to https://docs.confluent.io/platform/current/installation/configuration/producer-configs.html#producerconfigs_request.timeout.ms session.timeout.ms: "${TB_QUEUE_KAFKA_SESSION_TIMEOUT_MS:10000}" # (10 seconds) # refer to https://docs.confluent.io/platform/current/installation/configuration/consumer-configs.html#consumerconfigs_session.timeout.ms + auto_offset_reset: "${TB_QUEUE_KAFKA_AUTO_OFFSET_RESET:earliest}" # earliest, latest or none use_confluent_cloud: "${TB_QUEUE_KAFKA_USE_CONFLUENT_CLOUD:false}" confluent: ssl.algorithm: "${TB_QUEUE_KAFKA_CONFLUENT_SSL_ALGORITHM:https}" diff --git a/transport/coap/src/main/resources/tb-coap-transport.yml b/transport/coap/src/main/resources/tb-coap-transport.yml index 7ea553fe5c..5e0a7f2a9d 100644 --- a/transport/coap/src/main/resources/tb-coap-transport.yml +++ b/transport/coap/src/main/resources/tb-coap-transport.yml @@ -173,6 +173,7 @@ queue: fetch_max_bytes: "${TB_QUEUE_KAFKA_FETCH_MAX_BYTES:134217728}" request.timeout.ms: "${TB_QUEUE_KAFKA_REQUEST_TIMEOUT_MS:30000}" # (30 seconds) # refer to https://docs.confluent.io/platform/current/installation/configuration/producer-configs.html#producerconfigs_request.timeout.ms session.timeout.ms: "${TB_QUEUE_KAFKA_SESSION_TIMEOUT_MS:10000}" # (10 seconds) # refer to https://docs.confluent.io/platform/current/installation/configuration/consumer-configs.html#consumerconfigs_session.timeout.ms + auto_offset_reset: "${TB_QUEUE_KAFKA_AUTO_OFFSET_RESET:earliest}" # earliest, latest or none use_confluent_cloud: "${TB_QUEUE_KAFKA_USE_CONFLUENT_CLOUD:false}" confluent: ssl.algorithm: "${TB_QUEUE_KAFKA_CONFLUENT_SSL_ALGORITHM:https}" diff --git a/transport/http/src/main/resources/tb-http-transport.yml b/transport/http/src/main/resources/tb-http-transport.yml index 346ec48eae..f7af5c979a 100644 --- a/transport/http/src/main/resources/tb-http-transport.yml +++ b/transport/http/src/main/resources/tb-http-transport.yml @@ -158,6 +158,7 @@ queue: fetch_max_bytes: "${TB_QUEUE_KAFKA_FETCH_MAX_BYTES:134217728}" request.timeout.ms: "${TB_QUEUE_KAFKA_REQUEST_TIMEOUT_MS:30000}" # (30 seconds) # refer to https://docs.confluent.io/platform/current/installation/configuration/producer-configs.html#producerconfigs_request.timeout.ms session.timeout.ms: "${TB_QUEUE_KAFKA_SESSION_TIMEOUT_MS:10000}" # (10 seconds) # refer to https://docs.confluent.io/platform/current/installation/configuration/consumer-configs.html#consumerconfigs_session.timeout.ms + auto_offset_reset: "${TB_QUEUE_KAFKA_AUTO_OFFSET_RESET:earliest}" # earliest, latest or none use_confluent_cloud: "${TB_QUEUE_KAFKA_USE_CONFLUENT_CLOUD:false}" confluent: ssl.algorithm: "${TB_QUEUE_KAFKA_CONFLUENT_SSL_ALGORITHM:https}" diff --git a/transport/lwm2m/src/main/resources/tb-lwm2m-transport.yml b/transport/lwm2m/src/main/resources/tb-lwm2m-transport.yml index 4e8167d89d..3d649b3c2b 100644 --- a/transport/lwm2m/src/main/resources/tb-lwm2m-transport.yml +++ b/transport/lwm2m/src/main/resources/tb-lwm2m-transport.yml @@ -239,6 +239,7 @@ queue: fetch_max_bytes: "${TB_QUEUE_KAFKA_FETCH_MAX_BYTES:134217728}" request.timeout.ms: "${TB_QUEUE_KAFKA_REQUEST_TIMEOUT_MS:30000}" # (30 seconds) # refer to https://docs.confluent.io/platform/current/installation/configuration/producer-configs.html#producerconfigs_request.timeout.ms session.timeout.ms: "${TB_QUEUE_KAFKA_SESSION_TIMEOUT_MS:10000}" # (10 seconds) # refer to https://docs.confluent.io/platform/current/installation/configuration/consumer-configs.html#consumerconfigs_session.timeout.ms + auto_offset_reset: "${TB_QUEUE_KAFKA_AUTO_OFFSET_RESET:earliest}" # earliest, latest or none use_confluent_cloud: "${TB_QUEUE_KAFKA_USE_CONFLUENT_CLOUD:false}" confluent: ssl.algorithm: "${TB_QUEUE_KAFKA_CONFLUENT_SSL_ALGORITHM:https}" diff --git a/transport/mqtt/src/main/resources/tb-mqtt-transport.yml b/transport/mqtt/src/main/resources/tb-mqtt-transport.yml index 1e0b1ebcd4..38f569cfd7 100644 --- a/transport/mqtt/src/main/resources/tb-mqtt-transport.yml +++ b/transport/mqtt/src/main/resources/tb-mqtt-transport.yml @@ -188,6 +188,7 @@ queue: fetch_max_bytes: "${TB_QUEUE_KAFKA_FETCH_MAX_BYTES:134217728}" request.timeout.ms: "${TB_QUEUE_KAFKA_REQUEST_TIMEOUT_MS:30000}" # (30 seconds) # refer to https://docs.confluent.io/platform/current/installation/configuration/producer-configs.html#producerconfigs_request.timeout.ms session.timeout.ms: "${TB_QUEUE_KAFKA_SESSION_TIMEOUT_MS:10000}" # (10 seconds) # refer to https://docs.confluent.io/platform/current/installation/configuration/consumer-configs.html#consumerconfigs_session.timeout.ms + auto_offset_reset: "${TB_QUEUE_KAFKA_AUTO_OFFSET_RESET:earliest}" # earliest, latest or none use_confluent_cloud: "${TB_QUEUE_KAFKA_USE_CONFLUENT_CLOUD:false}" confluent: ssl.algorithm: "${TB_QUEUE_KAFKA_CONFLUENT_SSL_ALGORITHM:https}" diff --git a/transport/snmp/src/main/resources/tb-snmp-transport.yml b/transport/snmp/src/main/resources/tb-snmp-transport.yml index 9f086bcbc5..c4d76dbf30 100644 --- a/transport/snmp/src/main/resources/tb-snmp-transport.yml +++ b/transport/snmp/src/main/resources/tb-snmp-transport.yml @@ -134,6 +134,7 @@ queue: fetch_max_bytes: "${TB_QUEUE_KAFKA_FETCH_MAX_BYTES:134217728}" request.timeout.ms: "${TB_QUEUE_KAFKA_REQUEST_TIMEOUT_MS:30000}" # (30 seconds) # refer to https://docs.confluent.io/platform/current/installation/configuration/producer-configs.html#producerconfigs_request.timeout.ms session.timeout.ms: "${TB_QUEUE_KAFKA_SESSION_TIMEOUT_MS:10000}" # (10 seconds) # refer to https://docs.confluent.io/platform/current/installation/configuration/consumer-configs.html#consumerconfigs_session.timeout.ms + auto_offset_reset: "${TB_QUEUE_KAFKA_AUTO_OFFSET_RESET:earliest}" # earliest, latest or none use_confluent_cloud: "${TB_QUEUE_KAFKA_USE_CONFLUENT_CLOUD:false}" confluent: ssl.algorithm: "${TB_QUEUE_KAFKA_CONFLUENT_SSL_ALGORITHM:https}" From 5862b417aa48859c9f8038c927f27ed36f86aae8 Mon Sep 17 00:00:00 2001 From: ViacheslavKlimov Date: Fri, 28 Jul 2023 12:06:04 +0300 Subject: [PATCH 13/23] Add custom topic properties configuration --- .../entitiy/queue/DefaultTbQueueService.java | 8 ++++++-- .../org/thingsboard/server/queue/TbQueueAdmin.java | 6 +++++- .../server/common/data/queue/Queue.java | 14 +++++++++++++- .../queue/RuleEngineTbQueueAdminFactory.java | 2 +- .../queue/azure/servicebus/TbServiceBusAdmin.java | 7 ++++--- .../server/queue/kafka/TbKafkaAdmin.java | 6 +++--- .../provider/InMemoryTbTransportQueueFactory.java | 2 +- .../server/queue/pubsub/TbPubSubAdmin.java | 2 +- .../server/queue/rabbitmq/TbRabbitMqAdmin.java | 9 ++++++++- .../queue/rabbitmq/TbRabbitMqQueueArguments.java | 8 ++++---- .../server/queue/sqs/TbAwsSqsAdmin.java | 4 +++- .../server/queue/sqs/TbAwsSqsQueueAttributes.java | 8 +++++++- .../server/queue/util/PropertyUtils.java | 14 ++++++++++++++ .../queue/tenant-profile-queues.component.ts | 3 ++- .../components/profile/tenant-profile.component.ts | 9 ++++++--- .../components/queue/queue-form.component.html | 5 +++++ .../home/components/queue/queue-form.component.ts | 3 ++- ui-ngx/src/app/shared/models/queue.models.ts | 1 + .../src/assets/locale/locale.constant-en_US.json | 2 ++ 19 files changed, 88 insertions(+), 25 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/entitiy/queue/DefaultTbQueueService.java b/application/src/main/java/org/thingsboard/server/service/entitiy/queue/DefaultTbQueueService.java index 0166a425d3..9e3d38cf32 100644 --- a/application/src/main/java/org/thingsboard/server/service/entitiy/queue/DefaultTbQueueService.java +++ b/application/src/main/java/org/thingsboard/server/service/entitiy/queue/DefaultTbQueueService.java @@ -96,7 +96,9 @@ public class DefaultTbQueueService extends AbstractTbEntityService implements Tb private void onQueueCreated(Queue queue) { for (int i = 0; i < queue.getPartitions(); i++) { tbQueueAdmin.createTopicIfNotExists( - new TopicPartitionInfo(queue.getTopic(), queue.getTenantId(), i, false).getFullTopicName()); + new TopicPartitionInfo(queue.getTopic(), queue.getTenantId(), i, false).getFullTopicName(), + queue.getCustomProperties() + ); } tbClusterService.onQueueChange(queue); @@ -111,7 +113,9 @@ public class DefaultTbQueueService extends AbstractTbEntityService implements Tb log.info("Added [{}] new partitions to [{}] queue", currentPartitions - oldPartitions, queue.getName()); for (int i = oldPartitions; i < currentPartitions; i++) { tbQueueAdmin.createTopicIfNotExists( - new TopicPartitionInfo(queue.getTopic(), queue.getTenantId(), i, false).getFullTopicName()); + new TopicPartitionInfo(queue.getTopic(), queue.getTenantId(), i, false).getFullTopicName(), + queue.getCustomProperties() + ); } tbClusterService.onQueueChange(queue); } else { diff --git a/common/cluster-api/src/main/java/org/thingsboard/server/queue/TbQueueAdmin.java b/common/cluster-api/src/main/java/org/thingsboard/server/queue/TbQueueAdmin.java index 4b2bde733e..19aa0284ea 100644 --- a/common/cluster-api/src/main/java/org/thingsboard/server/queue/TbQueueAdmin.java +++ b/common/cluster-api/src/main/java/org/thingsboard/server/queue/TbQueueAdmin.java @@ -17,7 +17,11 @@ package org.thingsboard.server.queue; public interface TbQueueAdmin { - void createTopicIfNotExists(String topic); + default void createTopicIfNotExists(String topic) { + createTopicIfNotExists(topic, null); + } + + void createTopicIfNotExists(String topic, String properties); void destroy(); diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/queue/Queue.java b/common/data/src/main/java/org/thingsboard/server/common/data/queue/Queue.java index f757998d04..deb0e57f6b 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/queue/Queue.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/queue/Queue.java @@ -15,6 +15,8 @@ */ package org.thingsboard.server.common.data.queue; +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.fasterxml.jackson.databind.JsonNode; import lombok.Data; import org.thingsboard.server.common.data.HasName; import org.thingsboard.server.common.data.HasTenantId; @@ -25,6 +27,8 @@ import org.thingsboard.server.common.data.tenant.profile.TenantProfileQueueConfi import org.thingsboard.server.common.data.validation.Length; import org.thingsboard.server.common.data.validation.NoXss; +import java.util.Optional; + @Data public class Queue extends SearchTextBasedWithAdditionalInfo implements HasName, HasTenantId { private TenantId tenantId; @@ -65,4 +69,12 @@ public class Queue extends SearchTextBasedWithAdditionalInfo implements public String getSearchText() { return getName(); } -} \ No newline at end of file + + @JsonIgnore + public String getCustomProperties() { + return Optional.ofNullable(getAdditionalInfo()) + .map(info -> info.get("customProperties")) + .filter(JsonNode::isTextual).map(JsonNode::asText).orElse(null); + } + +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/RuleEngineTbQueueAdminFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/RuleEngineTbQueueAdminFactory.java index fe29c3a04f..7a8764325f 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/RuleEngineTbQueueAdminFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/RuleEngineTbQueueAdminFactory.java @@ -99,7 +99,7 @@ public class RuleEngineTbQueueAdminFactory { return new TbQueueAdmin() { @Override - public void createTopicIfNotExists(String topic) { + public void createTopicIfNotExists(String topic, String properties) { } @Override diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/azure/servicebus/TbServiceBusAdmin.java b/common/queue/src/main/java/org/thingsboard/server/queue/azure/servicebus/TbServiceBusAdmin.java index e171bb7a31..d95d2064a4 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/azure/servicebus/TbServiceBusAdmin.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/azure/servicebus/TbServiceBusAdmin.java @@ -22,6 +22,7 @@ import com.microsoft.azure.servicebus.primitives.MessagingEntityAlreadyExistsExc import com.microsoft.azure.servicebus.primitives.ServiceBusException; import lombok.extern.slf4j.Slf4j; import org.thingsboard.server.queue.TbQueueAdmin; +import org.thingsboard.server.queue.util.PropertyUtils; import java.io.IOException; import java.time.Duration; @@ -60,7 +61,7 @@ public class TbServiceBusAdmin implements TbQueueAdmin { } @Override - public void createTopicIfNotExists(String topic) { + public void createTopicIfNotExists(String topic, String properties) { if (queues.contains(topic)) { return; } @@ -68,7 +69,7 @@ public class TbServiceBusAdmin implements TbQueueAdmin { try { QueueDescription queueDescription = new QueueDescription(topic); queueDescription.setRequiresDuplicateDetection(false); - setQueueConfigs(queueDescription); + setQueueConfigs(queueDescription, PropertyUtils.getProps(queueConfigs, properties)); client.createQueue(queueDescription); queues.add(topic); @@ -107,7 +108,7 @@ public class TbServiceBusAdmin implements TbQueueAdmin { } } - private void setQueueConfigs(QueueDescription queueDescription) { + private void setQueueConfigs(QueueDescription queueDescription, Map queueConfigs) { queueConfigs.forEach((confKey, confValue) -> { switch (confKey) { case MAX_SIZE: diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaAdmin.java b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaAdmin.java index f15b9258e8..d486d04783 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaAdmin.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaAdmin.java @@ -21,6 +21,7 @@ import org.apache.kafka.clients.admin.CreateTopicsResult; import org.apache.kafka.clients.admin.NewTopic; import org.apache.kafka.common.errors.TopicExistsException; import org.thingsboard.server.queue.TbQueueAdmin; +import org.thingsboard.server.queue.util.PropertyUtils; import java.util.Collections; import java.util.Map; @@ -62,12 +63,12 @@ public class TbKafkaAdmin implements TbQueueAdmin { } @Override - public void createTopicIfNotExists(String topic) { + public void createTopicIfNotExists(String topic, String properties) { if (topics.contains(topic)) { return; } try { - NewTopic newTopic = new NewTopic(topic, numPartitions, replicationFactor).configs(topicConfigs); + NewTopic newTopic = new NewTopic(topic, numPartitions, replicationFactor).configs(PropertyUtils.getProps(topicConfigs, properties)); createTopic(newTopic).values().get(topic).get(); topics.add(topic); } catch (ExecutionException ee) { @@ -81,7 +82,6 @@ public class TbKafkaAdmin implements TbQueueAdmin { log.warn("[{}] Failed to create topic", topic, e); throw new RuntimeException(e); } - } @Override diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/InMemoryTbTransportQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/InMemoryTbTransportQueueFactory.java index 60c464ecab..4d0089457d 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/InMemoryTbTransportQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/InMemoryTbTransportQueueFactory.java @@ -73,7 +73,7 @@ public class InMemoryTbTransportQueueFactory implements TbTransportQueueFactory templateBuilder.queueAdmin(new TbQueueAdmin() { @Override - public void createTopicIfNotExists(String topic) {} + public void createTopicIfNotExists(String topic, String properties) {} @Override public void destroy() {} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/pubsub/TbPubSubAdmin.java b/common/queue/src/main/java/org/thingsboard/server/queue/pubsub/TbPubSubAdmin.java index d1a4942ad3..f9f20c2448 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/pubsub/TbPubSubAdmin.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/pubsub/TbPubSubAdmin.java @@ -103,7 +103,7 @@ public class TbPubSubAdmin implements TbQueueAdmin { } @Override - public void createTopicIfNotExists(String partition) { + public void createTopicIfNotExists(String partition, String properties) { TopicName topicName = TopicName.newBuilder() .setTopic(partition) .setProject(pubSubSettings.getProjectId()) diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/rabbitmq/TbRabbitMqAdmin.java b/common/queue/src/main/java/org/thingsboard/server/queue/rabbitmq/TbRabbitMqAdmin.java index 00a2ee4c6c..fb646f383a 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/rabbitmq/TbRabbitMqAdmin.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/rabbitmq/TbRabbitMqAdmin.java @@ -18,9 +18,11 @@ package org.thingsboard.server.queue.rabbitmq; import com.rabbitmq.client.Channel; import com.rabbitmq.client.Connection; import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.StringUtils; import org.thingsboard.server.queue.TbQueueAdmin; import java.io.IOException; +import java.util.HashMap; import java.util.Map; import java.util.concurrent.TimeoutException; @@ -50,7 +52,12 @@ public class TbRabbitMqAdmin implements TbQueueAdmin { } @Override - public void createTopicIfNotExists(String topic) { + public void createTopicIfNotExists(String topic, String properties) { + Map arguments = this.arguments; + if (StringUtils.isNotBlank(properties)) { + arguments = new HashMap<>(arguments); + arguments.putAll(TbRabbitMqQueueArguments.getArgs(properties)); + } try { channel.queueDeclare(topic, false, false, false, arguments); } catch (IOException e) { diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/rabbitmq/TbRabbitMqQueueArguments.java b/common/queue/src/main/java/org/thingsboard/server/queue/rabbitmq/TbRabbitMqQueueArguments.java index cb96abdf3c..8fa8c537e6 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/rabbitmq/TbRabbitMqQueueArguments.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/rabbitmq/TbRabbitMqQueueArguments.java @@ -65,7 +65,7 @@ public class TbRabbitMqQueueArguments { vcArgs = getArgs(vcProperties); } - private Map getArgs(String properties) { + public static Map getArgs(String properties) { Map configs = new HashMap<>(); if (StringUtils.isNotEmpty(properties)) { for (String property : properties.split(";")) { @@ -78,7 +78,7 @@ public class TbRabbitMqQueueArguments { return configs; } - private Object getObjectValue(String str) { + private static Object getObjectValue(String str) { if (str.equalsIgnoreCase("true") || str.equalsIgnoreCase("false")) { return Boolean.valueOf(str); } else if (isNumeric(str)) { @@ -87,7 +87,7 @@ public class TbRabbitMqQueueArguments { return str; } - private Object getNumericValue(String str) { + private static Object getNumericValue(String str) { if (str.contains(".")) { return Double.valueOf(str); } else { @@ -97,7 +97,7 @@ public class TbRabbitMqQueueArguments { private static final Pattern PATTERN = Pattern.compile("-?\\d+(\\.\\d+)?"); - public boolean isNumeric(String strNum) { + private static boolean isNumeric(String strNum) { if (strNum == null) { return false; } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/sqs/TbAwsSqsAdmin.java b/common/queue/src/main/java/org/thingsboard/server/queue/sqs/TbAwsSqsAdmin.java index f88a34941a..ba4eeb6ca4 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/sqs/TbAwsSqsAdmin.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/sqs/TbAwsSqsAdmin.java @@ -26,6 +26,7 @@ import com.amazonaws.services.sqs.model.CreateQueueRequest; import com.amazonaws.services.sqs.model.GetQueueUrlResult; import lombok.extern.slf4j.Slf4j; import org.thingsboard.server.queue.TbQueueAdmin; +import org.thingsboard.server.queue.util.PropertyUtils; import java.util.Map; import java.util.function.Function; @@ -63,11 +64,12 @@ public class TbAwsSqsAdmin implements TbQueueAdmin { } @Override - public void createTopicIfNotExists(String topic) { + public void createTopicIfNotExists(String topic, String properties) { String queueName = convertTopicToQueueName(topic); if (queues.containsKey(queueName)) { return; } + Map attributes = PropertyUtils.getProps(this.attributes, properties, TbAwsSqsQueueAttributes::toConfigs); final CreateQueueRequest createQueueRequest = new CreateQueueRequest(queueName).withAttributes(attributes); String queueUrl = sqsClient.createQueue(createQueueRequest).getQueueUrl(); queues.put(getQueueNameFromUrl(queueUrl), queueUrl); diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/sqs/TbAwsSqsQueueAttributes.java b/common/queue/src/main/java/org/thingsboard/server/queue/sqs/TbAwsSqsQueueAttributes.java index 66110ade74..faa8eccc90 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/sqs/TbAwsSqsQueueAttributes.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/sqs/TbAwsSqsQueueAttributes.java @@ -76,6 +76,12 @@ public class TbAwsSqsQueueAttributes { private Map getConfigs(String properties) { Map configs = new HashMap<>(defaultAttributes); + configs.putAll(toConfigs(properties)); + return configs; + } + + public static Map toConfigs(String properties) { + Map configs = new HashMap<>(); if (StringUtils.isNotEmpty(properties)) { for (String property : properties.split(";")) { int delimiterPosition = property.indexOf(":"); @@ -88,7 +94,7 @@ public class TbAwsSqsQueueAttributes { return configs; } - private void validateAttributeName(String key) { + private static void validateAttributeName(String key) { QueueAttributeName.fromValue(key); } } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/util/PropertyUtils.java b/common/queue/src/main/java/org/thingsboard/server/queue/util/PropertyUtils.java index 089d7f2219..afee64f382 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/util/PropertyUtils.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/util/PropertyUtils.java @@ -19,6 +19,7 @@ import org.thingsboard.server.common.data.StringUtils; import java.util.HashMap; import java.util.Map; +import java.util.function.Function; public class PropertyUtils { @@ -37,4 +38,17 @@ public class PropertyUtils { return configs; } + public static Map getProps(Map defaultProperties, String propertiesStr) { + return getProps(defaultProperties, propertiesStr, PropertyUtils::getProps); + } + + public static Map getProps(Map defaultProperties, String propertiesStr, Function> parser) { + Map properties = defaultProperties; + if (StringUtils.isNotBlank(propertiesStr)) { + properties = new HashMap<>(properties); + properties.putAll(parser.apply(propertiesStr)); + } + return properties; + } + } diff --git a/ui-ngx/src/app/modules/home/components/profile/queue/tenant-profile-queues.component.ts b/ui-ngx/src/app/modules/home/components/profile/queue/tenant-profile-queues.component.ts index ac128b3ec5..298b4fe575 100644 --- a/ui-ngx/src/app/modules/home/components/profile/queue/tenant-profile-queues.component.ts +++ b/ui-ngx/src/app/modules/home/components/profile/queue/tenant-profile-queues.component.ts @@ -173,7 +173,8 @@ export class TenantProfileQueuesComponent implements ControlValueAccessor, Valid }, topic: '', additionalInfo: { - description: '' + description: '', + customProperties: '' } }; this.idMap.push(queue.id); diff --git a/ui-ngx/src/app/modules/home/components/profile/tenant-profile.component.ts b/ui-ngx/src/app/modules/home/components/profile/tenant-profile.component.ts index fa8b1636ff..bed7531495 100644 --- a/ui-ngx/src/app/modules/home/components/profile/tenant-profile.component.ts +++ b/ui-ngx/src/app/modules/home/components/profile/tenant-profile.component.ts @@ -74,7 +74,8 @@ export class TenantProfileComponent extends EntityComponent { }, topic: 'tb_rule_engine.main', additionalInfo: { - description: '' + description: '', + customProperties: '' } }, { @@ -97,7 +98,8 @@ export class TenantProfileComponent extends EntityComponent { maxPauseBetweenRetries: 5 }, additionalInfo: { - description: '' + description: '', + customProperties: '' } }, { @@ -120,7 +122,8 @@ export class TenantProfileComponent extends EntityComponent { maxPauseBetweenRetries: 5 }, additionalInfo: { - description: '' + description: '', + customProperties: '' } } ]; diff --git a/ui-ngx/src/app/modules/home/components/queue/queue-form.component.html b/ui-ngx/src/app/modules/home/components/queue/queue-form.component.html index 56b20d8aa8..4845f7f25a 100644 --- a/ui-ngx/src/app/modules/home/components/queue/queue-form.component.html +++ b/ui-ngx/src/app/modules/home/components/queue/queue-form.component.html @@ -203,6 +203,11 @@ + + queue.custom-properties + + queue.custom-properties-hint + queue.description diff --git a/ui-ngx/src/app/modules/home/components/queue/queue-form.component.ts b/ui-ngx/src/app/modules/home/components/queue/queue-form.component.ts index b123cf2ad5..e4fbcd031b 100644 --- a/ui-ngx/src/app/modules/home/components/queue/queue-form.component.ts +++ b/ui-ngx/src/app/modules/home/components/queue/queue-form.component.ts @@ -117,7 +117,8 @@ export class QueueFormComponent implements ControlValueAccessor, OnInit, OnDestr }), topic: [''], additionalInfo: this.fb.group({ - description: [''] + description: [''], + customProperties: [''] }) }); this.valueChange$ = this.queueFormGroup.valueChanges.subscribe(() => { diff --git a/ui-ngx/src/app/shared/models/queue.models.ts b/ui-ngx/src/app/shared/models/queue.models.ts index 76ee0bf022..07a57e68c2 100644 --- a/ui-ngx/src/app/shared/models/queue.models.ts +++ b/ui-ngx/src/app/shared/models/queue.models.ts @@ -121,5 +121,6 @@ export interface QueueInfo extends BaseData { topic: string; additionalInfo: { description?: string; + customProperties?: string; }; } diff --git a/ui-ngx/src/assets/locale/locale.constant-en_US.json b/ui-ngx/src/assets/locale/locale.constant-en_US.json index c9d236bf2a..c1092f9dad 100644 --- a/ui-ngx/src/assets/locale/locale.constant-en_US.json +++ b/ui-ngx/src/assets/locale/locale.constant-en_US.json @@ -3465,6 +3465,8 @@ "description": "Description", "description-hint": "This text will be displayed in the Queue description instead of the selected strategy", "alt-description": "Submit Strategy: {{submitStrategy}}, Processing Strategy: {{processingStrategy}}", + "custom-properties": "Custom properties", + "custom-properties-hint": "Custom queue (topic) creation properties, e.g. 'retention.ms:604800000;retention.bytes:1048576000'", "strategies": { "sequential-by-originator-label": "Sequential by originator", "sequential-by-originator-hint": "New message for e.g. device A is not submitted until previous message for device A is acknowledged", From c0873590772c76abe149a2cd06c1727ebefe7461 Mon Sep 17 00:00:00 2001 From: ViacheslavKlimov Date: Tue, 1 Aug 2023 13:57:32 +0300 Subject: [PATCH 14/23] Move messages to other queue on deletion; improvements --- .../entitiy/queue/DefaultTbQueueService.java | 30 +------- .../queue/DefaultTbClusterService.java | 10 +-- .../DefaultTbRuleEngineConsumerService.java | 71 ++++++++++++++++--- .../src/main/resources/thingsboard.yml | 2 - .../server/queue/TbQueueConsumer.java | 6 ++ .../AbstractTbQueueConsumerTemplate.java | 18 ++++- .../server/queue/kafka/TbKafkaSettings.java | 1 + .../queue/memory/InMemoryTbQueueConsumer.java | 14 ++++ 8 files changed, 107 insertions(+), 45 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/entitiy/queue/DefaultTbQueueService.java b/application/src/main/java/org/thingsboard/server/service/entitiy/queue/DefaultTbQueueService.java index 9e3d38cf32..72b8c2252a 100644 --- a/application/src/main/java/org/thingsboard/server/service/entitiy/queue/DefaultTbQueueService.java +++ b/application/src/main/java/org/thingsboard/server/service/entitiy/queue/DefaultTbQueueService.java @@ -17,7 +17,6 @@ package org.thingsboard.server.service.entitiy.queue; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; -import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Service; import org.thingsboard.server.cluster.TbClusterService; import org.thingsboard.server.common.data.TenantProfile; @@ -29,7 +28,6 @@ import org.thingsboard.server.common.data.tenant.profile.TenantProfileQueueConfi import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; import org.thingsboard.server.dao.queue.QueueService; import org.thingsboard.server.queue.TbQueueAdmin; -import org.thingsboard.server.queue.scheduler.SchedulerComponent; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.entitiy.AbstractTbEntityService; @@ -37,7 +35,6 @@ import java.util.ArrayList; import java.util.Collections; import java.util.List; import java.util.Map; -import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; @Slf4j @@ -49,10 +46,6 @@ public class DefaultTbQueueService extends AbstractTbEntityService implements Tb private final QueueService queueService; private final TbClusterService tbClusterService; private final TbQueueAdmin tbQueueAdmin; - private final SchedulerComponent scheduler; - - @Value("${queue.rule-engine.topic_deletion_delay:60}") - private int topicDeletionDelay; @Override public Queue saveQueue(Queue queue) { @@ -121,14 +114,7 @@ public class DefaultTbQueueService extends AbstractTbEntityService implements Tb } else { log.info("Removed [{}] partitions from [{}] queue", oldPartitions - currentPartitions, queue.getName()); tbClusterService.onQueueChange(queue); - - scheduler.schedule(() -> { - for (int i = currentPartitions; i < oldPartitions; i++) { - String fullTopicName = new TopicPartitionInfo(queue.getTopic(), queue.getTenantId(), i, false).getFullTopicName(); - log.info("Removed partition [{}]", fullTopicName); - tbQueueAdmin.deleteTopic(fullTopicName); - } - }, topicDeletionDelay, TimeUnit.SECONDS); + // TODO: move all the messages left in old partitions and delete topics } } else if (!oldQueue.equals(queue)) { tbClusterService.onQueueChange(queue); @@ -137,21 +123,7 @@ public class DefaultTbQueueService extends AbstractTbEntityService implements Tb private void onQueueDeleted(Queue queue) { tbClusterService.onQueueDelete(queue); - // queueStatsService.deleteQueueStatsByQueueId(tenantId, queueId); - - scheduler.schedule(() -> { - for (int i = 0; i < queue.getPartitions(); i++) { - String fullTopicName = new TopicPartitionInfo(queue.getTopic(), queue.getTenantId(), i, false).getFullTopicName(); - log.info("Deleting queue [{}]", fullTopicName); - try { - tbQueueAdmin.deleteTopic(fullTopicName); - } catch (Exception e) { - log.error("Failed to delete queue [{}]", fullTopicName); - } - } - }, topicDeletionDelay, TimeUnit.SECONDS); - notificationEntityService.notifySendMsgToEdgeService(queue.getTenantId(), queue.getId(), EdgeEventActionType.DELETED); } diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java index 76631aaa95..5a2db8019e 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java @@ -591,11 +591,6 @@ public class DefaultTbClusterService implements TbClusterService { tbTransportServices.removeAll(tbCoreServices); tbCoreServices.removeAll(tbRuleEngineServices); - for (String ruleEngineServiceId : tbRuleEngineServices) { - TopicPartitionInfo tpi = notificationsTopicService.getNotificationsTopic(ServiceType.TB_RULE_ENGINE, ruleEngineServiceId); - producerProvider.getRuleEngineNotificationsMsgProducer().send(tpi, new TbProtoQueueMsg<>(UUID.randomUUID(), ruleEngineMsg), null); - toRuleEngineNfs.incrementAndGet(); - } for (String coreServiceId : tbCoreServices) { TopicPartitionInfo tpi = notificationsTopicService.getNotificationsTopic(ServiceType.TB_CORE, coreServiceId); producerProvider.getTbCoreNotificationsMsgProducer().send(tpi, new TbProtoQueueMsg<>(UUID.randomUUID(), coreMsg), null); @@ -606,5 +601,10 @@ public class DefaultTbClusterService implements TbClusterService { producerProvider.getTransportNotificationsMsgProducer().send(tpi, new TbProtoQueueMsg<>(UUID.randomUUID(), transportMsg), null); toTransportNfs.incrementAndGet(); } + for (String ruleEngineServiceId : tbRuleEngineServices) { + TopicPartitionInfo tpi = notificationsTopicService.getNotificationsTopic(ServiceType.TB_RULE_ENGINE, ruleEngineServiceId); + producerProvider.getRuleEngineNotificationsMsgProducer().send(tpi, new TbProtoQueueMsg<>(UUID.randomUUID(), ruleEngineMsg), null); + toRuleEngineNfs.incrementAndGet(); + } } } diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbRuleEngineConsumerService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbRuleEngineConsumerService.java index 598b41035f..ffcbc6ff60 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbRuleEngineConsumerService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbRuleEngineConsumerService.java @@ -23,11 +23,14 @@ import org.springframework.scheduling.annotation.Scheduled; import org.springframework.stereotype.Service; import org.thingsboard.common.util.ThingsBoardThreadFactory; import org.thingsboard.server.actors.ActorSystemContext; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.EntityIdFactory; import org.thingsboard.server.common.data.id.QueueId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.queue.Queue; import org.thingsboard.server.common.data.rpc.RpcError; import org.thingsboard.server.common.msg.TbMsg; +import org.thingsboard.server.common.msg.gen.MsgProtos; import org.thingsboard.server.common.msg.queue.QueueToRuleEngineMsg; import org.thingsboard.server.common.msg.queue.RuleEngineException; import org.thingsboard.server.common.msg.queue.RuleNodeInfo; @@ -42,12 +45,14 @@ import org.thingsboard.server.dao.tenant.TbTenantProfileCache; import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineNotificationMsg; +import org.thingsboard.server.queue.TbQueueAdmin; import org.thingsboard.server.queue.TbQueueConsumer; import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.queue.discovery.PartitionService; import org.thingsboard.server.queue.discovery.QueueKey; import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; import org.thingsboard.server.queue.discovery.event.PartitionChangeEvent; +import org.thingsboard.server.queue.provider.TbQueueProducerProvider; import org.thingsboard.server.queue.provider.TbRuleEngineQueueFactory; import org.thingsboard.server.queue.util.DataDecodingEncodingService; import org.thingsboard.server.queue.util.TbRuleEngineComponent; @@ -107,13 +112,14 @@ public class DefaultTbRuleEngineConsumerService extends AbstractConsumerService< private final TbRuleEngineDeviceRpcService tbDeviceRpcService; private final TbServiceInfoProvider serviceInfoProvider; private final QueueService queueService; - // private final TenantId tenantId; + private final TbQueueProducerProvider producerProvider; + private final TbQueueAdmin queueAdmin; private final ConcurrentMap>> consumers = new ConcurrentHashMap<>(); private final ConcurrentMap consumerConfigurations = new ConcurrentHashMap<>(); private final ConcurrentMap consumerStats = new ConcurrentHashMap<>(); private final ConcurrentMap topicsConsumerPerPartition = new ConcurrentHashMap<>(); final ExecutorService submitExecutor = Executors.newSingleThreadExecutor(ThingsBoardThreadFactory.forName("tb-rule-engine-consumer-submit")); - final ScheduledExecutorService repartitionExecutor = Executors.newScheduledThreadPool(1, ThingsBoardThreadFactory.forName("tb-rule-engine-consumer-repartition")); + final ScheduledExecutorService repartitionExecutor = Executors.newScheduledThreadPool(2, ThingsBoardThreadFactory.forName("tb-rule-engine-consumer-repartition")); public DefaultTbRuleEngineConsumerService(TbRuleEngineProcessingStrategyFactory processingStrategyFactory, TbRuleEngineSubmitStrategyFactory submitStrategyFactory, @@ -128,7 +134,8 @@ public class DefaultTbRuleEngineConsumerService extends AbstractConsumerService< TbTenantProfileCache tenantProfileCache, TbApiUsageStateService apiUsageStateService, PartitionService partitionService, ApplicationEventPublisher eventPublisher, - TbServiceInfoProvider serviceInfoProvider, QueueService queueService) { + TbServiceInfoProvider serviceInfoProvider, QueueService queueService, + TbQueueProducerProvider producerProvider, TbQueueAdmin queueAdmin) { super(actorContext, encodingService, tenantProfileCache, deviceProfileCache, assetProfileCache, apiUsageStateService, partitionService, eventPublisher, tbRuleEngineQueueFactory.createToRuleEngineNotificationsMsgConsumer(), Optional.empty()); this.statisticsService = statisticsService; this.tbRuleEngineQueueFactory = tbRuleEngineQueueFactory; @@ -138,6 +145,8 @@ public class DefaultTbRuleEngineConsumerService extends AbstractConsumerService< this.statsFactory = statsFactory; this.serviceInfoProvider = serviceInfoProvider; this.queueService = queueService; + this.producerProvider = producerProvider; + this.queueAdmin = queueAdmin; } @PostConstruct @@ -230,7 +239,6 @@ public class DefaultTbRuleEngineConsumerService extends AbstractConsumerService< launchConsumer(consumer, consumerConfigurations.get(queueKey), consumerStats.get(queueKey), "" + queueKey + "-" + tpi.getPartition().orElse(-999999)); consumer.subscribe(Collections.singleton(tpi)); }); - } finally { tbTopicWithConsumerPerPartition.getLock().unlock(); } @@ -264,7 +272,7 @@ public class DefaultTbRuleEngineConsumerService extends AbstractConsumerService< void consumerLoop(TbQueueConsumer> consumer, org.thingsboard.server.common.data.queue.Queue configuration, TbRuleEngineConsumerStats stats, String threadSuffix) { updateCurrentThreadName(threadSuffix); - while (!stopped && !consumer.isStopped()) { + while (!stopped && !consumer.isStopped() && !consumer.isDeleted()) { try { List> msgs = consumer.poll(configuration.getPollInterval()); if (msgs.isEmpty()) { @@ -314,6 +322,10 @@ public class DefaultTbRuleEngineConsumerService extends AbstractConsumerService< } } } + + if (consumer.isDeleted()) { + processQueueDeletion(configuration, consumer); + } log.info("TB Rule Engine Consumer stopped."); } @@ -448,22 +460,22 @@ public class DefaultTbRuleEngineConsumerService extends AbstractConsumerService< TenantId tenantId = new TenantId(new UUID(queueDeleteMsg.getTenantIdMSB(), queueDeleteMsg.getTenantIdLSB())); QueueKey queueKey = new QueueKey(ServiceType.TB_RULE_ENGINE, queueDeleteMsg.getQueueName(), tenantId); + partitionService.removeQueue(queueDeleteMsg); Queue queue = consumerConfigurations.remove(queueKey); if (queue != null) { if (queue.isConsumerPerPartition()) { TbTopicWithConsumerPerPartition tbTopicWithConsumerPerPartition = topicsConsumerPerPartition.remove(queueKey); if (tbTopicWithConsumerPerPartition != null) { - tbTopicWithConsumerPerPartition.getConsumers().values().forEach(TbQueueConsumer::unsubscribe); + tbTopicWithConsumerPerPartition.getConsumers().values().forEach(TbQueueConsumer::onQueueDelete); tbTopicWithConsumerPerPartition.getConsumers().clear(); } } else { TbQueueConsumer> consumer = consumers.remove(queueKey); if (consumer != null) { - consumer.unsubscribe(); + consumer.onQueueDelete(); } } } - partitionService.removeQueue(queueDeleteMsg); } private void forwardToRuleEngineActor(String queueName, TenantId tenantId, ToRuleEngineMsg toRuleEngineMsg, TbMsgCallback callback) { @@ -482,6 +494,49 @@ public class DefaultTbRuleEngineConsumerService extends AbstractConsumerService< actorContext.tell(msg); } + private void processQueueDeletion(Queue queue, TbQueueConsumer> consumer) { + long startTs = System.currentTimeMillis(); + long timeout = TimeUnit.SECONDS.toMillis(30); + try { + int n = 0; + while ((System.currentTimeMillis() - startTs <= timeout)) { + List> msgs = consumer.poll(queue.getPollInterval()); + if (!msgs.isEmpty()) { + for (TbProtoQueueMsg msg : msgs) { + try { + MsgProtos.TbMsgProto tbMsgProto = MsgProtos.TbMsgProto.parseFrom(msg.getValue().getTbMsg().toByteArray()); + EntityId originator = EntityIdFactory.getByTypeAndUuid(tbMsgProto.getEntityType(), new UUID(tbMsgProto.getEntityIdMSB(), tbMsgProto.getEntityIdLSB())); + + TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_RULE_ENGINE, queue.getName(), TenantId.SYS_TENANT_ID, originator); + producerProvider.getRuleEngineMsgProducer().send(tpi, msg, null); + n++; + } catch (Throwable e) { + log.debug("Failed to move message to system {}: {}", consumer.getTopic(), msg, e); + } + } + consumer.commit(); + } else { + break; + } + } + if (n > 0) { + log.info("Moved {} messages from {} to system {}", n, consumer.getFullTopicNames(), consumer.getTopic()); + } + + consumer.unsubscribe(); + for (String topic : consumer.getFullTopicNames()) { + try { + queueAdmin.deleteTopic(topic); + log.info("Deleted topic {}", topic); + } catch (Exception e) { + log.error("Failed to delete topic {} after unsubscribing", topic, e); + } + } + } catch (Exception e) { + log.error("Failed to process deletion of {} ({})", consumer.getTopic(), queue.getTenantId(), e); + } + } + @Scheduled(fixedDelayString = "${queue.rule-engine.stats.print-interval-ms}") public void printStats() { if (statsEnabled) { diff --git a/application/src/main/resources/thingsboard.yml b/application/src/main/resources/thingsboard.yml index 96ee2793ed..faf1ebcbea 100644 --- a/application/src/main/resources/thingsboard.yml +++ b/application/src/main/resources/thingsboard.yml @@ -1233,8 +1233,6 @@ queue: failure-percentage: "${TB_QUEUE_RE_SQ_PROCESSING_STRATEGY_FAILURE_PERCENTAGE:0}" # Skip retry if failures or timeouts are less then X percentage of messages; pause-between-retries: "${TB_QUEUE_RE_SQ_PROCESSING_STRATEGY_RETRY_PAUSE:5}" # Time in seconds to wait in consumer thread before retries; max-pause-between-retries: "${TB_QUEUE_RE_SQ_PROCESSING_STRATEGY_MAX_RETRY_PAUSE:5}" # Max allowed time in seconds for pause between retries. - # Delay between Queue update/delete and actual topic deletion. The delay is for Rule Engines to have time to unsubscribe from the topics, and for other services to stop publishing - topic_deletion_delay: "${TB_QUEUE_RULE_ENGINE_TOPIC_DELETION_DELAY_SECS:180}" transport: # For high priority notifications that require minimum latency and processing time notifications_topic: "${TB_QUEUE_TRANSPORT_NOTIFICATIONS_TOPIC:tb_transport.notifications}" diff --git a/common/cluster-api/src/main/java/org/thingsboard/server/queue/TbQueueConsumer.java b/common/cluster-api/src/main/java/org/thingsboard/server/queue/TbQueueConsumer.java index 04439fc85d..73bea7642f 100644 --- a/common/cluster-api/src/main/java/org/thingsboard/server/queue/TbQueueConsumer.java +++ b/common/cluster-api/src/main/java/org/thingsboard/server/queue/TbQueueConsumer.java @@ -36,4 +36,10 @@ public interface TbQueueConsumer { boolean isStopped(); + void onQueueDelete(); + + boolean isDeleted(); + + List getFullTopicNames(); + } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/common/AbstractTbQueueConsumerTemplate.java b/common/queue/src/main/java/org/thingsboard/server/queue/common/AbstractTbQueueConsumerTemplate.java index 5b0e84c2ed..8e91c1d134 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/common/AbstractTbQueueConsumerTemplate.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/common/AbstractTbQueueConsumerTemplate.java @@ -44,6 +44,7 @@ public abstract class AbstractTbQueueConsumerTemplate i protected volatile Set partitions; protected final ReentrantLock consumerLock = new ReentrantLock(); //NonfairSync final Queue> subscribeQueue = new ConcurrentLinkedQueue<>(); + protected volatile boolean deleted = false; @Getter private final String topic; @@ -94,7 +95,7 @@ public abstract class AbstractTbQueueConsumerTemplate i partitions = subscribeQueue.poll(); } if (!subscribed) { - List topicNames = partitions.stream().map(TopicPartitionInfo::getFullTopicName).collect(Collectors.toList()); + List topicNames = getFullTopicNames(); doSubscribe(topicNames); subscribed = true; } @@ -191,6 +192,21 @@ public abstract class AbstractTbQueueConsumerTemplate i abstract protected void doUnsubscribe(); + @Override + public void onQueueDelete() { + deleted = true; + } + + @Override + public boolean isDeleted() { + return deleted; + } + + @Override + public List getFullTopicNames() { + return partitions.stream().map(TopicPartitionInfo::getFullTopicName).collect(Collectors.toList()); + } + protected boolean isLongPollingSupported() { return false; } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaSettings.java b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaSettings.java index d2a54128e3..55f1721c46 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaSettings.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaSettings.java @@ -159,6 +159,7 @@ public class TbKafkaSettings { props.put(ConsumerConfig.FETCH_MAX_BYTES_CONFIG, fetchMaxBytes); props.put(ConsumerConfig.MAX_POLL_INTERVAL_MS_CONFIG, maxPollIntervalMs); props.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, autoOffsetReset); + props.put(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG, false); props.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class); props.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, ByteArrayDeserializer.class); diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/memory/InMemoryTbQueueConsumer.java b/common/queue/src/main/java/org/thingsboard/server/queue/memory/InMemoryTbQueueConsumer.java index 081202315e..a642d7e9cf 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/memory/InMemoryTbQueueConsumer.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/memory/InMemoryTbQueueConsumer.java @@ -103,4 +103,18 @@ public class InMemoryTbQueueConsumer implements TbQueueCon return stopped; } + @Override + public void onQueueDelete() { + } + + @Override + public boolean isDeleted() { + return false; + } + + @Override + public List getFullTopicNames() { + return partitions.stream().map(TopicPartitionInfo::getFullTopicName).collect(Collectors.toList()); + } + } From 0f13d4614473c056d703b7443c50bea404472c70 Mon Sep 17 00:00:00 2001 From: ViacheslavKlimov Date: Thu, 3 Aug 2023 13:54:59 +0300 Subject: [PATCH 15/23] Add more tests --- .../DefaultTbRuleEngineConsumerService.java | 36 +++---- .../src/main/resources/thingsboard.yml | 2 + .../controller/BaseTenantControllerTest.java | 100 +++++++++++++++--- .../queue/memory/InMemoryTbQueueConsumer.java | 4 +- 4 files changed, 110 insertions(+), 32 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbRuleEngineConsumerService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbRuleEngineConsumerService.java index ffcbc6ff60..863f0354c2 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbRuleEngineConsumerService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbRuleEngineConsumerService.java @@ -103,6 +103,8 @@ public class DefaultTbRuleEngineConsumerService extends AbstractConsumerService< private boolean statsEnabled; @Value("${queue.rule-engine.prometheus-stats.enabled:false}") boolean prometheusStatsEnabled; + @Value("${queue.rule-engine.topic-deletion-delay:30}") + private int topicDeletionDelay; private final StatsFactory statsFactory; private final TbRuleEngineSubmitStrategyFactory submitStrategyFactory; @@ -495,29 +497,27 @@ public class DefaultTbRuleEngineConsumerService extends AbstractConsumerService< } private void processQueueDeletion(Queue queue, TbQueueConsumer> consumer) { - long startTs = System.currentTimeMillis(); - long timeout = TimeUnit.SECONDS.toMillis(30); + long finishTs = System.currentTimeMillis() + TimeUnit.SECONDS.toMillis(topicDeletionDelay); try { int n = 0; - while ((System.currentTimeMillis() - startTs <= timeout)) { + while (System.currentTimeMillis() <= finishTs) { List> msgs = consumer.poll(queue.getPollInterval()); - if (!msgs.isEmpty()) { - for (TbProtoQueueMsg msg : msgs) { - try { - MsgProtos.TbMsgProto tbMsgProto = MsgProtos.TbMsgProto.parseFrom(msg.getValue().getTbMsg().toByteArray()); - EntityId originator = EntityIdFactory.getByTypeAndUuid(tbMsgProto.getEntityType(), new UUID(tbMsgProto.getEntityIdMSB(), tbMsgProto.getEntityIdLSB())); - - TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_RULE_ENGINE, queue.getName(), TenantId.SYS_TENANT_ID, originator); - producerProvider.getRuleEngineMsgProducer().send(tpi, msg, null); - n++; - } catch (Throwable e) { - log.debug("Failed to move message to system {}: {}", consumer.getTopic(), msg, e); - } + if (msgs.isEmpty()) { + continue; + } + for (TbProtoQueueMsg msg : msgs) { + try { + MsgProtos.TbMsgProto tbMsgProto = MsgProtos.TbMsgProto.parseFrom(msg.getValue().getTbMsg().toByteArray()); + EntityId originator = EntityIdFactory.getByTypeAndUuid(tbMsgProto.getEntityType(), new UUID(tbMsgProto.getEntityIdMSB(), tbMsgProto.getEntityIdLSB())); + + TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_RULE_ENGINE, queue.getName(), TenantId.SYS_TENANT_ID, originator); + producerProvider.getRuleEngineMsgProducer().send(tpi, msg, null); + n++; + } catch (Throwable e) { + log.debug("Failed to move message to system {}: {}", consumer.getTopic(), msg, e); } - consumer.commit(); - } else { - break; } + consumer.commit(); } if (n > 0) { log.info("Moved {} messages from {} to system {}", n, consumer.getFullTopicNames(), consumer.getTopic()); diff --git a/application/src/main/resources/thingsboard.yml b/application/src/main/resources/thingsboard.yml index faf1ebcbea..7754159992 100644 --- a/application/src/main/resources/thingsboard.yml +++ b/application/src/main/resources/thingsboard.yml @@ -1233,6 +1233,8 @@ queue: failure-percentage: "${TB_QUEUE_RE_SQ_PROCESSING_STRATEGY_FAILURE_PERCENTAGE:0}" # Skip retry if failures or timeouts are less then X percentage of messages; pause-between-retries: "${TB_QUEUE_RE_SQ_PROCESSING_STRATEGY_RETRY_PAUSE:5}" # Time in seconds to wait in consumer thread before retries; max-pause-between-retries: "${TB_QUEUE_RE_SQ_PROCESSING_STRATEGY_MAX_RETRY_PAUSE:5}" # Max allowed time in seconds for pause between retries. + # After a queue is deleted (or profile's isolation option was disabled), Rule Engine will continue reading related topics during this period, before deleting the actual topics + topic-deletion-delay: "${TB_QUEUE_RULE_ENGINE_TOPIC_DELETION_DELAY_SEC:30}" transport: # For high priority notifications that require minimum latency and processing time notifications_topic: "${TB_QUEUE_TRANSPORT_NOTIFICATIONS_TOPIC:tb_transport.notifications}" diff --git a/application/src/test/java/org/thingsboard/server/controller/BaseTenantControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/BaseTenantControllerTest.java index 85e3fa35bd..db5d804a54 100644 --- a/application/src/test/java/org/thingsboard/server/controller/BaseTenantControllerTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/BaseTenantControllerTest.java @@ -55,18 +55,24 @@ import org.thingsboard.server.common.data.tenant.profile.DefaultTenantProfileCon import org.thingsboard.server.common.data.tenant.profile.TenantProfileData; import org.thingsboard.server.common.data.tenant.profile.TenantProfileQueueConfiguration; import org.thingsboard.server.common.msg.TbMsg; +import org.thingsboard.server.common.msg.TbMsgMetaData; import org.thingsboard.server.common.msg.queue.QueueToRuleEngineMsg; import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; -import org.thingsboard.server.dao.service.DaoSqlTest; +import org.thingsboard.server.gen.transport.TransportProtos; +import org.thingsboard.server.queue.TbQueueAdmin; import org.thingsboard.server.queue.discovery.PartitionService; import java.util.ArrayList; +import java.util.Collections; import java.util.Comparator; +import java.util.Deque; import java.util.HashMap; +import java.util.LinkedList; import java.util.List; import java.util.Map; import java.util.Random; +import java.util.UUID; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicReference; import java.util.function.Consumer; @@ -77,14 +83,18 @@ import static org.assertj.core.api.Assertions.assertThat; import static org.awaitility.Awaitility.await; import static org.hamcrest.Matchers.containsString; import static org.mockito.ArgumentMatchers.argThat; +import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.never; import static org.mockito.Mockito.timeout; import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; +import static org.thingsboard.server.common.data.DataConstants.MAIN_QUEUE_NAME; +import static org.thingsboard.server.common.data.DataConstants.MAIN_QUEUE_TOPIC; @TestPropertySource(properties = { "js.evaluator=mock", + "queue.rule-engine.topic-deletion-delay=10" }) @Slf4j public abstract class BaseTenantControllerTest extends AbstractControllerTest { @@ -100,6 +110,8 @@ public abstract class BaseTenantControllerTest extends AbstractControllerTest { private PartitionService partitionService; @SpyBean private ActorSystemContext actorContext; + @SpyBean + private TbQueueAdmin queueAdmin; @Before public void setUp() throws Exception { @@ -110,7 +122,7 @@ public abstract class BaseTenantControllerTest extends AbstractControllerTest { public void tearDown() throws Exception { loginSysAdmin(); for (Queue queue : doGetTypedWithPageLink("/api/queues?serviceType=TB_RULE_ENGINE&", new TypeReference>() {}, new PageLink(100)).getData()) { - if (!queue.getName().equals(DataConstants.MAIN_QUEUE_NAME)) { + if (!queue.getName().equals(MAIN_QUEUE_NAME)) { doDelete("/api/queues/" + queue.getId()).andExpect(status().isOk()); } } @@ -457,7 +469,7 @@ public abstract class BaseTenantControllerTest extends AbstractControllerTest { tenantProfileData.setConfiguration(new DefaultTenantProfileConfiguration()); tenantProfile.setProfileData(tenantProfileData); tenantProfile.setIsolatedTbRuleEngine(true); - addQueueConfig(tenantProfile, DataConstants.MAIN_QUEUE_NAME); + addQueueConfig(tenantProfile, MAIN_QUEUE_NAME); addQueueConfig(tenantProfile, "Test"); tenantProfile = doPost("/api/tenantProfile", tenantProfile, TenantProfile.class); @@ -486,7 +498,7 @@ public abstract class BaseTenantControllerTest extends AbstractControllerTest { tenantProfileData2.setConfiguration(new DefaultTenantProfileConfiguration()); tenantProfile2.setProfileData(tenantProfileData2); tenantProfile2.setIsolatedTbRuleEngine(true); - addQueueConfig(tenantProfile2, DataConstants.MAIN_QUEUE_NAME); + addQueueConfig(tenantProfile2, MAIN_QUEUE_NAME); addQueueConfig(tenantProfile2, "Test"); addQueueConfig(tenantProfile2, "Test2"); tenantProfile2 = doPost("/api/tenantProfile", tenantProfile2, TenantProfile.class); @@ -571,7 +583,7 @@ public abstract class BaseTenantControllerTest extends AbstractControllerTest { Device hpQueueDevice = createDevice("HP", hpQueueProfile.getName(), "HP"); DeviceProfile mainQueueProfile = createDeviceProfile("Main profile"); - mainQueueProfile.setDefaultQueueName(DataConstants.MAIN_QUEUE_NAME); + mainQueueProfile.setDefaultQueueName(MAIN_QUEUE_NAME); mainQueueProfile = doPost("/api/deviceProfile", mainQueueProfile, DeviceProfile.class); Device mainQueueDevice = createDevice("Main", mainQueueProfile.getName(), "Main"); @@ -581,25 +593,25 @@ public abstract class BaseTenantControllerTest extends AbstractControllerTest { assertThat(usedTpi.getTopic()).isEqualTo(DataConstants.HP_QUEUE_TOPIC); assertThat(usedTpi.getTenantId()).get().isEqualTo(TenantId.SYS_TENANT_ID); }); - verifyUsedQueueAndMessage(DataConstants.MAIN_QUEUE_NAME, tenantId, mainQueueDevice.getId(), DataConstants.ATTRIBUTES_UPDATED, () -> { + verifyUsedQueueAndMessage(MAIN_QUEUE_NAME, tenantId, mainQueueDevice.getId(), DataConstants.ATTRIBUTES_UPDATED, () -> { doPost("/api/plugins/telemetry/DEVICE/" + mainQueueDevice.getId() + "/attributes/SERVER_SCOPE", "{\"test\":123}", String.class); }, usedTpi -> { - assertThat(usedTpi.getTopic()).isEqualTo(DataConstants.MAIN_QUEUE_TOPIC); + assertThat(usedTpi.getTopic()).isEqualTo(MAIN_QUEUE_TOPIC); assertThat(usedTpi.getTenantId()).get().isEqualTo(TenantId.SYS_TENANT_ID); }); loginSysAdmin(); tenantProfile.setIsolatedTbRuleEngine(true); tenantProfile.getProfileData().setQueueConfiguration(List.of( - getQueueConfig(DataConstants.MAIN_QUEUE_NAME, DataConstants.MAIN_QUEUE_TOPIC) + getQueueConfig(MAIN_QUEUE_NAME, MAIN_QUEUE_TOPIC) )); tenantProfile = doPost("/api/tenantProfile", tenantProfile, TenantProfile.class); loginDifferentTenant(); - verifyUsedQueueAndMessage(DataConstants.MAIN_QUEUE_NAME, tenantId, mainQueueDevice.getId(), DataConstants.ATTRIBUTES_UPDATED, () -> { + verifyUsedQueueAndMessage(MAIN_QUEUE_NAME, tenantId, mainQueueDevice.getId(), DataConstants.ATTRIBUTES_UPDATED, () -> { doPost("/api/plugins/telemetry/DEVICE/" + mainQueueDevice.getId() + "/attributes/SERVER_SCOPE", "{\"test\":123}", String.class); }, usedTpi -> { - assertThat(usedTpi.getTopic()).isEqualTo(DataConstants.MAIN_QUEUE_TOPIC); + assertThat(usedTpi.getTopic()).isEqualTo(MAIN_QUEUE_TOPIC); assertThat(usedTpi.getTenantId()).get().isEqualTo(tenantId); }); verifyUsedQueueAndMessage(DataConstants.HP_QUEUE_NAME, tenantId, hpQueueDevice.getId(), DataConstants.ATTRIBUTES_UPDATED, () -> { @@ -612,7 +624,7 @@ public abstract class BaseTenantControllerTest extends AbstractControllerTest { loginSysAdmin(); tenantProfile.setIsolatedTbRuleEngine(true); tenantProfile.getProfileData().setQueueConfiguration(List.of( - getQueueConfig(DataConstants.MAIN_QUEUE_NAME, DataConstants.MAIN_QUEUE_TOPIC), + getQueueConfig(MAIN_QUEUE_NAME, MAIN_QUEUE_TOPIC), getQueueConfig(DataConstants.HP_QUEUE_NAME, DataConstants.HP_QUEUE_TOPIC) )); tenantProfile = doPost("/api/tenantProfile", tenantProfile, TenantProfile.class); @@ -624,14 +636,76 @@ public abstract class BaseTenantControllerTest extends AbstractControllerTest { assertThat(usedTpi.getTopic()).isEqualTo(DataConstants.HP_QUEUE_TOPIC); assertThat(usedTpi.getTenantId()).get().isEqualTo(tenantId); }); - verifyUsedQueueAndMessage(DataConstants.MAIN_QUEUE_NAME, tenantId, mainQueueDevice.getId(), DataConstants.ATTRIBUTES_UPDATED, () -> { + verifyUsedQueueAndMessage(MAIN_QUEUE_NAME, tenantId, mainQueueDevice.getId(), DataConstants.ATTRIBUTES_UPDATED, () -> { doPost("/api/plugins/telemetry/DEVICE/" + mainQueueDevice.getId() + "/attributes/SERVER_SCOPE", "{\"test\":123}", String.class); }, usedTpi -> { - assertThat(usedTpi.getTopic()).isEqualTo(DataConstants.MAIN_QUEUE_TOPIC); + assertThat(usedTpi.getTopic()).isEqualTo(MAIN_QUEUE_TOPIC); assertThat(usedTpi.getTenantId()).get().isEqualTo(tenantId); }); } + @Test + public void testIsolatedQueueDeletion() throws Exception { + loginSysAdmin(); + TenantProfile tenantProfile = new TenantProfile(); + tenantProfile.setName("Test profile"); + TenantProfileData tenantProfileData = new TenantProfileData(); + tenantProfileData.setConfiguration(new DefaultTenantProfileConfiguration()); + tenantProfile.setProfileData(tenantProfileData); + tenantProfile.setIsolatedTbRuleEngine(true); + addQueueConfig(tenantProfile, MAIN_QUEUE_NAME); + tenantProfile = doPost("/api/tenantProfile", tenantProfile, TenantProfile.class); + createDifferentTenant(); + loginSysAdmin(); + savedDifferentTenant.setTenantProfileId(tenantProfile.getId()); + savedDifferentTenant = doPost("/api/tenant", savedDifferentTenant, Tenant.class); + TenantId tenantId = differentTenantId; + await().atMost(10, TimeUnit.SECONDS) + .until(() -> { + TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_RULE_ENGINE, MAIN_QUEUE_NAME, tenantId, tenantId); + return !tpi.getTenantId().get().isSysTenantId(); + }); + TopicPartitionInfo tpi = new TopicPartitionInfo(MAIN_QUEUE_TOPIC, tenantId, 0, false); + String isolatedTopic = tpi.getFullTopicName(); + TbMsg expectedMsg = publishTbMsg(tenantId, tpi); + awaitTbMsg(tbMsg -> tbMsg.getId().equals(expectedMsg.getId()), 10000); // to wait for consumer start + + loginSysAdmin(); + tenantProfile.setIsolatedTbRuleEngine(false); + tenantProfile.getProfileData().setQueueConfiguration(Collections.emptyList()); + tenantProfile = doPost("/api/tenantProfile", tenantProfile, TenantProfile.class); + await().atMost(10, TimeUnit.SECONDS) + .until(() -> partitionService.resolve(ServiceType.TB_RULE_ENGINE, MAIN_QUEUE_NAME, tenantId, tenantId) + .getTenantId().get().isSysTenantId()); + + Deque submittedMsgs = new LinkedList<>(); + await().atLeast(8, TimeUnit.SECONDS) // due to topic-deletion-delay + .atMost(20, TimeUnit.SECONDS) + .pollInterval(1, TimeUnit.SECONDS) + .untilAsserted(() -> { + TbMsg tbMsg = publishTbMsg(tenantId, tpi); + submittedMsgs.add(tbMsg.getId()); + + verify(queueAdmin, times(1)).deleteTopic(eq(isolatedTopic)); + }); + submittedMsgs.removeLast(); + for (UUID msgId : submittedMsgs) { + verify(actorContext, timeout(2000)).tell(argThat(msg -> { + return msg instanceof QueueToRuleEngineMsg && ((QueueToRuleEngineMsg) msg).getMsg().getId().equals(msgId); + })); + } + } + + private TbMsg publishTbMsg(TenantId tenantId, TopicPartitionInfo tpi) { + TbMsg tbMsg = TbMsg.newMsg("POST_TELEMETRY_REQUEST", tenantId, TbMsgMetaData.EMPTY, "{\"test\":1}"); + TransportProtos.ToRuleEngineMsg msg = TransportProtos.ToRuleEngineMsg.newBuilder() + .setTenantIdMSB(tenantId.getId().getMostSignificantBits()) + .setTenantIdLSB(tenantId.getId().getLeastSignificantBits()) + .setTbMsg(TbMsg.toByteString(tbMsg)).build(); + tbClusterService.pushMsgToRuleEngine(tpi, tbMsg.getId(), msg, null); + return tbMsg; + } + private void verifyUsedQueueAndMessage(String queue, TenantId tenantId, EntityId entityId, String msgType, Runnable action, Consumer tpiAssert) { await().atMost(15, TimeUnit.SECONDS) .untilAsserted(() -> { diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/memory/InMemoryTbQueueConsumer.java b/common/queue/src/main/java/org/thingsboard/server/queue/memory/InMemoryTbQueueConsumer.java index a642d7e9cf..dba6f6d588 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/memory/InMemoryTbQueueConsumer.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/memory/InMemoryTbQueueConsumer.java @@ -31,6 +31,7 @@ public class InMemoryTbQueueConsumer implements TbQueueCon private volatile Set partitions; private volatile boolean stopped; private volatile boolean subscribed; + private volatile boolean deleted; public InMemoryTbQueueConsumer(InMemoryStorage storage, String topic) { this.storage = storage; @@ -105,11 +106,12 @@ public class InMemoryTbQueueConsumer implements TbQueueCon @Override public void onQueueDelete() { + deleted = true; } @Override public boolean isDeleted() { - return false; + return deleted; } @Override From 12f8e14c0496a3affd02eaf2e07656d11f7348d9 Mon Sep 17 00:00:00 2001 From: ViacheslavKlimov Date: Thu, 3 Aug 2023 16:34:45 +0300 Subject: [PATCH 16/23] Increase default poll interval for isolated queues --- .../home/components/profile/tenant-profile.component.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/ui-ngx/src/app/modules/home/components/profile/tenant-profile.component.ts b/ui-ngx/src/app/modules/home/components/profile/tenant-profile.component.ts index bed7531495..e9142fa1fa 100644 --- a/ui-ngx/src/app/modules/home/components/profile/tenant-profile.component.ts +++ b/ui-ngx/src/app/modules/home/components/profile/tenant-profile.component.ts @@ -60,7 +60,7 @@ export class TenantProfileComponent extends EntityComponent { name: 'Main', packProcessingTimeout: 2000, partitions: 2, - pollInterval: 25, + pollInterval: 2000, processingStrategy: { failurePercentage: 0, maxPauseBetweenRetries: 3, @@ -82,7 +82,7 @@ export class TenantProfileComponent extends EntityComponent { id: guid(), name: 'HighPriority', topic: 'tb_rule_engine.hp', - pollInterval: 25, + pollInterval: 2000, partitions: 2, consumerPerPartition: true, packProcessingTimeout: 2000, @@ -106,7 +106,7 @@ export class TenantProfileComponent extends EntityComponent { id: guid(), name: 'SequentialByOriginator', topic: 'tb_rule_engine.sq', - pollInterval: 25, + pollInterval: 2000, partitions: 2, consumerPerPartition: true, packProcessingTimeout: 2000, From f32e2f6fdefb96cef3b8dc13008e76326b4faf00 Mon Sep 17 00:00:00 2001 From: ViacheslavKlimov Date: Thu, 10 Aug 2023 12:03:15 +0300 Subject: [PATCH 17/23] Refactoring after review --- .../service/entitiy/queue/DefaultTbQueueService.java | 4 ++++ .../queue/DefaultTbRuleEngineConsumerService.java | 6 +++--- .../org/thingsboard/server/queue/TbQueueConsumer.java | 2 +- .../queue/common/AbstractTbQueueConsumerTemplate.java | 9 ++++----- .../server/queue/memory/InMemoryTbQueueConsumer.java | 8 ++++---- 5 files changed, 16 insertions(+), 13 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/entitiy/queue/DefaultTbQueueService.java b/application/src/main/java/org/thingsboard/server/service/entitiy/queue/DefaultTbQueueService.java index 72b8c2252a..9aa9a421c8 100644 --- a/application/src/main/java/org/thingsboard/server/service/entitiy/queue/DefaultTbQueueService.java +++ b/application/src/main/java/org/thingsboard/server/service/entitiy/queue/DefaultTbQueueService.java @@ -176,6 +176,10 @@ public class DefaultTbQueueService extends AbstractTbEntityService implements Tb } } + if (log.isDebugEnabled()) { + log.debug("[{}] Handling profile queue config update: creating queues {}, updating {}, deleting {}. Affected tenants: {}", + newTenantProfile.getUuidId(), toCreate, toUpdate, toRemove, tenantIds); + } tenantIds.forEach(tenantId -> { toCreate.forEach(key -> saveQueue(new Queue(tenantId, newQueues.get(key)))); diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbRuleEngineConsumerService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbRuleEngineConsumerService.java index 863f0354c2..c8bf469a61 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbRuleEngineConsumerService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbRuleEngineConsumerService.java @@ -121,7 +121,7 @@ public class DefaultTbRuleEngineConsumerService extends AbstractConsumerService< private final ConcurrentMap consumerStats = new ConcurrentHashMap<>(); private final ConcurrentMap topicsConsumerPerPartition = new ConcurrentHashMap<>(); final ExecutorService submitExecutor = Executors.newSingleThreadExecutor(ThingsBoardThreadFactory.forName("tb-rule-engine-consumer-submit")); - final ScheduledExecutorService repartitionExecutor = Executors.newScheduledThreadPool(2, ThingsBoardThreadFactory.forName("tb-rule-engine-consumer-repartition")); + final ScheduledExecutorService repartitionExecutor = Executors.newScheduledThreadPool(1, ThingsBoardThreadFactory.forName("tb-rule-engine-consumer-repartition")); public DefaultTbRuleEngineConsumerService(TbRuleEngineProcessingStrategyFactory processingStrategyFactory, TbRuleEngineSubmitStrategyFactory submitStrategyFactory, @@ -274,7 +274,7 @@ public class DefaultTbRuleEngineConsumerService extends AbstractConsumerService< void consumerLoop(TbQueueConsumer> consumer, org.thingsboard.server.common.data.queue.Queue configuration, TbRuleEngineConsumerStats stats, String threadSuffix) { updateCurrentThreadName(threadSuffix); - while (!stopped && !consumer.isStopped() && !consumer.isDeleted()) { + while (!stopped && !consumer.isStopped() && !consumer.isQueueDeleted()) { try { List> msgs = consumer.poll(configuration.getPollInterval()); if (msgs.isEmpty()) { @@ -325,7 +325,7 @@ public class DefaultTbRuleEngineConsumerService extends AbstractConsumerService< } } - if (consumer.isDeleted()) { + if (consumer.isQueueDeleted()) { processQueueDeletion(configuration, consumer); } log.info("TB Rule Engine Consumer stopped."); diff --git a/common/cluster-api/src/main/java/org/thingsboard/server/queue/TbQueueConsumer.java b/common/cluster-api/src/main/java/org/thingsboard/server/queue/TbQueueConsumer.java index 73bea7642f..9c41f9d342 100644 --- a/common/cluster-api/src/main/java/org/thingsboard/server/queue/TbQueueConsumer.java +++ b/common/cluster-api/src/main/java/org/thingsboard/server/queue/TbQueueConsumer.java @@ -38,7 +38,7 @@ public interface TbQueueConsumer { void onQueueDelete(); - boolean isDeleted(); + boolean isQueueDeleted(); List getFullTopicNames(); diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/common/AbstractTbQueueConsumerTemplate.java b/common/queue/src/main/java/org/thingsboard/server/queue/common/AbstractTbQueueConsumerTemplate.java index 8e91c1d134..2ebe41850d 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/common/AbstractTbQueueConsumerTemplate.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/common/AbstractTbQueueConsumerTemplate.java @@ -44,7 +44,7 @@ public abstract class AbstractTbQueueConsumerTemplate i protected volatile Set partitions; protected final ReentrantLock consumerLock = new ReentrantLock(); //NonfairSync final Queue> subscribeQueue = new ConcurrentLinkedQueue<>(); - protected volatile boolean deleted = false; + protected volatile boolean queueDeleted = false; @Getter private final String topic; @@ -194,12 +194,11 @@ public abstract class AbstractTbQueueConsumerTemplate i @Override public void onQueueDelete() { - deleted = true; + queueDeleted = true; } - @Override - public boolean isDeleted() { - return deleted; + public boolean isQueueDeleted() { + return queueDeleted; } @Override diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/memory/InMemoryTbQueueConsumer.java b/common/queue/src/main/java/org/thingsboard/server/queue/memory/InMemoryTbQueueConsumer.java index dba6f6d588..8711cbbcf1 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/memory/InMemoryTbQueueConsumer.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/memory/InMemoryTbQueueConsumer.java @@ -31,7 +31,7 @@ public class InMemoryTbQueueConsumer implements TbQueueCon private volatile Set partitions; private volatile boolean stopped; private volatile boolean subscribed; - private volatile boolean deleted; + private volatile boolean queueDeleted; public InMemoryTbQueueConsumer(InMemoryStorage storage, String topic) { this.storage = storage; @@ -106,12 +106,12 @@ public class InMemoryTbQueueConsumer implements TbQueueCon @Override public void onQueueDelete() { - deleted = true; + queueDeleted = true; } @Override - public boolean isDeleted() { - return deleted; + public boolean isQueueDeleted() { + return queueDeleted; } @Override From 6751820e0a22d1fb3f07844bb758886e273816f0 Mon Sep 17 00:00:00 2001 From: ViacheslavKlimov Date: Mon, 14 Aug 2023 12:57:53 +0300 Subject: [PATCH 18/23] Dedicated Rule Engines for tenant profile --- .../server/actors/app/AppActor.java | 5 + .../DefaultTbRuleEngineConsumerService.java | 2 +- .../DefaultTenantRoutingInfoService.java | 8 +- .../src/main/resources/thingsboard.yml | 4 + .../discovery/HashPartitionServiceTest.java | 127 ++++++++++++++++-- common/cluster-api/src/main/proto/queue.proto | 1 + .../DefaultTbServiceInfoProvider.java | 11 +- .../queue/discovery/HashPartitionService.java | 84 +++++++++--- .../discovery/TbServiceInfoProvider.java | 5 + .../queue/discovery/TenantRoutingInfo.java | 2 + .../discovery/TenantRoutingInfoService.java | 1 + .../TransportTenantRoutingInfoService.java | 4 +- ...ersionControlTenantRoutingInfoService.java | 2 +- .../assets/locale/locale.constant-en_US.json | 2 +- 14 files changed, 223 insertions(+), 35 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/actors/app/AppActor.java b/application/src/main/java/org/thingsboard/server/actors/app/AppActor.java index fb6fbbdff2..e8a56fab16 100644 --- a/application/src/main/java/org/thingsboard/server/actors/app/AppActor.java +++ b/application/src/main/java/org/thingsboard/server/actors/app/AppActor.java @@ -47,6 +47,7 @@ import org.thingsboard.server.service.transport.msg.TransportToDeviceActorMsgWra import java.util.HashSet; import java.util.Set; +import java.util.UUID; @Slf4j public class AppActor extends ContextAwareActor { @@ -123,7 +124,11 @@ public class AppActor extends ContextAwareActor { try { if (systemContext.isTenantComponentsInitEnabled()) { PageDataIterable tenantIterator = new PageDataIterable<>(tenantService::findTenants, ENTITY_PACK_LIMIT); + Set assignedProfiles = systemContext.getServiceInfoProvider().getAssignedTenantProfiles(); for (Tenant tenant : tenantIterator) { + if (!assignedProfiles.isEmpty() && !assignedProfiles.contains(tenant.getTenantProfileId().getId())) { + continue; + } log.debug("[{}] Creating tenant actor", tenant.getId()); getOrCreateTenantActor(tenant.getId()); log.debug("[{}] Tenant actor created.", tenant.getId()); diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbRuleEngineConsumerService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbRuleEngineConsumerService.java index d560cfee1c..a18b5c099c 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbRuleEngineConsumerService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbRuleEngineConsumerService.java @@ -156,7 +156,7 @@ public class DefaultTbRuleEngineConsumerService extends AbstractConsumerService< super.init("tb-rule-engine-consumer", "tb-rule-engine-notifications-consumer"); List queues = queueService.findAllQueues(); for (Queue configuration : queues) { - initConsumer(configuration); + initConsumer(configuration); // TODO: if this Rule Engine is assigned specific profile, don't init other consumers and properly handle queue update events } } diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTenantRoutingInfoService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTenantRoutingInfoService.java index 400586235e..ea90364ef7 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTenantRoutingInfoService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTenantRoutingInfoService.java @@ -22,7 +22,6 @@ import org.thingsboard.server.common.data.TenantProfile; import org.thingsboard.server.common.data.exception.TenantNotFoundException; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.dao.tenant.TbTenantProfileCache; -import org.thingsboard.server.dao.tenant.TenantService; import org.thingsboard.server.queue.discovery.TenantRoutingInfo; import org.thingsboard.server.queue.discovery.TenantRoutingInfoService; @@ -31,12 +30,9 @@ import org.thingsboard.server.queue.discovery.TenantRoutingInfoService; @ConditionalOnExpression("'${service.type:null}'=='monolith' || '${service.type:null}'=='tb-core' || '${service.type:null}'=='tb-rule-engine'") public class DefaultTenantRoutingInfoService implements TenantRoutingInfoService { - private final TenantService tenantService; - private final TbTenantProfileCache tenantProfileCache; - public DefaultTenantRoutingInfoService(TenantService tenantService, TbTenantProfileCache tenantProfileCache) { - this.tenantService = tenantService; + public DefaultTenantRoutingInfoService(TbTenantProfileCache tenantProfileCache) { this.tenantProfileCache = tenantProfileCache; } @@ -44,7 +40,7 @@ public class DefaultTenantRoutingInfoService implements TenantRoutingInfoService public TenantRoutingInfo getRoutingInfo(TenantId tenantId) { TenantProfile tenantProfile = tenantProfileCache.get(tenantId); if (tenantProfile != null) { - return new TenantRoutingInfo(tenantId, tenantProfile.isIsolatedTbRuleEngine()); + return new TenantRoutingInfo(tenantId, tenantProfile.getId(), tenantProfile.isIsolatedTbRuleEngine()); } else { throw new TenantNotFoundException(tenantId); } diff --git a/application/src/main/resources/thingsboard.yml b/application/src/main/resources/thingsboard.yml index 8576f20783..2ce5c3ba21 100644 --- a/application/src/main/resources/thingsboard.yml +++ b/application/src/main/resources/thingsboard.yml @@ -1253,6 +1253,10 @@ service: type: "${TB_SERVICE_TYPE:monolith}" # monolith or tb-core or tb-rule-engine # Unique id for this service (autogenerated if empty) id: "${TB_SERVICE_ID:}" + rule_engine: + # Comma-separated list of tenant profiles ids assigned to this Rule Engine. + # This Rule Engine will only be responsible for tenants with these profiles (in case 'isolation' option is enabled in profile). + assigned_tenant_profiles: "${TB_RULE_ENGINE_ASSIGNED_TENANT_PROFILES:}" metrics: # Enable/disable actuator metrics. diff --git a/application/src/test/java/org/thingsboard/server/queue/discovery/HashPartitionServiceTest.java b/application/src/test/java/org/thingsboard/server/queue/discovery/HashPartitionServiceTest.java index 25b06e7840..84024ecd1e 100644 --- a/application/src/test/java/org/thingsboard/server/queue/discovery/HashPartitionServiceTest.java +++ b/application/src/test/java/org/thingsboard/server/queue/discovery/HashPartitionServiceTest.java @@ -18,6 +18,7 @@ package org.thingsboard.server.queue.discovery; import com.datastax.oss.driver.api.core.uuid.Uuids; import lombok.SneakyThrows; import lombok.extern.slf4j.Slf4j; +import org.apache.commons.collections4.ListUtils; import org.junit.Assert; import org.junit.Before; import org.junit.Test; @@ -25,24 +26,35 @@ import org.junit.runner.RunWith; import org.mockito.junit.MockitoJUnitRunner; import org.springframework.context.ApplicationEventPublisher; import org.springframework.test.util.ReflectionTestUtils; +import org.thingsboard.server.common.data.DataConstants; import org.thingsboard.server.common.data.id.DeviceId; +import org.thingsboard.server.common.data.id.QueueId; import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.id.TenantProfileId; +import org.thingsboard.server.common.data.id.UUIDBased; +import org.thingsboard.server.common.data.queue.Queue; import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; -import org.thingsboard.server.gen.transport.TransportProtos; +import org.thingsboard.server.gen.transport.TransportProtos.ServiceInfo; import java.text.SimpleDateFormat; import java.util.ArrayList; +import java.util.Collection; import java.util.Collections; import java.util.Comparator; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Random; +import java.util.UUID; import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; +import java.util.stream.Stream; +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; @Slf4j @RunWith(MockitoJUnitRunner.class) @@ -57,7 +69,7 @@ public class HashPartitionServiceTest { private ApplicationEventPublisher applicationEventPublisher; private QueueRoutingInfoService queueRoutingInfoService; - private String hashFunctionName = "sha256"; + private String hashFunctionName = "murmur3_128"; @Before public void setup() throws Exception { @@ -74,15 +86,15 @@ public class HashPartitionServiceTest { ReflectionTestUtils.setField(clusterRoutingService, "vcTopic", "tb.vc"); ReflectionTestUtils.setField(clusterRoutingService, "vcPartitions", 10); ReflectionTestUtils.setField(clusterRoutingService, "hashFunctionName", hashFunctionName); - TransportProtos.ServiceInfo currentServer = TransportProtos.ServiceInfo.newBuilder() + ServiceInfo currentServer = ServiceInfo.newBuilder() .setServiceId("tb-core-0") .addAllServiceTypes(Collections.singletonList(ServiceType.TB_CORE.name())) .build(); // when(queueService.resolve(Mockito.any(), Mockito.anyString())).thenAnswer(i -> i.getArguments()[1]); // when(discoveryService.getServiceInfo()).thenReturn(currentServer); - List otherServers = new ArrayList<>(); + List otherServers = new ArrayList<>(); for (int i = 1; i < SERVER_COUNT; i++) { - otherServers.add(TransportProtos.ServiceInfo.newBuilder() + otherServers.add(ServiceInfo.newBuilder() .setServiceId("tb-rule-" + i) .addAllServiceTypes(Collections.singletonList(ServiceType.TB_CORE.name())) .build()); @@ -122,10 +134,10 @@ public class HashPartitionServiceTest { int queueCount = 3; int partitionCount = 3; - List services = new ArrayList<>(); + List services = new ArrayList<>(); for (int i = 0; i < serverCount; i++) { - services.add(TransportProtos.ServiceInfo.newBuilder().setServiceId("RE-" + i).build()); + services.add(ServiceInfo.newBuilder().setServiceId("RE-" + i).build()); } long start = System.currentTimeMillis(); @@ -140,7 +152,7 @@ public class HashPartitionServiceTest { for (int queueIndex = 0; queueIndex < queueCount; queueIndex++) { QueueKey queueKey = new QueueKey(ServiceType.TB_RULE_ENGINE, "queue" + queueIndex, tenantId); for (int partition = 0; partition < partitionCount; partition++) { - TransportProtos.ServiceInfo serviceInfo = clusterRoutingService.resolveByPartitionIdx(services, queueKey, partition); + ServiceInfo serviceInfo = clusterRoutingService.resolveByPartitionIdx(services, queueKey, partition); String serviceId = serviceInfo.getServiceId(); map.put(serviceId, map.get(serviceId) + 1); } @@ -163,4 +175,103 @@ public class HashPartitionServiceTest { Assert.assertTrue(diffPercent < maxDiffPercent); } + @Test + public void testPartitionsAssignmentWithDedicatedServers() { + int isolatedProfilesCount = 5; + int tenantsCountPerProfile = 100; + int dedicatedServerSetsCount = 3; + int serversCountPerSet = 3; + int profilesPerSet = (int) Math.ceil((double) isolatedProfilesCount / dedicatedServerSetsCount); + + List isolatedTenantProfiles = Stream.generate(() -> new TenantProfileId(UUID.randomUUID())) + .limit(isolatedProfilesCount).collect(Collectors.toList()); + Map tenants = new HashMap<>(); + for (TenantProfileId tenantProfileId : isolatedTenantProfiles) { + for (int i = 0; i < tenantsCountPerProfile; i++) { + tenants.put(new TenantId(UUID.randomUUID()), tenantProfileId); + } + } + + List queues = new ArrayList<>(); + Queue systemQueue = new Queue(); + systemQueue.setTenantId(TenantId.SYS_TENANT_ID); + systemQueue.setName("Main"); + systemQueue.setTopic(DataConstants.MAIN_QUEUE_TOPIC); + systemQueue.setPartitions(10); + systemQueue.setId(new QueueId(UUID.randomUUID())); + queues.add(systemQueue); + tenants.forEach((tenantId, profileId) -> { + Queue isolatedQueue = new Queue(); + isolatedQueue.setTenantId(tenantId); + isolatedQueue.setName("Main"); + isolatedQueue.setTopic(DataConstants.MAIN_QUEUE_TOPIC); + isolatedQueue.setPartitions(2); + isolatedQueue.setId(new QueueId(UUID.randomUUID())); + queues.add(isolatedQueue); + when(routingInfoService.getRoutingInfo(eq(tenantId))).thenReturn(new TenantRoutingInfo(tenantId, profileId, true)); + }); + when(queueRoutingInfoService.getAllQueuesRoutingInfo()).thenReturn(queues.stream() + .map(QueueRoutingInfo::new).collect(Collectors.toList())); + + List ruleEngines = new ArrayList<>(); + Map> dedicatedServers = new HashMap<>(); + int serviceId = 0; + for (int i = 0; i < serversCountPerSet; i++) { + ServiceInfo commonServer = ServiceInfo.newBuilder() + .setServiceId("tb-rule-engine-" + serviceId) + .addAllServiceTypes(List.of(ServiceType.TB_RULE_ENGINE.name())) + .build(); + ruleEngines.add(commonServer); + serviceId++; + } + for (int i = 0; i < dedicatedServerSetsCount; i++) { + List assignedProfiles = ListUtils.partition(isolatedTenantProfiles, profilesPerSet).get(i); + for (int j = 0; j < serversCountPerSet; j++) { + ServiceInfo dedicatedServer = ServiceInfo.newBuilder() + .setServiceId("tb-rule-engine-" + serviceId) + .addAllServiceTypes(List.of(ServiceType.TB_RULE_ENGINE.name())) + .addAllAssignedTenantProfiles(assignedProfiles.stream().map(UUIDBased::toString).collect(Collectors.toList())) + .build(); + ruleEngines.add(dedicatedServer); + serviceId++; + + for (TenantProfileId assignedProfileId : assignedProfiles) { + dedicatedServers.computeIfAbsent(assignedProfileId, p -> new ArrayList<>()).add(dedicatedServer); + } + } + } + + Map>> serversPartitions = new HashMap<>(); + clusterRoutingService.init(); + for (ServiceInfo ruleEngine : ruleEngines) { + List other = new ArrayList<>(ruleEngines); + other.removeIf(serviceInfo -> serviceInfo.getServiceId().equals(ruleEngine.getServiceId())); + + clusterRoutingService.recalculatePartitions(ruleEngine, other); + clusterRoutingService.myPartitions.forEach((queueKey, partitions) -> { + serversPartitions.computeIfAbsent(queueKey, k -> new HashMap<>()).put(ruleEngine, partitions); + }); + } + assertThat(serversPartitions.keySet()).containsAll(queues.stream().map(queue -> new QueueKey(ServiceType.TB_RULE_ENGINE, queue)).collect(Collectors.toList())); + + serversPartitions.forEach((queueKey, partitionsPerServer) -> { + if (queueKey.getTenantId().isSysTenantId()) { + partitionsPerServer.forEach((server, partitions) -> { + assertThat(server.getAssignedTenantProfilesCount()).as("system queues are not assigned to dedicated servers").isZero(); + }); + } else { + List responsibleServers = dedicatedServers.get(tenants.get(queueKey.getTenantId())); + partitionsPerServer.forEach((server, partitions) -> { + assertThat(server.getAssignedTenantProfilesCount()).as("isolated queues are only assigned to dedicated servers").isPositive(); + assertThat(responsibleServers).contains(server); + }); + } + + List allPartitions = partitionsPerServer.values().stream() + .flatMap(Collection::stream) + .collect(Collectors.toList()); + assertThat(allPartitions).doesNotHaveDuplicates(); + }); + } + } diff --git a/common/cluster-api/src/main/proto/queue.proto b/common/cluster-api/src/main/proto/queue.proto index 80f44e59be..78614911d4 100644 --- a/common/cluster-api/src/main/proto/queue.proto +++ b/common/cluster-api/src/main/proto/queue.proto @@ -28,6 +28,7 @@ message ServiceInfo { repeated string serviceTypes = 2; repeated string transports = 6; SystemInfoProto systemInfo = 10; + repeated string assignedTenantProfiles = 11; } message SystemInfoProto { diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/DefaultTbServiceInfoProvider.java b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/DefaultTbServiceInfoProvider.java index 3c85aef350..e9013ef345 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/DefaultTbServiceInfoProvider.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/DefaultTbServiceInfoProvider.java @@ -23,6 +23,7 @@ import org.springframework.context.ApplicationContext; import org.springframework.stereotype.Component; import org.thingsboard.server.common.data.StringUtils; import org.thingsboard.server.common.data.TbTransportService; +import org.thingsboard.server.common.data.util.CollectionsUtil; import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.gen.transport.TransportProtos.ServiceInfo; @@ -35,6 +36,8 @@ import java.util.Arrays; import java.util.Collection; import java.util.Collections; import java.util.List; +import java.util.Set; +import java.util.UUID; import java.util.stream.Collectors; import static org.thingsboard.common.util.SystemUtil.getCpuCount; @@ -57,6 +60,10 @@ public class DefaultTbServiceInfoProvider implements TbServiceInfoProvider { @Value("${service.type:monolith}") private String serviceType; + @Getter + @Value("${service.rule_engine.assigned_tenant_profiles:}") + private Set assignedTenantProfiles; + @Autowired private ApplicationContext applicationContext; @@ -111,7 +118,9 @@ public class DefaultTbServiceInfoProvider implements TbServiceInfoProvider { .setServiceId(serviceId) .addAllServiceTypes(serviceTypes.stream().map(ServiceType::name).collect(Collectors.toList())) .setSystemInfo(getCurrentSystemInfoProto()); - + if (CollectionsUtil.isNotEmpty(assignedTenantProfiles)) { + builder.addAllAssignedTenantProfiles(assignedTenantProfiles.stream().map(UUID::toString).collect(Collectors.toList())); + } return serviceInfo = builder.build(); } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/HashPartitionService.java b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/HashPartitionService.java index a8954b2b33..4c1894eae1 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/HashPartitionService.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/HashPartitionService.java @@ -24,6 +24,7 @@ import org.springframework.stereotype.Service; import org.thingsboard.server.common.data.exception.TenantNotFoundException; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.id.TenantProfileId; import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; import org.thingsboard.server.gen.transport.TransportProtos; @@ -36,6 +37,7 @@ import org.thingsboard.server.queue.util.AfterStartUp; import javax.annotation.PostConstruct; import java.nio.charset.StandardCharsets; import java.util.ArrayList; +import java.util.Collection; import java.util.Collections; import java.util.Comparator; import java.util.HashMap; @@ -70,15 +72,16 @@ public class HashPartitionService implements PartitionService { private final TenantRoutingInfoService tenantRoutingInfoService; private final QueueRoutingInfoService queueRoutingInfoService; - private volatile ConcurrentMap> myPartitions = new ConcurrentHashMap<>(); + protected volatile ConcurrentMap> myPartitions = new ConcurrentHashMap<>(); private final ConcurrentMap partitionTopicsMap = new ConcurrentHashMap<>(); private final ConcurrentMap partitionSizesMap = new ConcurrentHashMap<>(); private final ConcurrentMap tenantRoutingInfoMap = new ConcurrentHashMap<>(); - private Map> tbTransportServicesByType = new HashMap<>(); private List currentOtherServices; + private final Map> tbTransportServicesByType = new HashMap<>(); + private final Map> responsibleServices = new HashMap<>(); private HashFunction hashFunction; @@ -215,14 +218,12 @@ public class HashPartitionService implements PartitionService { } private TopicPartitionInfo resolve(QueueKey queueKey, EntityId entityId) { - int hash = hashFunction.newHasher() - .putLong(entityId.getId().getMostSignificantBits()) - .putLong(entityId.getId().getLeastSignificantBits()).hash().asInt(); - Integer partitionSize = partitionSizesMap.get(queueKey); if (partitionSize == null) { throw new IllegalStateException("Partitions info for queue " + queueKey + " is missing"); } + + int hash = hash(entityId.getId()); int partition = Math.abs(hash % partitionSize); return buildTopicPartitionInfo(queueKey, partition); @@ -231,6 +232,7 @@ public class HashPartitionService implements PartitionService { @Override public synchronized void recalculatePartitions(ServiceInfo currentService, List otherServices) { tbTransportServicesByType.clear(); + responsibleServices.clear(); logServiceInfo(currentService); otherServices.forEach(this::logServiceInfo); @@ -240,6 +242,7 @@ public class HashPartitionService implements PartitionService { addNode(queueServicesMap, other); } queueServicesMap.values().forEach(list -> list.sort(Comparator.comparing(ServiceInfo::getServiceId))); + responsibleServices.values().forEach(list -> list.sort(Comparator.comparing(ServiceInfo::getServiceId))); final ConcurrentMap> newPartitions = new ConcurrentHashMap<>(); partitionSizesMap.forEach((queueKey, size) -> { @@ -287,6 +290,9 @@ public class HashPartitionService implements PartitionService { changes.addAll(newMap.keySet()); if (!changes.isEmpty()) { applicationEventPublisher.publishEvent(new ClusterTopologyChangeEvent(this, changes)); + responsibleServices.forEach((profileId, serviceInfos) -> { + log.info("Servers responsible for tenant profile {}: {}", profileId, toServiceIds(serviceInfos)); + }); } } @@ -324,9 +330,7 @@ public class HashPartitionService implements PartitionService { @Override public int resolvePartitionIndex(UUID entityId, int partitions) { - int hash = hashFunction.newHasher() - .putLong(entityId.getMostSignificantBits()) - .putLong(entityId.getLeastSignificantBits()).hash().asInt(); + int hash = hash(entityId); return Math.abs(hash % partitions); } @@ -408,6 +412,19 @@ public class HashPartitionService implements PartitionService { queueServiceList.computeIfAbsent(key, k -> new ArrayList<>()).add(instance); } }); + + if (instance.getAssignedTenantProfilesCount() > 0) { + for (String profileIdStr : instance.getAssignedTenantProfilesList()) { + TenantProfileId profileId; + try { + profileId = new TenantProfileId(UUID.fromString(profileIdStr)); + } catch (IllegalArgumentException e) { + log.warn("Failed to parse '{}' as tenant profile id", profileIdStr); + continue; + } + responsibleServices.computeIfAbsent(profileId, k -> new ArrayList<>()).add(instance); + } + } } else if (ServiceType.TB_CORE.equals(serviceType) || ServiceType.TB_VC_EXECUTOR.equals(serviceType)) { queueServiceList.computeIfAbsent(new QueueKey(serviceType), key -> new ArrayList<>()).add(instance); } @@ -423,18 +440,51 @@ public class HashPartitionService implements PartitionService { return null; } - if (!ServiceType.TB_RULE_ENGINE.equals(queueKey.getType()) || TenantId.SYS_TENANT_ID.equals(queueKey.getTenantId())) { - return servers.get(partition % servers.size()); - } else { - int hash = hashFunction.newHasher().putLong(queueKey.getTenantId().getId().getMostSignificantBits()) - .putLong(queueKey.getTenantId().getId().getLeastSignificantBits()) + TenantId tenantId = queueKey.getTenantId(); + if (queueKey.getType() == ServiceType.TB_RULE_ENGINE) { + if (!responsibleServices.isEmpty()) { // if there are any dedicated servers + TenantProfileId profileId; + if (tenantId != null && !tenantId.isSysTenantId()) { + TenantRoutingInfo routingInfo = tenantRoutingInfoService.getRoutingInfo(tenantId); + profileId = routingInfo.getProfileId(); + } else { + profileId = null; + } + + List responsible = responsibleServices.get(profileId); + if (responsible == null) { + // if there are no dedicated servers for this tenant profile, or for system queues, + // using the servers that are not responsible for any profile + responsible = servers.stream() + .filter(serviceInfo -> serviceInfo.getAssignedTenantProfilesCount() == 0) + .sorted(Comparator.comparing(ServiceInfo::getServiceId)) + .collect(Collectors.toList()); + if (profileId != null) { + log.debug("Using servers {} for profile {}", toServiceIds(responsible), profileId); + } + responsibleServices.put(profileId, responsible); + } + servers = responsible; + } + + int hash = hashFunction.newHasher() + .putLong(tenantId.getId().getMostSignificantBits()) + .putLong(tenantId.getId().getLeastSignificantBits()) .putString(queueKey.getQueueName(), StandardCharsets.UTF_8) .hash().asInt(); - return servers.get(Math.abs((hash + partition) % servers.size())); + } else { + return servers.get(partition % servers.size()); } } + private int hash(UUID key) { + return hashFunction.newHasher() + .putLong(key.getMostSignificantBits()) + .putLong(key.getLeastSignificantBits()) + .hash().asInt(); + } + public static HashFunction forName(String name) { switch (name) { case "murmur3_32": @@ -448,4 +498,8 @@ public class HashPartitionService implements PartitionService { } } + private List toServiceIds(Collection serviceInfos) { + return serviceInfos.stream().map(ServiceInfo::getServiceId).collect(Collectors.toList()); + } + } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/TbServiceInfoProvider.java b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/TbServiceInfoProvider.java index e49cbbcfd9..9c7d1630ec 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/TbServiceInfoProvider.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/TbServiceInfoProvider.java @@ -18,6 +18,9 @@ package org.thingsboard.server.queue.discovery; import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.gen.transport.TransportProtos.ServiceInfo; +import java.util.Set; +import java.util.UUID; + public interface TbServiceInfoProvider { String getServiceId(); @@ -30,4 +33,6 @@ public interface TbServiceInfoProvider { ServiceInfo generateNewServiceInfoWithCurrentSystemInfo(); + Set getAssignedTenantProfiles(); + } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/TenantRoutingInfo.java b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/TenantRoutingInfo.java index c1c0b49dab..8dee68da49 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/TenantRoutingInfo.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/TenantRoutingInfo.java @@ -17,9 +17,11 @@ package org.thingsboard.server.queue.discovery; import lombok.Data; import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.id.TenantProfileId; @Data public class TenantRoutingInfo { private final TenantId tenantId; + private final TenantProfileId profileId; private final boolean isolatedTbRuleEngine; } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/TenantRoutingInfoService.java b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/TenantRoutingInfoService.java index 8dd3ff95e7..e4c0ac8250 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/TenantRoutingInfoService.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/TenantRoutingInfoService.java @@ -20,4 +20,5 @@ import org.thingsboard.server.common.data.id.TenantId; public interface TenantRoutingInfoService { TenantRoutingInfo getRoutingInfo(TenantId tenantId); + } diff --git a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/TransportTenantRoutingInfoService.java b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/TransportTenantRoutingInfoService.java index c9f126b808..e1192391d5 100644 --- a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/TransportTenantRoutingInfoService.java +++ b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/TransportTenantRoutingInfoService.java @@ -29,7 +29,7 @@ import org.thingsboard.server.queue.discovery.TenantRoutingInfoService; @ConditionalOnExpression("'${service.type:null}'=='tb-transport'") public class TransportTenantRoutingInfoService implements TenantRoutingInfoService { - private TransportTenantProfileCache tenantProfileCache; + private final TransportTenantProfileCache tenantProfileCache; public TransportTenantRoutingInfoService(TransportTenantProfileCache tenantProfileCache) { this.tenantProfileCache = tenantProfileCache; @@ -38,7 +38,7 @@ public class TransportTenantRoutingInfoService implements TenantRoutingInfoServi @Override public TenantRoutingInfo getRoutingInfo(TenantId tenantId) { TenantProfile profile = tenantProfileCache.get(tenantId); - return new TenantRoutingInfo(tenantId, profile.isIsolatedTbRuleEngine()); + return new TenantRoutingInfo(tenantId, profile.getId(), profile.isIsolatedTbRuleEngine()); } } diff --git a/msa/vc-executor/src/main/java/org/thingsboard/server/vc/service/VersionControlTenantRoutingInfoService.java b/msa/vc-executor/src/main/java/org/thingsboard/server/vc/service/VersionControlTenantRoutingInfoService.java index fb33ff9931..b343a4791f 100644 --- a/msa/vc-executor/src/main/java/org/thingsboard/server/vc/service/VersionControlTenantRoutingInfoService.java +++ b/msa/vc-executor/src/main/java/org/thingsboard/server/vc/service/VersionControlTenantRoutingInfoService.java @@ -25,6 +25,6 @@ public class VersionControlTenantRoutingInfoService implements TenantRoutingInfo @Override public TenantRoutingInfo getRoutingInfo(TenantId tenantId) { //This dummy implementation is ok since Version Control service does not produce any rule engine messages. - return new TenantRoutingInfo(tenantId, false); + return new TenantRoutingInfo(tenantId, null, false); } } diff --git a/ui-ngx/src/assets/locale/locale.constant-en_US.json b/ui-ngx/src/assets/locale/locale.constant-en_US.json index c1092f9dad..b449cf1ba5 100644 --- a/ui-ngx/src/assets/locale/locale.constant-en_US.json +++ b/ui-ngx/src/assets/locale/locale.constant-en_US.json @@ -3535,7 +3535,7 @@ "search": "Search tenants", "selected-tenants": "{ count, plural, =1 {1 tenant} other {# tenants} } selected", "isolated-tb-rule-engine": "Processing in isolated ThingsBoard Rule Engine container", - "isolated-tb-rule-engine-details": "Requires separate microservice(s) per isolated Tenant" + "isolated-tb-rule-engine-details": "Requires separate microservice(s) for the profile" }, "tenant-profile": { "tenant-profile": "Tenant profile", From b8a7c6a3cd758508c4cdcb7b811a839f5fe080d9 Mon Sep 17 00:00:00 2001 From: ViacheslavKlimov Date: Tue, 15 Aug 2023 16:40:26 +0300 Subject: [PATCH 19/23] Don't init unneeded actors and consumers for dedicated Rule Engine --- .../server/actors/ActorSystemContext.java | 1 + .../server/actors/app/AppActor.java | 54 ++++++++++-------- .../server/actors/tenant/TenantActor.java | 23 ++++---- .../DefaultTbRuleEngineConsumerService.java | 55 +++++++++++-------- .../discovery/HashPartitionServiceTest.java | 22 ++++++++ .../DefaultTbServiceInfoProvider.java | 3 + .../queue/discovery/HashPartitionService.java | 15 +++++ .../queue/discovery/PartitionService.java | 3 + 8 files changed, 118 insertions(+), 58 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java b/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java index 7e5dfed5f0..1abfe79a76 100644 --- a/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java +++ b/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java @@ -249,6 +249,7 @@ public class ActorSystemContext { private RuleNodeStateService ruleNodeStateService; @Autowired + @Getter private PartitionService partitionService; @Autowired diff --git a/application/src/main/java/org/thingsboard/server/actors/app/AppActor.java b/application/src/main/java/org/thingsboard/server/actors/app/AppActor.java index e8a56fab16..52abeca2c0 100644 --- a/application/src/main/java/org/thingsboard/server/actors/app/AppActor.java +++ b/application/src/main/java/org/thingsboard/server/actors/app/AppActor.java @@ -46,8 +46,8 @@ import org.thingsboard.server.dao.tenant.TenantService; import org.thingsboard.server.service.transport.msg.TransportToDeviceActorMsgWrapper; import java.util.HashSet; +import java.util.Optional; import java.util.Set; -import java.util.UUID; @Slf4j public class AppActor extends ContextAwareActor { @@ -124,14 +124,13 @@ public class AppActor extends ContextAwareActor { try { if (systemContext.isTenantComponentsInitEnabled()) { PageDataIterable tenantIterator = new PageDataIterable<>(tenantService::findTenants, ENTITY_PACK_LIMIT); - Set assignedProfiles = systemContext.getServiceInfoProvider().getAssignedTenantProfiles(); for (Tenant tenant : tenantIterator) { - if (!assignedProfiles.isEmpty() && !assignedProfiles.contains(tenant.getTenantProfileId().getId())) { - continue; - } log.debug("[{}] Creating tenant actor", tenant.getId()); - getOrCreateTenantActor(tenant.getId()); - log.debug("[{}] Tenant actor created.", tenant.getId()); + getOrCreateTenantActor(tenant.getId()).ifPresentOrElse(tenantActor -> { + log.debug("[{}] Tenant actor created.", tenant.getId()); + }, () -> { + log.debug("[{}] Skipped actor creation", tenant.getId()); + }); } } log.info("Main system actor started."); @@ -145,7 +144,9 @@ public class AppActor extends ContextAwareActor { msg.getMsg().getCallback().onFailure(new RuleEngineException("Message has system tenant id!")); } else { if (!deletedTenants.contains(msg.getTenantId())) { - getOrCreateTenantActor(msg.getTenantId()).tell(msg); + getOrCreateTenantActor(msg.getTenantId()).ifPresentOrElse(actor -> { + actor.tell(msg); + }, () -> msg.getMsg().getCallback().onSuccess()); } else { msg.getMsg().getCallback().onSuccess(); } @@ -165,12 +166,13 @@ public class AppActor extends ContextAwareActor { log.info("[{}] Handling tenant deleted notification: {}", msg.getTenantId(), msg); deletedTenants.add(tenantId); ctx.stop(new TbEntityActorId(tenantId)); - } else { - target = getOrCreateTenantActor(msg.getTenantId()); + return; } - } else { - target = getOrCreateTenantActor(msg.getTenantId()); } + target = getOrCreateTenantActor(msg.getTenantId()).orElseGet(() -> { + log.debug("Ignoring component lifecycle msg for tenant {} because it is not managed by this service", msg.getTenantId()); + return null; + }); } if (target != null) { target.tellWithHighPriority(msg); @@ -181,12 +183,13 @@ public class AppActor extends ContextAwareActor { private void onToDeviceActorMsg(TenantAwareMsg msg, boolean priority) { if (!deletedTenants.contains(msg.getTenantId())) { - TbActorRef tenantActor = getOrCreateTenantActor(msg.getTenantId()); - if (priority) { - tenantActor.tellWithHighPriority(msg); - } else { - tenantActor.tell(msg); - } + getOrCreateTenantActor(msg.getTenantId()).ifPresent(tenantActor -> { + if (priority) { + tenantActor.tellWithHighPriority(msg); + } else { + tenantActor.tell(msg); + } + }); } else { if (msg instanceof TransportToDeviceActorMsgWrapper) { ((TransportToDeviceActorMsgWrapper) msg).getCallback().onSuccess(); @@ -194,10 +197,15 @@ public class AppActor extends ContextAwareActor { } } - private TbActorRef getOrCreateTenantActor(TenantId tenantId) { - return ctx.getOrCreateChildActor(new TbEntityActorId(tenantId), - () -> DefaultActorService.TENANT_DISPATCHER_NAME, - () -> new TenantActor.ActorCreator(systemContext, tenantId)); + private Optional getOrCreateTenantActor(TenantId tenantId) { + if (systemContext.getServiceInfoProvider().isService(ServiceType.TB_CORE) || + systemContext.getPartitionService().isManagedByCurrentService(tenantId)) { + return Optional.of(ctx.getOrCreateChildActor(new TbEntityActorId(tenantId), + () -> DefaultActorService.TENANT_DISPATCHER_NAME, + () -> new TenantActor.ActorCreator(systemContext, tenantId))); + } else { + return Optional.empty(); + } } private void onToEdgeSessionMsg(EdgeSessionMsg msg) { @@ -205,7 +213,7 @@ public class AppActor extends ContextAwareActor { if (ModelConstants.SYSTEM_TENANT.equals(msg.getTenantId())) { log.warn("Message has system tenant id: {}", msg); } else { - target = getOrCreateTenantActor(msg.getTenantId()); + target = getOrCreateTenantActor(msg.getTenantId()).orElse(null); } if (target != null) { target.tellWithHighPriority(msg); diff --git a/application/src/main/java/org/thingsboard/server/actors/tenant/TenantActor.java b/application/src/main/java/org/thingsboard/server/actors/tenant/TenantActor.java index 84b7757846..6cc249c87c 100644 --- a/application/src/main/java/org/thingsboard/server/actors/tenant/TenantActor.java +++ b/application/src/main/java/org/thingsboard/server/actors/tenant/TenantActor.java @@ -32,7 +32,6 @@ import org.thingsboard.server.actors.service.DefaultActorService; import org.thingsboard.server.common.data.ApiUsageState; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.Tenant; -import org.thingsboard.server.common.data.TenantProfile; import org.thingsboard.server.common.data.edge.Edge; import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.EdgeId; @@ -83,21 +82,21 @@ public class TenantActor extends RuleChainManagerActor { cantFindTenant = true; log.info("[{}] Started tenant actor for missing tenant.", tenantId); } else { - TenantProfile tenantProfile = systemContext.getTenantProfileCache().get(tenant.getTenantProfileId()); - isCore = systemContext.getServiceInfoProvider().isService(ServiceType.TB_CORE); isRuleEngine = systemContext.getServiceInfoProvider().isService(ServiceType.TB_RULE_ENGINE); if (isRuleEngine) { - try { - if (getApiUsageState().isReExecEnabled()) { - log.debug("[{}] Going to init rule chains", tenantId); - initRuleChains(); - } else { - log.info("[{}] Skip init of the rule chains due to API limits", tenantId); + if (systemContext.getPartitionService().isManagedByCurrentService(tenantId)) { + try { + if (getApiUsageState().isReExecEnabled()) { + log.debug("[{}] Going to init rule chains", tenantId); + initRuleChains(); + } else { + log.info("[{}] Skip init of the rule chains due to API limits", tenantId); + } + } catch (Exception e) { + log.info("Failed to check ApiUsage \"ReExecEnabled\"!!!", e); + cantFindTenant = true; } - } catch (Exception e) { - log.info("Failed to check ApiUsage \"ReExecEnabled\"!!!", e); - cantFindTenant = true; } } log.debug("[{}] Tenant actor started.", tenantId); diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbRuleEngineConsumerService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbRuleEngineConsumerService.java index a18b5c099c..19dea11665 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbRuleEngineConsumerService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbRuleEngineConsumerService.java @@ -156,7 +156,9 @@ public class DefaultTbRuleEngineConsumerService extends AbstractConsumerService< super.init("tb-rule-engine-consumer", "tb-rule-engine-notifications-consumer"); List queues = queueService.findAllQueues(); for (Queue configuration : queues) { - initConsumer(configuration); // TODO: if this Rule Engine is assigned specific profile, don't init other consumers and properly handle queue update events + if (partitionService.isManagedByCurrentService(configuration.getTenantId())) { + initConsumer(configuration); + } } } @@ -183,7 +185,12 @@ public class DefaultTbRuleEngineConsumerService extends AbstractConsumerService< if (event.getServiceType().equals(getServiceType())) { String serviceQueue = event.getQueueKey().getQueueName(); log.info("[{}] Subscribing to partitions: {}", serviceQueue, event.getPartitions()); - if (!consumerConfigurations.get(event.getQueueKey()).isConsumerPerPartition()) { + Queue configuration = consumerConfigurations.get(event.getQueueKey()); + if (configuration == null) { + log.warn("Received invalid partition change event for {} that is not managed by this service", event.getQueueKey()); + return; + } + if (!configuration.isConsumerPerPartition()) { consumers.get(event.getQueueKey()).subscribe(event.getPartitions()); } else { log.info("[{}] Subscribing consumer per partition: {}", serviceQueue, event.getPartitions()); @@ -425,32 +432,34 @@ public class DefaultTbRuleEngineConsumerService extends AbstractConsumerService< private void updateQueue(TransportProtos.QueueUpdateMsg queueUpdateMsg) { log.info("Received queue update msg: [{}]", queueUpdateMsg); - String queueName = queueUpdateMsg.getQueueName(); TenantId tenantId = new TenantId(new UUID(queueUpdateMsg.getTenantIdMSB(), queueUpdateMsg.getTenantIdLSB())); - QueueId queueId = new QueueId(new UUID(queueUpdateMsg.getQueueIdMSB(), queueUpdateMsg.getQueueIdLSB())); - QueueKey queueKey = new QueueKey(ServiceType.TB_RULE_ENGINE, queueUpdateMsg.getQueueName(), tenantId); - Queue queue = queueService.findQueueById(tenantId, queueId); - Queue oldQueue = consumerConfigurations.remove(queueKey); - if (oldQueue != null) { - if (oldQueue.isConsumerPerPartition()) { - TbTopicWithConsumerPerPartition consumerPerPartition = topicsConsumerPerPartition.remove(queueKey); - ReentrantLock lock = consumerPerPartition.getLock(); - try { - lock.lock(); - consumerPerPartition.getConsumers().values().forEach(TbQueueConsumer::unsubscribe); - } finally { - lock.unlock(); + if (partitionService.isManagedByCurrentService(tenantId)) { + QueueId queueId = new QueueId(new UUID(queueUpdateMsg.getQueueIdMSB(), queueUpdateMsg.getQueueIdLSB())); + String queueName = queueUpdateMsg.getQueueName(); + QueueKey queueKey = new QueueKey(ServiceType.TB_RULE_ENGINE, queueName, tenantId); + Queue queue = queueService.findQueueById(tenantId, queueId); + Queue oldQueue = consumerConfigurations.remove(queueKey); + if (oldQueue != null) { + if (oldQueue.isConsumerPerPartition()) { + TbTopicWithConsumerPerPartition consumerPerPartition = topicsConsumerPerPartition.remove(queueKey); + ReentrantLock lock = consumerPerPartition.getLock(); + try { + lock.lock(); + consumerPerPartition.getConsumers().values().forEach(TbQueueConsumer::unsubscribe); + } finally { + lock.unlock(); + } + } else { + TbQueueConsumer> consumer = consumers.remove(queueKey); + consumer.unsubscribe(); } - } else { - TbQueueConsumer> consumer = consumers.remove(queueKey); - consumer.unsubscribe(); } - } - initConsumer(queue); + initConsumer(queue); - if (!queue.isConsumerPerPartition()) { - launchConsumer(consumers.get(queueKey), consumerConfigurations.get(queueKey), consumerStats.get(queueKey), queueName); + if (!queue.isConsumerPerPartition()) { + launchConsumer(consumers.get(queueKey), consumerConfigurations.get(queueKey), consumerStats.get(queueKey), queueName); + } } partitionService.updateQueue(queueUpdateMsg); diff --git a/application/src/test/java/org/thingsboard/server/queue/discovery/HashPartitionServiceTest.java b/application/src/test/java/org/thingsboard/server/queue/discovery/HashPartitionServiceTest.java index 84024ecd1e..7bd9ec576f 100644 --- a/application/src/test/java/org/thingsboard/server/queue/discovery/HashPartitionServiceTest.java +++ b/application/src/test/java/org/thingsboard/server/queue/discovery/HashPartitionServiceTest.java @@ -46,6 +46,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Random; +import java.util.Set; import java.util.UUID; import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; @@ -274,4 +275,25 @@ public class HashPartitionServiceTest { }); } + @Test + public void testIsManagedByCurrentServiceCheck() { + TenantProfileId isolatedProfileId = new TenantProfileId(UUID.randomUUID()); + when(discoveryService.getAssignedTenantProfiles()).thenReturn(Set.of(isolatedProfileId.getId())); // dedicated server + TenantProfileId regularProfileId = new TenantProfileId(UUID.randomUUID()); + + TenantId isolatedTenantId = new TenantId(UUID.randomUUID()); + when(routingInfoService.getRoutingInfo(eq(isolatedTenantId))).thenReturn(new TenantRoutingInfo(isolatedTenantId, isolatedProfileId, true)); + TenantId regularTenantId = new TenantId(UUID.randomUUID()); + when(routingInfoService.getRoutingInfo(eq(regularTenantId))).thenReturn(new TenantRoutingInfo(regularTenantId, regularProfileId, false)); + + assertThat(clusterRoutingService.isManagedByCurrentService(isolatedTenantId)).isTrue(); + assertThat(clusterRoutingService.isManagedByCurrentService(regularTenantId)).isFalse(); + + + when(discoveryService.getAssignedTenantProfiles()).thenReturn(Collections.emptySet()); // common server + + assertThat(clusterRoutingService.isManagedByCurrentService(isolatedTenantId)).isTrue(); + assertThat(clusterRoutingService.isManagedByCurrentService(regularTenantId)).isTrue(); + } + } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/DefaultTbServiceInfoProvider.java b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/DefaultTbServiceInfoProvider.java index e9013ef345..64a8b70a1f 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/DefaultTbServiceInfoProvider.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/DefaultTbServiceInfoProvider.java @@ -85,6 +85,9 @@ public class DefaultTbServiceInfoProvider implements TbServiceInfoProvider { } else { serviceTypes = Collections.singletonList(ServiceType.of(serviceType)); } + if (!serviceTypes.contains(ServiceType.TB_RULE_ENGINE) || assignedTenantProfiles == null) { + assignedTenantProfiles = Collections.emptySet(); + } generateNewServiceInfoWithCurrentSystemInfo(); } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/HashPartitionService.java b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/HashPartitionService.java index 4c1894eae1..2db342d4e6 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/HashPartitionService.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/HashPartitionService.java @@ -186,6 +186,21 @@ public class HashPartitionService implements PartitionService { removeTenant(tenantId); } + @Override + public boolean isManagedByCurrentService(TenantId tenantId) { + Set assignedTenantProfiles = serviceInfoProvider.getAssignedTenantProfiles(); + if (assignedTenantProfiles.isEmpty()) { + // TODO: refactor this for common servers + return true; + } else { + if (tenantId.isSysTenantId()) { + return false; + } + TenantProfileId profileId = tenantRoutingInfoService.getRoutingInfo(tenantId).getProfileId(); + return assignedTenantProfiles.contains(profileId.getId()); + } + } + @Override public TopicPartitionInfo resolve(ServiceType serviceType, String queueName, TenantId tenantId, EntityId entityId) { TenantId isolatedOrSystemTenantId = getIsolatedOrSystemTenantId(serviceType, tenantId); diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/PartitionService.java b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/PartitionService.java index faa4d956a8..b55ba79f67 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/PartitionService.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/PartitionService.java @@ -64,4 +64,7 @@ public interface PartitionService { void updateQueue(TransportProtos.QueueUpdateMsg queueUpdateMsg); void removeQueue(TransportProtos.QueueDeleteMsg queueDeleteMsg); + + boolean isManagedByCurrentService(TenantId tenantId); + } From 97e9f43f65173e2813a11e0e952592bdebe2ce55 Mon Sep 17 00:00:00 2001 From: ViacheslavKlimov Date: Tue, 15 Aug 2023 17:20:52 +0300 Subject: [PATCH 20/23] Add createCondition to TbActorCtx --- .../org/thingsboard/server/actors/app/AppActor.java | 13 +++++-------- .../ruleChain/RuleChainActorMessageProcessor.java | 3 ++- .../actors/ruleChain/RuleChainManagerActor.java | 3 ++- .../server/actors/tenant/TenantActor.java | 3 ++- .../org/thingsboard/server/actors/TbActorCtx.java | 2 +- .../thingsboard/server/actors/TbActorMailbox.java | 4 ++-- 6 files changed, 14 insertions(+), 14 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/actors/app/AppActor.java b/application/src/main/java/org/thingsboard/server/actors/app/AppActor.java index 52abeca2c0..89a000594b 100644 --- a/application/src/main/java/org/thingsboard/server/actors/app/AppActor.java +++ b/application/src/main/java/org/thingsboard/server/actors/app/AppActor.java @@ -198,14 +198,11 @@ public class AppActor extends ContextAwareActor { } private Optional getOrCreateTenantActor(TenantId tenantId) { - if (systemContext.getServiceInfoProvider().isService(ServiceType.TB_CORE) || - systemContext.getPartitionService().isManagedByCurrentService(tenantId)) { - return Optional.of(ctx.getOrCreateChildActor(new TbEntityActorId(tenantId), - () -> DefaultActorService.TENANT_DISPATCHER_NAME, - () -> new TenantActor.ActorCreator(systemContext, tenantId))); - } else { - return Optional.empty(); - } + return Optional.ofNullable(ctx.getOrCreateChildActor(new TbEntityActorId(tenantId), + () -> DefaultActorService.TENANT_DISPATCHER_NAME, + () -> new TenantActor.ActorCreator(systemContext, tenantId), + () -> systemContext.getServiceInfoProvider().isService(ServiceType.TB_CORE) || + systemContext.getPartitionService().isManagedByCurrentService(tenantId))); } private void onToEdgeSessionMsg(EdgeSessionMsg msg) { diff --git a/application/src/main/java/org/thingsboard/server/actors/ruleChain/RuleChainActorMessageProcessor.java b/application/src/main/java/org/thingsboard/server/actors/ruleChain/RuleChainActorMessageProcessor.java index 4b868dfcbf..b3d7120969 100644 --- a/application/src/main/java/org/thingsboard/server/actors/ruleChain/RuleChainActorMessageProcessor.java +++ b/application/src/main/java/org/thingsboard/server/actors/ruleChain/RuleChainActorMessageProcessor.java @@ -167,7 +167,8 @@ public class RuleChainActorMessageProcessor extends ComponentMsgProcessor DefaultActorService.RULE_DISPATCHER_NAME, - () -> new RuleNodeActor.ActorCreator(systemContext, tenantId, entityId, ruleChainName, ruleNode.getId())); + () -> new RuleNodeActor.ActorCreator(systemContext, tenantId, entityId, ruleChainName, ruleNode.getId()), + () -> true); } private void initRoutes(RuleChain ruleChain, List ruleNodeList) { diff --git a/application/src/main/java/org/thingsboard/server/actors/ruleChain/RuleChainManagerActor.java b/application/src/main/java/org/thingsboard/server/actors/ruleChain/RuleChainManagerActor.java index 7f919754fc..987554c683 100644 --- a/application/src/main/java/org/thingsboard/server/actors/ruleChain/RuleChainManagerActor.java +++ b/application/src/main/java/org/thingsboard/server/actors/ruleChain/RuleChainManagerActor.java @@ -94,7 +94,8 @@ public abstract class RuleChainManagerActor extends ContextAwareActor { } else { return new RuleChainActor.ActorCreator(systemContext, tenantId, ruleChain); } - }); + }, + () -> true); } protected TbActorRef getEntityActorRef(EntityId entityId) { diff --git a/application/src/main/java/org/thingsboard/server/actors/tenant/TenantActor.java b/application/src/main/java/org/thingsboard/server/actors/tenant/TenantActor.java index 6cc249c87c..eddb932084 100644 --- a/application/src/main/java/org/thingsboard/server/actors/tenant/TenantActor.java +++ b/application/src/main/java/org/thingsboard/server/actors/tenant/TenantActor.java @@ -274,7 +274,8 @@ public class TenantActor extends RuleChainManagerActor { private TbActorRef getOrCreateDeviceActor(DeviceId deviceId) { return ctx.getOrCreateChildActor(new TbEntityActorId(deviceId), () -> DefaultActorService.DEVICE_DISPATCHER_NAME, - () -> new DeviceActorCreator(systemContext, tenantId, deviceId)); + () -> new DeviceActorCreator(systemContext, tenantId, deviceId), + () -> true); } private void onToEdgeSessionMsg(EdgeSessionMsg msg) { diff --git a/common/actor/src/main/java/org/thingsboard/server/actors/TbActorCtx.java b/common/actor/src/main/java/org/thingsboard/server/actors/TbActorCtx.java index 2a8f641c4e..bbe19fb3dd 100644 --- a/common/actor/src/main/java/org/thingsboard/server/actors/TbActorCtx.java +++ b/common/actor/src/main/java/org/thingsboard/server/actors/TbActorCtx.java @@ -32,7 +32,7 @@ public interface TbActorCtx extends TbActorRef { void stop(TbActorId target); - TbActorRef getOrCreateChildActor(TbActorId actorId, Supplier dispatcher, Supplier creator); + TbActorRef getOrCreateChildActor(TbActorId actorId, Supplier dispatcher, Supplier creator, Supplier createCondition); void broadcastToChildren(TbActorMsg msg); diff --git a/common/actor/src/main/java/org/thingsboard/server/actors/TbActorMailbox.java b/common/actor/src/main/java/org/thingsboard/server/actors/TbActorMailbox.java index 857c45ad84..34537c143c 100644 --- a/common/actor/src/main/java/org/thingsboard/server/actors/TbActorMailbox.java +++ b/common/actor/src/main/java/org/thingsboard/server/actors/TbActorMailbox.java @@ -214,9 +214,9 @@ public final class TbActorMailbox implements TbActorCtx { } @Override - public TbActorRef getOrCreateChildActor(TbActorId actorId, Supplier dispatcher, Supplier creator) { + public TbActorRef getOrCreateChildActor(TbActorId actorId, Supplier dispatcher, Supplier creator, Supplier createCondition) { TbActorRef actorRef = system.getActor(actorId); - if (actorRef == null) { + if (actorRef == null && createCondition.get()) { return system.createChildActor(dispatcher.get(), creator.get(), selfId); } else { return actorRef; From 916487105a0277fb76d071d672331456d82bd7c3 Mon Sep 17 00:00:00 2001 From: ViacheslavKlimov Date: Wed, 16 Aug 2023 15:36:01 +0300 Subject: [PATCH 21/23] Update profile isolation option description --- ui-ngx/src/assets/locale/locale.constant-en_US.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ui-ngx/src/assets/locale/locale.constant-en_US.json b/ui-ngx/src/assets/locale/locale.constant-en_US.json index b449cf1ba5..cb046cea48 100644 --- a/ui-ngx/src/assets/locale/locale.constant-en_US.json +++ b/ui-ngx/src/assets/locale/locale.constant-en_US.json @@ -3534,8 +3534,8 @@ "tenant-required": "Tenant is required", "search": "Search tenants", "selected-tenants": "{ count, plural, =1 {1 tenant} other {# tenants} } selected", - "isolated-tb-rule-engine": "Processing in isolated ThingsBoard Rule Engine container", - "isolated-tb-rule-engine-details": "Requires separate microservice(s) for the profile" + "isolated-tb-rule-engine": "Use isolated ThingsBoard Rule Engine queues", + "isolated-tb-rule-engine-details": "Each tenant will have its own processing queues" }, "tenant-profile": { "tenant-profile": "Tenant profile", From 46e0262419f92ce6f168800726fc4a2138c7664e Mon Sep 17 00:00:00 2001 From: ViacheslavKlimov Date: Wed, 16 Aug 2023 15:46:24 +0300 Subject: [PATCH 22/23] Update profile isolation option description --- ui-ngx/src/assets/locale/locale.constant-en_US.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ui-ngx/src/assets/locale/locale.constant-en_US.json b/ui-ngx/src/assets/locale/locale.constant-en_US.json index cb046cea48..513a10dba3 100644 --- a/ui-ngx/src/assets/locale/locale.constant-en_US.json +++ b/ui-ngx/src/assets/locale/locale.constant-en_US.json @@ -3535,7 +3535,7 @@ "search": "Search tenants", "selected-tenants": "{ count, plural, =1 {1 tenant} other {# tenants} } selected", "isolated-tb-rule-engine": "Use isolated ThingsBoard Rule Engine queues", - "isolated-tb-rule-engine-details": "Each tenant will have its own processing queues" + "isolated-tb-rule-engine-details": "Each tenant will have dedicated Rule Engine queues" }, "tenant-profile": { "tenant-profile": "Tenant profile", From d30f8a8352791529f145d8434827401aeeefe0e8 Mon Sep 17 00:00:00 2001 From: ViacheslavKlimov Date: Wed, 16 Aug 2023 17:53:44 +0300 Subject: [PATCH 23/23] Minor refactoring --- .../server/actors/app/AppActor.java | 31 +++++++++---------- .../queue/DefaultTbClusterService.java | 10 +++--- .../DefaultTbRuleEngineConsumerService.java | 4 +-- .../provider/KafkaMonolithQueueFactory.java | 2 +- .../KafkaTbRuleEngineQueueFactory.java | 2 +- .../profile/tenant-profile.component.ts | 12 +++---- 6 files changed, 29 insertions(+), 32 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/actors/app/AppActor.java b/application/src/main/java/org/thingsboard/server/actors/app/AppActor.java index 89a000594b..d4f04486e6 100644 --- a/application/src/main/java/org/thingsboard/server/actors/app/AppActor.java +++ b/application/src/main/java/org/thingsboard/server/actors/app/AppActor.java @@ -143,13 +143,9 @@ public class AppActor extends ContextAwareActor { if (TenantId.SYS_TENANT_ID.equals(msg.getTenantId())) { msg.getMsg().getCallback().onFailure(new RuleEngineException("Message has system tenant id!")); } else { - if (!deletedTenants.contains(msg.getTenantId())) { - getOrCreateTenantActor(msg.getTenantId()).ifPresentOrElse(actor -> { - actor.tell(msg); - }, () -> msg.getMsg().getCallback().onSuccess()); - } else { - msg.getMsg().getCallback().onSuccess(); - } + getOrCreateTenantActor(msg.getTenantId()).ifPresentOrElse(actor -> { + actor.tell(msg); + }, () -> msg.getMsg().getCallback().onSuccess()); } } @@ -182,22 +178,23 @@ public class AppActor extends ContextAwareActor { } private void onToDeviceActorMsg(TenantAwareMsg msg, boolean priority) { - if (!deletedTenants.contains(msg.getTenantId())) { - getOrCreateTenantActor(msg.getTenantId()).ifPresent(tenantActor -> { - if (priority) { - tenantActor.tellWithHighPriority(msg); - } else { - tenantActor.tell(msg); - } - }); - } else { + getOrCreateTenantActor(msg.getTenantId()).ifPresentOrElse(tenantActor -> { + if (priority) { + tenantActor.tellWithHighPriority(msg); + } else { + tenantActor.tell(msg); + } + }, () -> { if (msg instanceof TransportToDeviceActorMsgWrapper) { ((TransportToDeviceActorMsgWrapper) msg).getCallback().onSuccess(); } - } + }); } private Optional getOrCreateTenantActor(TenantId tenantId) { + if (deletedTenants.contains(tenantId)) { + return Optional.empty(); + } return Optional.ofNullable(ctx.getOrCreateChildActor(new TbEntityActorId(tenantId), () -> DefaultActorService.TENANT_DISPATCHER_NAME, () -> new TenantActor.ActorCreator(systemContext, tenantId), diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java index 5a2db8019e..76631aaa95 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java @@ -591,6 +591,11 @@ public class DefaultTbClusterService implements TbClusterService { tbTransportServices.removeAll(tbCoreServices); tbCoreServices.removeAll(tbRuleEngineServices); + for (String ruleEngineServiceId : tbRuleEngineServices) { + TopicPartitionInfo tpi = notificationsTopicService.getNotificationsTopic(ServiceType.TB_RULE_ENGINE, ruleEngineServiceId); + producerProvider.getRuleEngineNotificationsMsgProducer().send(tpi, new TbProtoQueueMsg<>(UUID.randomUUID(), ruleEngineMsg), null); + toRuleEngineNfs.incrementAndGet(); + } for (String coreServiceId : tbCoreServices) { TopicPartitionInfo tpi = notificationsTopicService.getNotificationsTopic(ServiceType.TB_CORE, coreServiceId); producerProvider.getTbCoreNotificationsMsgProducer().send(tpi, new TbProtoQueueMsg<>(UUID.randomUUID(), coreMsg), null); @@ -601,10 +606,5 @@ public class DefaultTbClusterService implements TbClusterService { producerProvider.getTransportNotificationsMsgProducer().send(tpi, new TbProtoQueueMsg<>(UUID.randomUUID(), transportMsg), null); toTransportNfs.incrementAndGet(); } - for (String ruleEngineServiceId : tbRuleEngineServices) { - TopicPartitionInfo tpi = notificationsTopicService.getNotificationsTopic(ServiceType.TB_RULE_ENGINE, ruleEngineServiceId); - producerProvider.getRuleEngineNotificationsMsgProducer().send(tpi, new TbProtoQueueMsg<>(UUID.randomUUID(), ruleEngineMsg), null); - toRuleEngineNfs.incrementAndGet(); - } } } diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbRuleEngineConsumerService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbRuleEngineConsumerService.java index 19dea11665..f4716b925f 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbRuleEngineConsumerService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbRuleEngineConsumerService.java @@ -104,7 +104,7 @@ public class DefaultTbRuleEngineConsumerService extends AbstractConsumerService< @Value("${queue.rule-engine.prometheus-stats.enabled:false}") boolean prometheusStatsEnabled; @Value("${queue.rule-engine.topic-deletion-delay:30}") - private int topicDeletionDelay; + private int topicDeletionDelayInSec; private final StatsFactory statsFactory; private final TbRuleEngineSubmitStrategyFactory submitStrategyFactory; @@ -506,7 +506,7 @@ public class DefaultTbRuleEngineConsumerService extends AbstractConsumerService< } private void processQueueDeletion(Queue queue, TbQueueConsumer> consumer) { - long finishTs = System.currentTimeMillis() + TimeUnit.SECONDS.toMillis(topicDeletionDelay); + long finishTs = System.currentTimeMillis() + TimeUnit.SECONDS.toMillis(topicDeletionDelayInSec); try { int n = 0; while (System.currentTimeMillis() <= finishTs) { diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaMonolithQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaMonolithQueueFactory.java index 779bf9fae3..22a16de64f 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaMonolithQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaMonolithQueueFactory.java @@ -187,7 +187,7 @@ public class KafkaMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEngi consumerBuilder.settings(kafkaSettings); consumerBuilder.topic(configuration.getTopic()); consumerBuilder.clientId("re-" + queueName + "-consumer-" + serviceInfoProvider.getServiceId() + "-" + consumerCount.incrementAndGet()); - consumerBuilder.groupId("re-" + queueName + (!configuration.getTenantId().isSysTenantId() ? "-" + configuration.getTenantId() : "") + "-consumer"); + consumerBuilder.groupId("re-" + queueName + (configuration.getTenantId().isSysTenantId() ? "" : ("-" + configuration.getTenantId())) + "-consumer"); consumerBuilder.decoder(msg -> new TbProtoQueueMsg<>(msg.getKey(), ToRuleEngineMsg.parseFrom(msg.getData()), msg.getHeaders())); consumerBuilder.admin(ruleEngineAdmin); consumerBuilder.statsService(consumerStatsService); diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbRuleEngineQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbRuleEngineQueueFactory.java index eb387a84f4..2e3bf784d7 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbRuleEngineQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbRuleEngineQueueFactory.java @@ -166,7 +166,7 @@ public class KafkaTbRuleEngineQueueFactory implements TbRuleEngineQueueFactory { consumerBuilder.settings(kafkaSettings); consumerBuilder.topic(configuration.getTopic()); consumerBuilder.clientId("re-" + queueName + "-consumer-" + serviceInfoProvider.getServiceId() + "-" + consumerCount.incrementAndGet()); - consumerBuilder.groupId("re-" + queueName + (!configuration.getTenantId().isSysTenantId() ? "-" + configuration.getTenantId() : "") + "-consumer"); + consumerBuilder.groupId("re-" + queueName + (configuration.getTenantId().isSysTenantId() ? "" : ("-" + configuration.getTenantId())) + "-consumer"); consumerBuilder.decoder(msg -> new TbProtoQueueMsg<>(msg.getKey(), ToRuleEngineMsg.parseFrom(msg.getData()), msg.getHeaders())); consumerBuilder.admin(ruleEngineAdmin); consumerBuilder.statsService(consumerStatsService); diff --git a/ui-ngx/src/app/modules/home/components/profile/tenant-profile.component.ts b/ui-ngx/src/app/modules/home/components/profile/tenant-profile.component.ts index e9142fa1fa..26d6611a3c 100644 --- a/ui-ngx/src/app/modules/home/components/profile/tenant-profile.component.ts +++ b/ui-ngx/src/app/modules/home/components/profile/tenant-profile.component.ts @@ -58,8 +58,8 @@ export class TenantProfileComponent extends EntityComponent { id: guid(), consumerPerPartition: true, name: 'Main', - packProcessingTimeout: 2000, - partitions: 2, + packProcessingTimeout: 10000, + partitions: 1, pollInterval: 2000, processingStrategy: { failurePercentage: 0, @@ -83,9 +83,9 @@ export class TenantProfileComponent extends EntityComponent { name: 'HighPriority', topic: 'tb_rule_engine.hp', pollInterval: 2000, - partitions: 2, + partitions: 1, consumerPerPartition: true, - packProcessingTimeout: 2000, + packProcessingTimeout: 10000, submitStrategy: { type: 'BURST', batchSize: 100 @@ -107,9 +107,9 @@ export class TenantProfileComponent extends EntityComponent { name: 'SequentialByOriginator', topic: 'tb_rule_engine.sq', pollInterval: 2000, - partitions: 2, + partitions: 1, consumerPerPartition: true, - packProcessingTimeout: 2000, + packProcessingTimeout: 10000, submitStrategy: { type: 'SEQUENTIAL_BY_ORIGINATOR', batchSize: 100