From 18ce96fd10580d04912a589012fa1249c6824542 Mon Sep 17 00:00:00 2001 From: ViacheslavKlimov Date: Fri, 14 Jul 2023 13:14:01 +0300 Subject: [PATCH 01/26] Ability to update isolated processing option in tenant profile --- .../DefaultTbTenantProfileService.java | 9 +- .../server/controller/AbstractWebTest.java | 21 +- .../service/queue/QueueServiceTest.java | 225 ++++++++++++++++++ .../service/ttl/AlarmsCleanUpServiceTest.java | 2 +- .../server/actors/TbActorMailbox.java | 6 +- .../queue/discovery/HashPartitionService.java | 16 +- .../validator/TenantProfileDataValidator.java | 2 - .../profile/tenant-profile.component.ts | 49 +++- 8 files changed, 300 insertions(+), 30 deletions(-) create mode 100644 application/src/test/java/org/thingsboard/server/service/queue/QueueServiceTest.java diff --git a/application/src/main/java/org/thingsboard/server/service/entitiy/tenant/profile/DefaultTbTenantProfileService.java b/application/src/main/java/org/thingsboard/server/service/entitiy/tenant/profile/DefaultTbTenantProfileService.java index 1219a5b929..98385ef8f7 100644 --- a/application/src/main/java/org/thingsboard/server/service/entitiy/tenant/profile/DefaultTbTenantProfileService.java +++ b/application/src/main/java/org/thingsboard/server/service/entitiy/tenant/profile/DefaultTbTenantProfileService.java @@ -44,12 +44,11 @@ public class DefaultTbTenantProfileService extends AbstractTbEntityService imple @Override public TenantProfile save(TenantId tenantId, TenantProfile tenantProfile, TenantProfile oldTenantProfile) throws ThingsboardException { TenantProfile savedTenantProfile = checkNotNull(tenantProfileService.saveTenantProfile(tenantId, tenantProfile)); - if (oldTenantProfile != null && savedTenantProfile.isIsolatedTbRuleEngine()) { - List tenantIds = tenantService.findTenantIdsByTenantProfileId(savedTenantProfile.getId()); - tbQueueService.updateQueuesByTenants(tenantIds, savedTenantProfile, oldTenantProfile); - } - tenantProfileCache.put(savedTenantProfile); + + List tenantIds = tenantService.findTenantIdsByTenantProfileId(savedTenantProfile.getId()); + tbQueueService.updateQueuesByTenants(tenantIds, savedTenantProfile, oldTenantProfile); + tbClusterService.onTenantProfileChange(savedTenantProfile, null); tbClusterService.broadcastEntityStateChangeEvent(TenantId.SYS_TENANT_ID, savedTenantProfile.getId(), tenantProfile.getId() == null ? ComponentLifecycleEvent.CREATED : ComponentLifecycleEvent.UPDATED); diff --git a/application/src/test/java/org/thingsboard/server/controller/AbstractWebTest.java b/application/src/test/java/org/thingsboard/server/controller/AbstractWebTest.java index c7580867e0..449164776a 100644 --- a/application/src/test/java/org/thingsboard/server/controller/AbstractWebTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/AbstractWebTest.java @@ -969,13 +969,20 @@ public abstract class AbstractWebTest extends AbstractInMemoryStorageTest { return (DeviceActorMessageProcessor) ReflectionTestUtils.getField(actor, "processor"); } - protected void updateDefaultTenantProfile(Consumer updater) throws ThingsboardException { - TenantProfile tenantProfile = tenantProfileService.findDefaultTenantProfile(TenantId.SYS_TENANT_ID); - TenantProfileData profileData = tenantProfile.getProfileData(); - DefaultTenantProfileConfiguration profileConfiguration = (DefaultTenantProfileConfiguration) profileData.getConfiguration(); - updater.accept(profileConfiguration); - tenantProfile.setProfileData(profileData); - tbTenantProfileService.save(TenantId.SYS_TENANT_ID, tenantProfile, null); + protected void updateDefaultTenantProfileConfig(Consumer updater) throws ThingsboardException { + updateDefaultTenantProfile(tenantProfile -> { + TenantProfileData profileData = tenantProfile.getProfileData(); + DefaultTenantProfileConfiguration profileConfiguration = (DefaultTenantProfileConfiguration) profileData.getConfiguration(); + updater.accept(profileConfiguration); + tenantProfile.setProfileData(profileData); + }); + } + + protected void updateDefaultTenantProfile(Consumer updater) throws ThingsboardException { + TenantProfile oldTenantProfile = tenantProfileService.findDefaultTenantProfile(TenantId.SYS_TENANT_ID); + TenantProfile tenantProfile = JacksonUtil.clone(oldTenantProfile); + updater.accept(tenantProfile); + tbTenantProfileService.save(TenantId.SYS_TENANT_ID, tenantProfile, oldTenantProfile); } } diff --git a/application/src/test/java/org/thingsboard/server/service/queue/QueueServiceTest.java b/application/src/test/java/org/thingsboard/server/service/queue/QueueServiceTest.java new file mode 100644 index 0000000000..4d2c50add3 --- /dev/null +++ b/application/src/test/java/org/thingsboard/server/service/queue/QueueServiceTest.java @@ -0,0 +1,225 @@ +/** + * Copyright © 2016-2023 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.queue; + +import org.junit.Before; +import org.junit.Test; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.mock.mockito.SpyBean; +import org.thingsboard.server.actors.ActorSystemContext; +import org.thingsboard.server.common.data.DataConstants; +import org.thingsboard.server.common.data.Device; +import org.thingsboard.server.common.data.DeviceProfile; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.queue.ProcessingStrategy; +import org.thingsboard.server.common.data.queue.ProcessingStrategyType; +import org.thingsboard.server.common.data.queue.Queue; +import org.thingsboard.server.common.data.queue.SubmitStrategy; +import org.thingsboard.server.common.data.queue.SubmitStrategyType; +import org.thingsboard.server.common.data.tenant.profile.TenantProfileQueueConfiguration; +import org.thingsboard.server.common.msg.TbMsg; +import org.thingsboard.server.common.msg.queue.QueueToRuleEngineMsg; +import org.thingsboard.server.common.msg.queue.ServiceType; +import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; +import org.thingsboard.server.controller.AbstractControllerTest; +import org.thingsboard.server.dao.queue.QueueService; +import org.thingsboard.server.dao.service.DaoSqlTest; +import org.thingsboard.server.queue.discovery.PartitionService; +import org.thingsboard.server.service.entitiy.queue.TbQueueService; +import org.thingsboard.server.service.profile.TbDeviceProfileCache; + +import java.util.List; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicReference; +import java.util.function.Consumer; +import java.util.function.Predicate; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.awaitility.Awaitility.await; +import static org.mockito.ArgumentMatchers.argThat; +import static org.mockito.Mockito.timeout; +import static org.mockito.Mockito.verify; + +@DaoSqlTest +public class QueueServiceTest extends AbstractControllerTest { + + @SpyBean + private ActorSystemContext actorContext; + @Autowired + private TbQueueService tbQueueService; + @Autowired + private QueueService queueService; + @SpyBean + private PartitionService partitionService; + @Autowired + private TbDeviceProfileCache deviceProfileCache; + + @Before + public void beforeEach() { + Queue mainQueue = queueService.findQueueByTenantIdAndName(TenantId.SYS_TENANT_ID, DataConstants.MAIN_QUEUE_NAME); + if (mainQueue == null) { + mainQueue = new Queue(TenantId.SYS_TENANT_ID, getMainQueueConfig()); + tbQueueService.saveQueue(mainQueue); + } + + Queue hpQueue = queueService.findQueueByTenantIdAndName(TenantId.SYS_TENANT_ID, DataConstants.HP_QUEUE_NAME); + if (hpQueue == null) { + hpQueue = new Queue(TenantId.SYS_TENANT_ID, getHighPriorityQueueConfig()); + tbQueueService.saveQueue(hpQueue); + } + } + + @Test + public void testQueuesUpdateOnTenantProfileUpdate() throws Exception { + loginTenantAdmin(); + DeviceProfile hpQueueProfile = createDeviceProfile("HighPriority profile"); + hpQueueProfile.setDefaultQueueName(DataConstants.HP_QUEUE_NAME); + hpQueueProfile = doPost("/api/deviceProfile", hpQueueProfile, DeviceProfile.class); + Device hpQueueDevice = createDevice("HP", hpQueueProfile.getName(), "HP"); + deviceProfileCache.evict(tenantId, hpQueueProfile.getId()); + + DeviceProfile mainQueueProfile = createDeviceProfile("Main profile"); + mainQueueProfile.setDefaultQueueName(DataConstants.MAIN_QUEUE_NAME); + mainQueueProfile = doPost("/api/deviceProfile", mainQueueProfile, DeviceProfile.class); + Device mainQueueDevice = createDevice("Main", mainQueueProfile.getName(), "Main"); + + verifyUsedQueueAndMessage(DataConstants.HP_QUEUE_NAME, hpQueueDevice.getId(), DataConstants.ATTRIBUTES_UPDATED, () -> { + doPost("/api/plugins/telemetry/DEVICE/" + hpQueueDevice.getId() + "/attributes/SERVER_SCOPE", "{\"test\":123}", String.class); + }, usedTpi -> { + assertThat(usedTpi.getTenantId()).get().isEqualTo(TenantId.SYS_TENANT_ID); + }); + verifyUsedQueueAndMessage(DataConstants.MAIN_QUEUE_NAME, mainQueueDevice.getId(), DataConstants.ATTRIBUTES_UPDATED, () -> { + doPost("/api/plugins/telemetry/DEVICE/" + mainQueueDevice.getId() + "/attributes/SERVER_SCOPE", "{\"test\":123}", String.class); + }, usedTpi -> { + assertThat(usedTpi.getTenantId()).get().isEqualTo(TenantId.SYS_TENANT_ID); + }); + + updateDefaultTenantProfile(tenantProfile -> { + tenantProfile.setIsolatedTbRuleEngine(true); + tenantProfile.getProfileData().setQueueConfiguration(List.of( + getMainQueueConfig() + )); + }); + + verifyUsedQueueAndMessage(DataConstants.MAIN_QUEUE_NAME, mainQueueDevice.getId(), DataConstants.ATTRIBUTES_UPDATED, () -> { + doPost("/api/plugins/telemetry/DEVICE/" + mainQueueDevice.getId() + "/attributes/SERVER_SCOPE", "{\"test\":123}", String.class); + }, usedTpi -> { + assertThat(usedTpi.getTenantId()).get().isEqualTo(tenantId); + }); + verifyUsedQueueAndMessage(DataConstants.HP_QUEUE_NAME, hpQueueDevice.getId(), DataConstants.ATTRIBUTES_UPDATED, () -> { + doPost("/api/plugins/telemetry/DEVICE/" + hpQueueDevice.getId() + "/attributes/SERVER_SCOPE", "{\"test\":123}", String.class); + }, usedTpi -> { + assertThat(usedTpi.getTopic()).endsWith("main"); + assertThat(usedTpi.getTenantId()).get().isEqualTo(tenantId); + }); + + updateDefaultTenantProfile(tenantProfile -> { + tenantProfile.setIsolatedTbRuleEngine(true); + tenantProfile.getProfileData().setQueueConfiguration(List.of( + getMainQueueConfig(), getHighPriorityQueueConfig() + )); + }); + + verifyUsedQueueAndMessage(DataConstants.HP_QUEUE_NAME, hpQueueDevice.getId(), DataConstants.ATTRIBUTES_UPDATED, () -> { + doPost("/api/plugins/telemetry/DEVICE/" + hpQueueDevice.getId() + "/attributes/SERVER_SCOPE", "{\"test\":123}", String.class); + }, usedTpi -> { + assertThat(usedTpi.getTopic()).endsWith("hp"); + assertThat(usedTpi.getTenantId()).get().isEqualTo(tenantId); + }); + verifyUsedQueueAndMessage(DataConstants.MAIN_QUEUE_NAME, mainQueueDevice.getId(), DataConstants.ATTRIBUTES_UPDATED, () -> { + doPost("/api/plugins/telemetry/DEVICE/" + mainQueueDevice.getId() + "/attributes/SERVER_SCOPE", "{\"test\":123}", String.class); + }, usedTpi -> { + assertThat(usedTpi.getTenantId()).get().isEqualTo(tenantId); + }); + } + + private void verifyUsedQueueAndMessage(String queue, EntityId entityId, String msgType, Runnable action, Consumer tpiAssert) { + await().atMost(15, TimeUnit.SECONDS) + .untilAsserted(() -> { + TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_RULE_ENGINE, queue, tenantId, entityId); + tpiAssert.accept(tpi); + }); + action.run(); + TbMsg tbMsg = awaitTbMsg(msg -> msg.getOriginator().equals(entityId) + && msg.getType().equals(msgType), 10000); + assertThat(tbMsg.getQueueName()).isEqualTo(queue); + + TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_RULE_ENGINE, queue, tenantId, entityId); + tpiAssert.accept(tpi); + } + + protected TbMsg awaitTbMsg(Predicate predicate, int timeoutMillis) { + AtomicReference tbMsgCaptor = new AtomicReference<>(); + verify(actorContext, timeout(timeoutMillis).atLeastOnce()).tell(argThat(actorMsg -> { + if (!(actorMsg instanceof QueueToRuleEngineMsg)) { + return false; + } + TbMsg tbMsg = ((QueueToRuleEngineMsg) actorMsg).getMsg(); + if (predicate.test(tbMsg)) { + tbMsgCaptor.set(tbMsg); + return true; + } + return false; + })); + return tbMsgCaptor.get(); + } + + private TenantProfileQueueConfiguration getHighPriorityQueueConfig() { + TenantProfileQueueConfiguration hpQueueConfig = new TenantProfileQueueConfiguration(); + hpQueueConfig.setName(DataConstants.HP_QUEUE_NAME); + hpQueueConfig.setTopic(DataConstants.HP_QUEUE_TOPIC); + hpQueueConfig.setPollInterval(25); + hpQueueConfig.setPartitions(10); + hpQueueConfig.setConsumerPerPartition(true); + hpQueueConfig.setPackProcessingTimeout(2000); + SubmitStrategy highPriorityQueueSubmitStrategy = new SubmitStrategy(); + highPriorityQueueSubmitStrategy.setType(SubmitStrategyType.BURST); + highPriorityQueueSubmitStrategy.setBatchSize(100); + hpQueueConfig.setSubmitStrategy(highPriorityQueueSubmitStrategy); + ProcessingStrategy highPriorityQueueProcessingStrategy = new ProcessingStrategy(); + highPriorityQueueProcessingStrategy.setType(ProcessingStrategyType.RETRY_FAILED_AND_TIMED_OUT); + highPriorityQueueProcessingStrategy.setRetries(0); + highPriorityQueueProcessingStrategy.setFailurePercentage(0); + highPriorityQueueProcessingStrategy.setPauseBetweenRetries(5); + highPriorityQueueProcessingStrategy.setMaxPauseBetweenRetries(5); + hpQueueConfig.setProcessingStrategy(highPriorityQueueProcessingStrategy); + return hpQueueConfig; + } + + private TenantProfileQueueConfiguration getMainQueueConfig() { + TenantProfileQueueConfiguration mainQueue = new TenantProfileQueueConfiguration(); + mainQueue.setName(DataConstants.MAIN_QUEUE_NAME); + mainQueue.setTopic(DataConstants.MAIN_QUEUE_TOPIC); + mainQueue.setPollInterval(25); + mainQueue.setPartitions(10); + mainQueue.setConsumerPerPartition(true); + mainQueue.setPackProcessingTimeout(2000); + SubmitStrategy mainQueueSubmitStrategy = new SubmitStrategy(); + mainQueueSubmitStrategy.setType(SubmitStrategyType.BURST); + mainQueueSubmitStrategy.setBatchSize(1000); + mainQueue.setSubmitStrategy(mainQueueSubmitStrategy); + ProcessingStrategy mainQueueProcessingStrategy = new ProcessingStrategy(); + mainQueueProcessingStrategy.setType(ProcessingStrategyType.SKIP_ALL_FAILURES); + mainQueueProcessingStrategy.setRetries(3); + mainQueueProcessingStrategy.setFailurePercentage(0); + mainQueueProcessingStrategy.setPauseBetweenRetries(3); + mainQueueProcessingStrategy.setMaxPauseBetweenRetries(3); + mainQueue.setProcessingStrategy(mainQueueProcessingStrategy); + return mainQueue; + } + +} diff --git a/application/src/test/java/org/thingsboard/server/service/ttl/AlarmsCleanUpServiceTest.java b/application/src/test/java/org/thingsboard/server/service/ttl/AlarmsCleanUpServiceTest.java index 6cabff788e..09e78df9ff 100644 --- a/application/src/test/java/org/thingsboard/server/service/ttl/AlarmsCleanUpServiceTest.java +++ b/application/src/test/java/org/thingsboard/server/service/ttl/AlarmsCleanUpServiceTest.java @@ -67,7 +67,7 @@ public class AlarmsCleanUpServiceTest extends AbstractControllerTest { @Test public void testAlarmsCleanUp() throws Exception { int ttlDays = 1; - updateDefaultTenantProfile(profileConfiguration -> { + updateDefaultTenantProfileConfig(profileConfiguration -> { profileConfiguration.setAlarmsTtlDays(ttlDays); }); diff --git a/common/actor/src/main/java/org/thingsboard/server/actors/TbActorMailbox.java b/common/actor/src/main/java/org/thingsboard/server/actors/TbActorMailbox.java index ad1604f7b0..857c45ad84 100644 --- a/common/actor/src/main/java/org/thingsboard/server/actors/TbActorMailbox.java +++ b/common/actor/src/main/java/org/thingsboard/server/actors/TbActorMailbox.java @@ -15,7 +15,8 @@ */ package org.thingsboard.server.actors; -import lombok.Data; +import lombok.Getter; +import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.msg.MsgType; @@ -31,7 +32,8 @@ import java.util.function.Predicate; import java.util.function.Supplier; @Slf4j -@Data +@Getter +@RequiredArgsConstructor public final class TbActorMailbox implements TbActorCtx { private static final boolean HIGH_PRIORITY = true; private static final boolean NORMAL_PRIORITY = false; diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/HashPartitionService.java b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/HashPartitionService.java index 14fb0369ee..f56233b144 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/HashPartitionService.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/HashPartitionService.java @@ -165,6 +165,9 @@ public class HashPartitionService implements PartitionService { partitionTopicsMap.put(queueKey, queueUpdateMsg.getQueueTopic()); partitionSizesMap.put(queueKey, queueUpdateMsg.getPartitions()); myPartitions.remove(queueKey); + if (!tenantId.isSysTenantId()) { + tenantRoutingInfoMap.remove(tenantId); + } } @Override @@ -358,16 +361,9 @@ public class HashPartitionService implements PartitionService { if (TenantId.SYS_TENANT_ID.equals(tenantId)) { return false; } - TenantRoutingInfo routingInfo = tenantRoutingInfoMap.get(tenantId); - if (routingInfo == null) { - synchronized (tenantRoutingInfoMap) { - routingInfo = tenantRoutingInfoMap.get(tenantId); - if (routingInfo == null) { - routingInfo = tenantRoutingInfoService.getRoutingInfo(tenantId); - tenantRoutingInfoMap.put(tenantId, routingInfo); - } - } - } + TenantRoutingInfo routingInfo = tenantRoutingInfoMap.computeIfAbsent(tenantId, k -> { + return tenantRoutingInfoService.getRoutingInfo(tenantId); + }); if (routingInfo == null) { throw new TenantNotFoundException(tenantId); } diff --git a/dao/src/main/java/org/thingsboard/server/dao/service/validator/TenantProfileDataValidator.java b/dao/src/main/java/org/thingsboard/server/dao/service/validator/TenantProfileDataValidator.java index 0477ccf32f..aa166ba12e 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/service/validator/TenantProfileDataValidator.java +++ b/dao/src/main/java/org/thingsboard/server/dao/service/validator/TenantProfileDataValidator.java @@ -99,8 +99,6 @@ public class TenantProfileDataValidator extends DataValidator { TenantProfile old = tenantProfileDao.findById(TenantId.SYS_TENANT_ID, tenantProfile.getId().getId()); if (old == null) { throw new DataValidationException("Can't update non existing tenant profile!"); - } else if (old.isIsolatedTbRuleEngine() != tenantProfile.isIsolatedTbRuleEngine()) { - throw new DataValidationException("Can't update isolatedTbRuleEngine property!"); } return old; } diff --git a/ui-ngx/src/app/modules/home/components/profile/tenant-profile.component.ts b/ui-ngx/src/app/modules/home/components/profile/tenant-profile.component.ts index 0257fb35af..2aafa97d99 100644 --- a/ui-ngx/src/app/modules/home/components/profile/tenant-profile.component.ts +++ b/ui-ngx/src/app/modules/home/components/profile/tenant-profile.component.ts @@ -76,6 +76,52 @@ export class TenantProfileComponent extends EntityComponent { additionalInfo: { description: '' } + }, + { + id: guid(), + name: 'HighPriority', + topic: 'tb_rule_engine.hp', + pollInterval: 25, + partitions: 10, + consumerPerPartition: true, + packProcessingTimeout: 2000, + submitStrategy: { + type: 'BURST', + batchSize: 100 + }, + processingStrategy: { + type: 'RETRY_FAILED_AND_TIMED_OUT', + retries: 0, + failurePercentage: 0, + pauseBetweenRetries: 5, + maxPauseBetweenRetries: 5 + }, + additionalInfo: { + description: '' + } + }, + { + id: guid(), + name: 'SequentialByOriginator', + topic: 'tb_rule_engine.sq', + pollInterval: 25, + partitions: 10, + consumerPerPartition: true, + packProcessingTimeout: 2000, + submitStrategy: { + type: 'SEQUENTIAL_BY_ORIGINATOR', + batchSize: 100 + }, + processingStrategy: { + type: 'RETRY_FAILED_AND_TIMED_OUT', + retries: 3, + failurePercentage: 0, + pauseBetweenRetries: 5, + maxPauseBetweenRetries: 5 + }, + additionalInfo: { + description: '' + } } ]; const formGroup = this.fb.group( @@ -118,9 +164,6 @@ export class TenantProfileComponent extends EntityComponent { if (this.entityForm) { if (this.isEditValue) { this.entityForm.enable({emitEvent: false}); - if (!this.isAdd) { - this.entityForm.get('isolatedTbRuleEngine').disable({emitEvent: false}); - } } else { this.entityForm.disable({emitEvent: false}); } From c1a07db64a4534e77410421d60bba7e429689b21 Mon Sep 17 00:00:00 2001 From: ViacheslavKlimov Date: Wed, 19 Jul 2023 13:39:44 +0300 Subject: [PATCH 02/26] Fix tenant profile update handling --- .../tenant/profile/DefaultTbTenantProfileService.java | 7 +++---- .../service/DefaultTransportTenantProfileCache.java | 1 + 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/entitiy/tenant/profile/DefaultTbTenantProfileService.java b/application/src/main/java/org/thingsboard/server/service/entitiy/tenant/profile/DefaultTbTenantProfileService.java index 98385ef8f7..15bcc2e040 100644 --- a/application/src/main/java/org/thingsboard/server/service/entitiy/tenant/profile/DefaultTbTenantProfileService.java +++ b/application/src/main/java/org/thingsboard/server/service/entitiy/tenant/profile/DefaultTbTenantProfileService.java @@ -45,14 +45,13 @@ public class DefaultTbTenantProfileService extends AbstractTbEntityService imple public TenantProfile save(TenantId tenantId, TenantProfile tenantProfile, TenantProfile oldTenantProfile) throws ThingsboardException { TenantProfile savedTenantProfile = checkNotNull(tenantProfileService.saveTenantProfile(tenantId, tenantProfile)); tenantProfileCache.put(savedTenantProfile); - - List tenantIds = tenantService.findTenantIdsByTenantProfileId(savedTenantProfile.getId()); - tbQueueService.updateQueuesByTenants(tenantIds, savedTenantProfile, oldTenantProfile); - tbClusterService.onTenantProfileChange(savedTenantProfile, null); tbClusterService.broadcastEntityStateChangeEvent(TenantId.SYS_TENANT_ID, savedTenantProfile.getId(), tenantProfile.getId() == null ? ComponentLifecycleEvent.CREATED : ComponentLifecycleEvent.UPDATED); + List tenantIds = tenantService.findTenantIdsByTenantProfileId(savedTenantProfile.getId()); + tbQueueService.updateQueuesByTenants(tenantIds, savedTenantProfile, oldTenantProfile); + return savedTenantProfile; } diff --git a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/DefaultTransportTenantProfileCache.java b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/DefaultTransportTenantProfileCache.java index d73e445516..12f89f29e6 100644 --- a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/DefaultTransportTenantProfileCache.java +++ b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/DefaultTransportTenantProfileCache.java @@ -82,6 +82,7 @@ public class DefaultTransportTenantProfileCache implements TransportTenantProfil if (profileOpt.isPresent()) { TenantProfile newProfile = profileOpt.get(); log.trace("[{}] put: {}", newProfile.getId(), newProfile); + profiles.put(newProfile.getId(), newProfile); Set affectedTenants = tenantProfileIds.get(newProfile.getId()); return new TenantProfileUpdateResult(newProfile, affectedTenants != null ? affectedTenants : Collections.emptySet()); } else { From 2fef3858a3bebcfb7bbc550d092c4f73431b05f3 Mon Sep 17 00:00:00 2001 From: ViacheslavKlimov Date: Wed, 19 Jul 2023 13:48:07 +0300 Subject: [PATCH 03/26] Fix queue poll duration config ignored; 1 partition by default for isolated queues --- .../queue/DefaultTbRuleEngineConsumerService.java | 2 +- .../components/profile/tenant-profile.component.ts | 12 ++++++------ 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbRuleEngineConsumerService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbRuleEngineConsumerService.java index f8f6a7d25f..3d02a1f60e 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbRuleEngineConsumerService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbRuleEngineConsumerService.java @@ -266,7 +266,7 @@ public class DefaultTbRuleEngineConsumerService extends AbstractConsumerService< updateCurrentThreadName(threadSuffix); while (!stopped && !consumer.isStopped()) { try { - List> msgs = consumer.poll(pollDuration); + List> msgs = consumer.poll(configuration.getPollInterval()); if (msgs.isEmpty()) { continue; } diff --git a/ui-ngx/src/app/modules/home/components/profile/tenant-profile.component.ts b/ui-ngx/src/app/modules/home/components/profile/tenant-profile.component.ts index 2aafa97d99..e96a237033 100644 --- a/ui-ngx/src/app/modules/home/components/profile/tenant-profile.component.ts +++ b/ui-ngx/src/app/modules/home/components/profile/tenant-profile.component.ts @@ -56,10 +56,10 @@ export class TenantProfileComponent extends EntityComponent { const mainQueue = [ { id: guid(), - consumerPerPartition: true, + consumerPerPartition: false, name: 'Main', packProcessingTimeout: 2000, - partitions: 10, + partitions: 1, pollInterval: 25, processingStrategy: { failurePercentage: 0, @@ -82,8 +82,8 @@ export class TenantProfileComponent extends EntityComponent { name: 'HighPriority', topic: 'tb_rule_engine.hp', pollInterval: 25, - partitions: 10, - consumerPerPartition: true, + partitions: 1, + consumerPerPartition: false, packProcessingTimeout: 2000, submitStrategy: { type: 'BURST', @@ -105,8 +105,8 @@ export class TenantProfileComponent extends EntityComponent { name: 'SequentialByOriginator', topic: 'tb_rule_engine.sq', pollInterval: 25, - partitions: 10, - consumerPerPartition: true, + partitions: 1, + consumerPerPartition: false, packProcessingTimeout: 2000, submitStrategy: { type: 'SEQUENTIAL_BY_ORIGINATOR', From bfd8ff934f8e626947374752d477761fc5e96782 Mon Sep 17 00:00:00 2001 From: ViacheslavKlimov Date: Wed, 19 Jul 2023 16:39:41 +0300 Subject: [PATCH 04/26] Use system queue with same name instead of Main when missing --- .../server/queue/discovery/HashPartitionService.java | 6 +++++- .../dao/service/BaseTenantProfileServiceTest.java | 11 ----------- 2 files changed, 5 insertions(+), 12 deletions(-) diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/HashPartitionService.java b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/HashPartitionService.java index f56233b144..e2312fa7cd 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/HashPartitionService.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/HashPartitionService.java @@ -186,7 +186,8 @@ public class HashPartitionService implements PartitionService { TenantId isolatedOrSystemTenantId = getIsolatedOrSystemTenantId(serviceType, tenantId); QueueKey queueKey = new QueueKey(serviceType, queueName, isolatedOrSystemTenantId); if (!partitionSizesMap.containsKey(queueKey)) { - queueKey = new QueueKey(serviceType, isolatedOrSystemTenantId); + // TODO: fallback to Main in case no system queue + queueKey = new QueueKey(serviceType, queueName, TenantId.SYS_TENANT_ID); } return resolve(queueKey, entityId); } @@ -207,6 +208,9 @@ public class HashPartitionService implements PartitionService { .putLong(entityId.getId().getLeastSignificantBits()).hash().asInt(); Integer partitionSize = partitionSizesMap.get(queueKey); + // if (partitionSize == null) { +// throw new IllegalStateException("Can't get partition ") +// } int partition = Math.abs(hash % partitionSize); return buildTopicPartitionInfo(queueKey, partition); diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/BaseTenantProfileServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/BaseTenantProfileServiceTest.java index 6d41da2965..4129a991cc 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/BaseTenantProfileServiceTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/BaseTenantProfileServiceTest.java @@ -187,17 +187,6 @@ public abstract class BaseTenantProfileServiceTest extends AbstractServiceTest { }); } - @Test - public void testSaveSameTenantProfileWithDifferentIsolatedTbRuleEngine() { - TenantProfile tenantProfile = this.createTenantProfile("Tenant Profile"); - TenantProfile savedTenantProfile = tenantProfileService.saveTenantProfile(TenantId.SYS_TENANT_ID, tenantProfile); - savedTenantProfile.setIsolatedTbRuleEngine(true); - addMainQueueConfig(savedTenantProfile); - Assertions.assertThrows(DataValidationException.class, () -> { - tenantProfileService.saveTenantProfile(TenantId.SYS_TENANT_ID, savedTenantProfile); - }); - } - @Test public void testDeleteTenantProfileWithExistingTenant() { TenantProfile tenantProfile = this.createTenantProfile("Tenant Profile"); From 1ee3c24532f5ee6b3183ae8c6bc8c69373feaaff Mon Sep 17 00:00:00 2001 From: ViacheslavKlimov Date: Wed, 19 Jul 2023 17:37:49 +0300 Subject: [PATCH 05/26] Refactor HashPartitionService.resolve(..) --- .../queue/discovery/HashPartitionService.java | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/HashPartitionService.java b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/HashPartitionService.java index e2312fa7cd..65f750f830 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/HashPartitionService.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/HashPartitionService.java @@ -48,6 +48,8 @@ import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; import java.util.stream.Collectors; +import static org.thingsboard.server.common.data.DataConstants.MAIN_QUEUE_NAME; + @Service @Slf4j public class HashPartitionService implements PartitionService { @@ -186,8 +188,15 @@ public class HashPartitionService implements PartitionService { TenantId isolatedOrSystemTenantId = getIsolatedOrSystemTenantId(serviceType, tenantId); QueueKey queueKey = new QueueKey(serviceType, queueName, isolatedOrSystemTenantId); if (!partitionSizesMap.containsKey(queueKey)) { - // TODO: fallback to Main in case no system queue - queueKey = new QueueKey(serviceType, queueName, TenantId.SYS_TENANT_ID); + if (isolatedOrSystemTenantId.isSysTenantId()) { + queueKey = new QueueKey(serviceType, TenantId.SYS_TENANT_ID); + } else { + queueKey = new QueueKey(serviceType, queueName, TenantId.SYS_TENANT_ID); + if (!MAIN_QUEUE_NAME.equals(queueName) && !partitionSizesMap.containsKey(queueKey)) { + queueKey = new QueueKey(serviceType, TenantId.SYS_TENANT_ID); + } + log.warn("Using queue {} instead of isolated {}", queueKey, queueName); + } } return resolve(queueKey, entityId); } @@ -208,9 +217,9 @@ public class HashPartitionService implements PartitionService { .putLong(entityId.getId().getLeastSignificantBits()).hash().asInt(); Integer partitionSize = partitionSizesMap.get(queueKey); - // if (partitionSize == null) { -// throw new IllegalStateException("Can't get partition ") -// } + if (partitionSize == null) { + throw new IllegalStateException("Partitions info for queue " + queueKey + " is missing"); + } int partition = Math.abs(hash % partitionSize); return buildTopicPartitionInfo(queueKey, partition); From 3b86c8c1f50b631ac8a28f031c3d63962d8f510e Mon Sep 17 00:00:00 2001 From: ViacheslavKlimov Date: Thu, 20 Jul 2023 13:34:06 +0300 Subject: [PATCH 06/26] Refactor HashPartitionService.resolve(..) --- .../server/queue/discovery/HashPartitionService.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/HashPartitionService.java b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/HashPartitionService.java index 65f750f830..31f517c236 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/HashPartitionService.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/HashPartitionService.java @@ -186,6 +186,9 @@ public class HashPartitionService implements PartitionService { @Override public TopicPartitionInfo resolve(ServiceType serviceType, String queueName, TenantId tenantId, EntityId entityId) { TenantId isolatedOrSystemTenantId = getIsolatedOrSystemTenantId(serviceType, tenantId); + if (queueName == null) { + queueName = MAIN_QUEUE_NAME; + } QueueKey queueKey = new QueueKey(serviceType, queueName, isolatedOrSystemTenantId); if (!partitionSizesMap.containsKey(queueKey)) { if (isolatedOrSystemTenantId.isSysTenantId()) { From 829433151207a3aa0cc6d4cd81238f92ce87fd62 Mon Sep 17 00:00:00 2001 From: ViacheslavKlimov Date: Fri, 21 Jul 2023 11:10:39 +0300 Subject: [PATCH 07/26] Configurable topic deletion delay --- .../service/entitiy/queue/DefaultTbQueueService.java | 12 +++++++----- application/src/main/resources/thingsboard.yml | 2 ++ 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/entitiy/queue/DefaultTbQueueService.java b/application/src/main/java/org/thingsboard/server/service/entitiy/queue/DefaultTbQueueService.java index 63e11aeb7a..abf7b694d9 100644 --- a/application/src/main/java/org/thingsboard/server/service/entitiy/queue/DefaultTbQueueService.java +++ b/application/src/main/java/org/thingsboard/server/service/entitiy/queue/DefaultTbQueueService.java @@ -17,6 +17,7 @@ package org.thingsboard.server.service.entitiy.queue; import lombok.AllArgsConstructor; import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Service; import org.thingsboard.server.cluster.TbClusterService; import org.thingsboard.server.common.data.TenantProfile; @@ -44,13 +45,15 @@ import java.util.stream.Collectors; @TbCoreComponent @AllArgsConstructor public class DefaultTbQueueService extends AbstractTbEntityService implements TbQueueService { - private static final long DELETE_DELAY = 30; private final QueueService queueService; private final TbClusterService tbClusterService; private final TbQueueAdmin tbQueueAdmin; private final SchedulerComponent scheduler; + @Value("${queue.rule-engine.topic_deletion_delay:60}") + private int topicDeletionDelay; + @Override public Queue saveQueue(Queue queue) { boolean create = queue.getId() == null; @@ -119,10 +122,9 @@ public class DefaultTbQueueService extends AbstractTbEntityService implements Tb for (int i = currentPartitions; i < oldPartitions; i++) { String fullTopicName = new TopicPartitionInfo(queue.getTopic(), queue.getTenantId(), i, false).getFullTopicName(); log.info("Removed partition [{}]", fullTopicName); - tbQueueAdmin.deleteTopic( - fullTopicName); + tbQueueAdmin.deleteTopic(fullTopicName); } - }, DELETE_DELAY, TimeUnit.SECONDS); + }, topicDeletionDelay, TimeUnit.SECONDS); } } else if (!oldQueue.equals(queue)) { tbClusterService.onQueueChange(queue); @@ -144,7 +146,7 @@ public class DefaultTbQueueService extends AbstractTbEntityService implements Tb log.error("Failed to delete queue [{}]", fullTopicName); } } - }, DELETE_DELAY, TimeUnit.SECONDS); + }, topicDeletionDelay, TimeUnit.SECONDS); notificationEntityService.notifySendMsgToEdgeService(queue.getTenantId(), queue.getId(), EdgeEventActionType.DELETED); } diff --git a/application/src/main/resources/thingsboard.yml b/application/src/main/resources/thingsboard.yml index 589540096c..2c09221148 100644 --- a/application/src/main/resources/thingsboard.yml +++ b/application/src/main/resources/thingsboard.yml @@ -1232,6 +1232,8 @@ queue: failure-percentage: "${TB_QUEUE_RE_SQ_PROCESSING_STRATEGY_FAILURE_PERCENTAGE:0}" # Skip retry if failures or timeouts are less then X percentage of messages; pause-between-retries: "${TB_QUEUE_RE_SQ_PROCESSING_STRATEGY_RETRY_PAUSE:5}" # Time in seconds to wait in consumer thread before retries; max-pause-between-retries: "${TB_QUEUE_RE_SQ_PROCESSING_STRATEGY_MAX_RETRY_PAUSE:5}" # Max allowed time in seconds for pause between retries. + # Delay between Queue update/delete and actual topic deletion. The delay is for Rule Engines to have time to unsubscribe from the topics, and for other services to stop publishing + topic_deletion_delay: "${TB_QUEUE_RULE_ENGINE_TOPIC_DELETION_DELAY_SECS:60}" transport: # For high priority notifications that require minimum latency and processing time notifications_topic: "${TB_QUEUE_TRANSPORT_NOTIFICATIONS_TOPIC:tb_transport.notifications}" From b2fe451bd2cfcdda6158c1c6d8028271d503e505 Mon Sep 17 00:00:00 2001 From: ViacheslavKlimov Date: Fri, 21 Jul 2023 13:20:05 +0300 Subject: [PATCH 08/26] Fix DefaultTbQueueService init --- .../server/service/entitiy/queue/DefaultTbQueueService.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/entitiy/queue/DefaultTbQueueService.java b/application/src/main/java/org/thingsboard/server/service/entitiy/queue/DefaultTbQueueService.java index abf7b694d9..0166a425d3 100644 --- a/application/src/main/java/org/thingsboard/server/service/entitiy/queue/DefaultTbQueueService.java +++ b/application/src/main/java/org/thingsboard/server/service/entitiy/queue/DefaultTbQueueService.java @@ -15,7 +15,7 @@ */ package org.thingsboard.server.service.entitiy.queue; -import lombok.AllArgsConstructor; +import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Service; @@ -43,7 +43,7 @@ import java.util.stream.Collectors; @Slf4j @Service @TbCoreComponent -@AllArgsConstructor +@RequiredArgsConstructor public class DefaultTbQueueService extends AbstractTbEntityService implements TbQueueService { private final QueueService queueService; From fa61783ac64110f4cb3123cb2975dd4e223674a3 Mon Sep 17 00:00:00 2001 From: ViacheslavKlimov Date: Tue, 25 Jul 2023 10:51:22 +0300 Subject: [PATCH 09/26] Log topics list on unsubscribe --- application/src/main/resources/thingsboard.yml | 2 +- .../server/queue/common/AbstractTbQueueConsumerTemplate.java | 4 +++- .../server/queue/kafka/TbKafkaConsumerTemplate.java | 1 - 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/application/src/main/resources/thingsboard.yml b/application/src/main/resources/thingsboard.yml index 2c09221148..163c325f52 100644 --- a/application/src/main/resources/thingsboard.yml +++ b/application/src/main/resources/thingsboard.yml @@ -1233,7 +1233,7 @@ queue: pause-between-retries: "${TB_QUEUE_RE_SQ_PROCESSING_STRATEGY_RETRY_PAUSE:5}" # Time in seconds to wait in consumer thread before retries; max-pause-between-retries: "${TB_QUEUE_RE_SQ_PROCESSING_STRATEGY_MAX_RETRY_PAUSE:5}" # Max allowed time in seconds for pause between retries. # Delay between Queue update/delete and actual topic deletion. The delay is for Rule Engines to have time to unsubscribe from the topics, and for other services to stop publishing - topic_deletion_delay: "${TB_QUEUE_RULE_ENGINE_TOPIC_DELETION_DELAY_SECS:60}" + topic_deletion_delay: "${TB_QUEUE_RULE_ENGINE_TOPIC_DELETION_DELAY_SECS:180}" transport: # For high priority notifications that require minimum latency and processing time notifications_topic: "${TB_QUEUE_TRANSPORT_NOTIFICATIONS_TOPIC:tb_transport.notifications}" diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/common/AbstractTbQueueConsumerTemplate.java b/common/queue/src/main/java/org/thingsboard/server/queue/common/AbstractTbQueueConsumerTemplate.java index 86146dabd1..a152e0c466 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/common/AbstractTbQueueConsumerTemplate.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/common/AbstractTbQueueConsumerTemplate.java @@ -162,7 +162,9 @@ public abstract class AbstractTbQueueConsumerTemplate i @Override public void unsubscribe() { - log.info("unsubscribe topic and stop consumer {}", getTopic()); + log.info("Unsubscribing from topics and stopping consumer for topics {}", partitions.stream() + .map(TopicPartitionInfo::getFullTopicName) + .collect(Collectors.joining(", "))); stopped = true; consumerLock.lock(); try { diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaConsumerTemplate.java b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaConsumerTemplate.java index c17a563d46..00bb7aa541 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaConsumerTemplate.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaConsumerTemplate.java @@ -114,7 +114,6 @@ public class TbKafkaConsumerTemplate extends AbstractTbQue @Override protected void doUnsubscribe() { - log.info("unsubscribe topic and close consumer for topic {}", getTopic()); if (consumer != null) { consumer.unsubscribe(); consumer.close(); From 6db3171ffb499323accd772b42e82ba76446149f Mon Sep 17 00:00:00 2001 From: ViacheslavKlimov Date: Tue, 25 Jul 2023 15:27:54 +0300 Subject: [PATCH 10/26] Refactoring --- .../server/controller/QueueController.java | 1 - .../DefaultTbRuleEngineConsumerService.java | 2 +- .../queue/TbRuleEngineConsumerStats.java | 13 +- .../server/controller/AbstractWebTest.java | 1 + .../controller/BaseTenantControllerTest.java | 170 ++++++++++++- .../BaseTenantProfileControllerTest.java | 17 -- .../notification/NotificationRuleApiTest.java | 4 +- .../service/queue/QueueServiceTest.java | 225 ------------------ .../queue/discovery/HashPartitionService.java | 2 +- .../profile/tenant-profile.component.ts | 12 +- 10 files changed, 181 insertions(+), 266 deletions(-) delete mode 100644 application/src/test/java/org/thingsboard/server/service/queue/QueueServiceTest.java diff --git a/application/src/main/java/org/thingsboard/server/controller/QueueController.java b/application/src/main/java/org/thingsboard/server/controller/QueueController.java index 0e79ae7955..faeea1a4aa 100644 --- a/application/src/main/java/org/thingsboard/server/controller/QueueController.java +++ b/application/src/main/java/org/thingsboard/server/controller/QueueController.java @@ -126,7 +126,6 @@ public class QueueController extends BaseController { @PreAuthorize("hasAnyAuthority('SYS_ADMIN')") @RequestMapping(value = "/queues", params = {"serviceType"}, method = RequestMethod.POST) @ResponseBody - public Queue saveQueue(@ApiParam(value = "A JSON value representing the queue.") @RequestBody Queue queue, @ApiParam(value = QUEUE_SERVICE_TYPE_DESCRIPTION, allowableValues = QUEUE_SERVICE_TYPE_ALLOWABLE_VALUES, required = true) diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbRuleEngineConsumerService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbRuleEngineConsumerService.java index 3d02a1f60e..598b41035f 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbRuleEngineConsumerService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbRuleEngineConsumerService.java @@ -152,7 +152,7 @@ public class DefaultTbRuleEngineConsumerService extends AbstractConsumerService< private void initConsumer(Queue configuration) { QueueKey queueKey = new QueueKey(ServiceType.TB_RULE_ENGINE, configuration); consumerConfigurations.putIfAbsent(queueKey, configuration); - consumerStats.putIfAbsent(queueKey, new TbRuleEngineConsumerStats(configuration.getName(), statsFactory)); + consumerStats.putIfAbsent(queueKey, new TbRuleEngineConsumerStats(configuration, statsFactory)); if (!configuration.isConsumerPerPartition()) { consumers.computeIfAbsent(queueKey, queueName -> tbRuleEngineQueueFactory.createToRuleEngineMsgConsumer(configuration)); } else { diff --git a/application/src/main/java/org/thingsboard/server/service/queue/TbRuleEngineConsumerStats.java b/application/src/main/java/org/thingsboard/server/service/queue/TbRuleEngineConsumerStats.java index 77e805eb62..2904c299ce 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/TbRuleEngineConsumerStats.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/TbRuleEngineConsumerStats.java @@ -18,6 +18,7 @@ package org.thingsboard.server.service.queue; import io.micrometer.core.instrument.Timer; import lombok.extern.slf4j.Slf4j; import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.queue.Queue; import org.thingsboard.server.common.msg.queue.RuleEngineException; import org.thingsboard.server.common.stats.StatsCounter; import org.thingsboard.server.common.stats.StatsFactory; @@ -63,9 +64,11 @@ public class TbRuleEngineConsumerStats { private final ConcurrentMap tenantExceptions = new ConcurrentHashMap<>(); private final String queueName; + private final TenantId tenantId; - public TbRuleEngineConsumerStats(String queueName, StatsFactory statsFactory) { - this.queueName = queueName; + public TbRuleEngineConsumerStats(Queue queue, StatsFactory statsFactory) { + this.queueName = queue.getName(); + this.tenantId = queue.getTenantId(); this.statsFactory = statsFactory; String statsKey = StatsType.RULE_ENGINE.getName() + "." + queueName; @@ -156,7 +159,11 @@ public class TbRuleEngineConsumerStats { counters.forEach(counter -> { stats.append(counter.getName()).append(" = [").append(counter.get()).append("] "); }); - log.info("[{}] Stats: {}", queueName, stats); + if (tenantId.isSysTenantId()) { + log.info("[{}] Stats: {}", queueName, stats); + } else { + log.info("[{}][{}] Stats: {}", queueName, tenantId, stats); + } } } diff --git a/application/src/test/java/org/thingsboard/server/controller/AbstractWebTest.java b/application/src/test/java/org/thingsboard/server/controller/AbstractWebTest.java index 449164776a..d3a77e28d1 100644 --- a/application/src/test/java/org/thingsboard/server/controller/AbstractWebTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/AbstractWebTest.java @@ -57,6 +57,7 @@ import org.springframework.test.web.servlet.request.MockHttpServletRequestBuilde import org.springframework.util.LinkedMultiValueMap; import org.springframework.util.MultiValueMap; import org.springframework.web.context.WebApplicationContext; +import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.rule.engine.api.MailService; import org.thingsboard.server.actors.DefaultTbActorSystem; import org.thingsboard.server.actors.TbActorId; diff --git a/application/src/test/java/org/thingsboard/server/controller/BaseTenantControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/BaseTenantControllerTest.java index 2e42009e5b..85e3fa35bd 100644 --- a/application/src/test/java/org/thingsboard/server/controller/BaseTenantControllerTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/BaseTenantControllerTest.java @@ -27,15 +27,20 @@ import org.junit.Before; import org.junit.Test; import org.mockito.ArgumentMatcher; import org.mockito.Mockito; +import org.springframework.boot.test.mock.mockito.SpyBean; import org.springframework.test.context.TestPropertySource; import org.springframework.test.web.servlet.ResultActions; import org.thingsboard.common.util.ThingsBoardExecutors; +import org.thingsboard.server.actors.ActorSystemContext; import org.thingsboard.server.common.data.DataConstants; +import org.thingsboard.server.common.data.Device; +import org.thingsboard.server.common.data.DeviceProfile; import org.thingsboard.server.common.data.StringUtils; import org.thingsboard.server.common.data.Tenant; import org.thingsboard.server.common.data.TenantInfo; import org.thingsboard.server.common.data.TenantProfile; import org.thingsboard.server.common.data.User; +import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; @@ -49,6 +54,12 @@ import org.thingsboard.server.common.data.security.Authority; import org.thingsboard.server.common.data.tenant.profile.DefaultTenantProfileConfiguration; import org.thingsboard.server.common.data.tenant.profile.TenantProfileData; import org.thingsboard.server.common.data.tenant.profile.TenantProfileQueueConfiguration; +import org.thingsboard.server.common.msg.TbMsg; +import org.thingsboard.server.common.msg.queue.QueueToRuleEngineMsg; +import org.thingsboard.server.common.msg.queue.ServiceType; +import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; +import org.thingsboard.server.dao.service.DaoSqlTest; +import org.thingsboard.server.queue.discovery.PartitionService; import java.util.ArrayList; import java.util.Comparator; @@ -57,12 +68,19 @@ import java.util.List; import java.util.Map; import java.util.Random; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicReference; +import java.util.function.Consumer; +import java.util.function.Predicate; import java.util.stream.Collectors; import static org.assertj.core.api.Assertions.assertThat; +import static org.awaitility.Awaitility.await; import static org.hamcrest.Matchers.containsString; +import static org.mockito.ArgumentMatchers.argThat; import static org.mockito.Mockito.never; +import static org.mockito.Mockito.timeout; import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; @TestPropertySource(properties = { @@ -78,6 +96,11 @@ public abstract class BaseTenantControllerTest extends AbstractControllerTest { ListeningExecutorService executor; + @SpyBean + private PartitionService partitionService; + @SpyBean + private ActorSystemContext actorContext; + @Before public void setUp() throws Exception { executor = MoreExecutors.listeningDecorator(ThingsBoardExecutors.newWorkStealingPool(8, getClass())); @@ -85,6 +108,12 @@ public abstract class BaseTenantControllerTest extends AbstractControllerTest { @After public void tearDown() throws Exception { + loginSysAdmin(); + for (Queue queue : doGetTypedWithPageLink("/api/queues?serviceType=TB_RULE_ENGINE&", new TypeReference>() {}, new PageLink(100)).getData()) { + if (!queue.getName().equals(DataConstants.MAIN_QUEUE_NAME)) { + doDelete("/api/queues/" + queue.getId()).andExpect(status().isOk()); + } + } executor.shutdownNow(); } @@ -518,10 +547,139 @@ public abstract class BaseTenantControllerTest extends AbstractControllerTest { doDelete("/api/tenant/" + tenant.getId().getId().toString()).andExpect(status().isOk()); } + @Test + public void testUpdateTenantProfileToIsolated() throws Exception { + loginSysAdmin(); + doPost("/api/queues?serviceType=TB_RULE_ENGINE", new Queue(TenantId.SYS_TENANT_ID, getQueueConfig(DataConstants.HP_QUEUE_NAME, DataConstants.HP_QUEUE_TOPIC))).andExpect(status().isOk()); + TenantProfile tenantProfile = new TenantProfile(); + tenantProfile.setName("Test profile"); + TenantProfileData tenantProfileData = new TenantProfileData(); + tenantProfileData.setConfiguration(new DefaultTenantProfileConfiguration()); + tenantProfile.setProfileData(tenantProfileData); + tenantProfile.setIsolatedTbRuleEngine(false); + tenantProfile = doPost("/api/tenantProfile", tenantProfile, TenantProfile.class); + createDifferentTenant(); + loginSysAdmin(); + savedDifferentTenant.setTenantProfileId(tenantProfile.getId()); + savedDifferentTenant = doPost("/api/tenant", savedDifferentTenant, Tenant.class); + TenantId tenantId = differentTenantId; + + loginDifferentTenant(); + DeviceProfile hpQueueProfile = createDeviceProfile("HighPriority profile"); + hpQueueProfile.setDefaultQueueName(DataConstants.HP_QUEUE_NAME); + hpQueueProfile = doPost("/api/deviceProfile", hpQueueProfile, DeviceProfile.class); + Device hpQueueDevice = createDevice("HP", hpQueueProfile.getName(), "HP"); + + DeviceProfile mainQueueProfile = createDeviceProfile("Main profile"); + mainQueueProfile.setDefaultQueueName(DataConstants.MAIN_QUEUE_NAME); + mainQueueProfile = doPost("/api/deviceProfile", mainQueueProfile, DeviceProfile.class); + Device mainQueueDevice = createDevice("Main", mainQueueProfile.getName(), "Main"); + + verifyUsedQueueAndMessage(DataConstants.HP_QUEUE_NAME, tenantId, hpQueueDevice.getId(), DataConstants.ATTRIBUTES_UPDATED, () -> { + doPost("/api/plugins/telemetry/DEVICE/" + hpQueueDevice.getId() + "/attributes/SERVER_SCOPE", "{\"test\":123}", String.class); + }, usedTpi -> { + assertThat(usedTpi.getTopic()).isEqualTo(DataConstants.HP_QUEUE_TOPIC); + assertThat(usedTpi.getTenantId()).get().isEqualTo(TenantId.SYS_TENANT_ID); + }); + verifyUsedQueueAndMessage(DataConstants.MAIN_QUEUE_NAME, tenantId, mainQueueDevice.getId(), DataConstants.ATTRIBUTES_UPDATED, () -> { + doPost("/api/plugins/telemetry/DEVICE/" + mainQueueDevice.getId() + "/attributes/SERVER_SCOPE", "{\"test\":123}", String.class); + }, usedTpi -> { + assertThat(usedTpi.getTopic()).isEqualTo(DataConstants.MAIN_QUEUE_TOPIC); + assertThat(usedTpi.getTenantId()).get().isEqualTo(TenantId.SYS_TENANT_ID); + }); + + loginSysAdmin(); + tenantProfile.setIsolatedTbRuleEngine(true); + tenantProfile.getProfileData().setQueueConfiguration(List.of( + getQueueConfig(DataConstants.MAIN_QUEUE_NAME, DataConstants.MAIN_QUEUE_TOPIC) + )); + tenantProfile = doPost("/api/tenantProfile", tenantProfile, TenantProfile.class); + + loginDifferentTenant(); + verifyUsedQueueAndMessage(DataConstants.MAIN_QUEUE_NAME, tenantId, mainQueueDevice.getId(), DataConstants.ATTRIBUTES_UPDATED, () -> { + doPost("/api/plugins/telemetry/DEVICE/" + mainQueueDevice.getId() + "/attributes/SERVER_SCOPE", "{\"test\":123}", String.class); + }, usedTpi -> { + assertThat(usedTpi.getTopic()).isEqualTo(DataConstants.MAIN_QUEUE_TOPIC); + assertThat(usedTpi.getTenantId()).get().isEqualTo(tenantId); + }); + verifyUsedQueueAndMessage(DataConstants.HP_QUEUE_NAME, tenantId, hpQueueDevice.getId(), DataConstants.ATTRIBUTES_UPDATED, () -> { + doPost("/api/plugins/telemetry/DEVICE/" + hpQueueDevice.getId() + "/attributes/SERVER_SCOPE", "{\"test\":123}", String.class); + }, usedTpi -> { + assertThat(usedTpi.getTopic()).isEqualTo(DataConstants.HP_QUEUE_TOPIC); + assertThat(usedTpi.getTenantId()).get().isEqualTo(TenantId.SYS_TENANT_ID); + }); + + loginSysAdmin(); + tenantProfile.setIsolatedTbRuleEngine(true); + tenantProfile.getProfileData().setQueueConfiguration(List.of( + getQueueConfig(DataConstants.MAIN_QUEUE_NAME, DataConstants.MAIN_QUEUE_TOPIC), + getQueueConfig(DataConstants.HP_QUEUE_NAME, DataConstants.HP_QUEUE_TOPIC) + )); + tenantProfile = doPost("/api/tenantProfile", tenantProfile, TenantProfile.class); + + loginDifferentTenant(); + verifyUsedQueueAndMessage(DataConstants.HP_QUEUE_NAME, tenantId, hpQueueDevice.getId(), DataConstants.ATTRIBUTES_UPDATED, () -> { + doPost("/api/plugins/telemetry/DEVICE/" + hpQueueDevice.getId() + "/attributes/SERVER_SCOPE", "{\"test\":123}", String.class); + }, usedTpi -> { + assertThat(usedTpi.getTopic()).isEqualTo(DataConstants.HP_QUEUE_TOPIC); + assertThat(usedTpi.getTenantId()).get().isEqualTo(tenantId); + }); + verifyUsedQueueAndMessage(DataConstants.MAIN_QUEUE_NAME, tenantId, mainQueueDevice.getId(), DataConstants.ATTRIBUTES_UPDATED, () -> { + doPost("/api/plugins/telemetry/DEVICE/" + mainQueueDevice.getId() + "/attributes/SERVER_SCOPE", "{\"test\":123}", String.class); + }, usedTpi -> { + assertThat(usedTpi.getTopic()).isEqualTo(DataConstants.MAIN_QUEUE_TOPIC); + assertThat(usedTpi.getTenantId()).get().isEqualTo(tenantId); + }); + } + + private void verifyUsedQueueAndMessage(String queue, TenantId tenantId, EntityId entityId, String msgType, Runnable action, Consumer tpiAssert) { + await().atMost(15, TimeUnit.SECONDS) + .untilAsserted(() -> { + TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_RULE_ENGINE, queue, tenantId, entityId); + tpiAssert.accept(tpi); + }); + action.run(); + TbMsg tbMsg = awaitTbMsg(msg -> msg.getOriginator().equals(entityId) + && msg.getType().equals(msgType), 10000); + assertThat(tbMsg.getQueueName()).isEqualTo(queue); + + TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_RULE_ENGINE, queue, tenantId, entityId); + tpiAssert.accept(tpi); + } + + protected TbMsg awaitTbMsg(Predicate predicate, int timeoutMillis) { + AtomicReference tbMsgCaptor = new AtomicReference<>(); + verify(actorContext, timeout(timeoutMillis).atLeastOnce()).tell(argThat(actorMsg -> { + if (!(actorMsg instanceof QueueToRuleEngineMsg)) { + return false; + } + TbMsg tbMsg = ((QueueToRuleEngineMsg) actorMsg).getMsg(); + if (predicate.test(tbMsg)) { + tbMsgCaptor.set(tbMsg); + return true; + } + return false; + })); + return tbMsgCaptor.get(); + } + private void addQueueConfig(TenantProfile tenantProfile, String queueName) { + TenantProfileQueueConfiguration queueConfiguration = getQueueConfig(queueName, "tb_rule_engine." + queueName.toLowerCase()); + TenantProfileData profileData = tenantProfile.getProfileData(); + + List configs = profileData.getQueueConfiguration(); + if (configs == null) { + configs = new ArrayList<>(); + } + configs.add(queueConfiguration); + profileData.setQueueConfiguration(configs); + tenantProfile.setProfileData(profileData); + } + + private TenantProfileQueueConfiguration getQueueConfig(String queueName, String topic) { TenantProfileQueueConfiguration queueConfiguration = new TenantProfileQueueConfiguration(); queueConfiguration.setName(queueName); - queueConfiguration.setTopic("tb_rule_engine." + queueName.toLowerCase()); + queueConfiguration.setTopic(topic); queueConfiguration.setPollInterval(25); queueConfiguration.setPartitions(1 + new Random().nextInt(99)); queueConfiguration.setConsumerPerPartition(true); @@ -537,15 +695,7 @@ public abstract class BaseTenantControllerTest extends AbstractControllerTest { processingStrategy.setPauseBetweenRetries(3); processingStrategy.setMaxPauseBetweenRetries(3); queueConfiguration.setProcessingStrategy(processingStrategy); - TenantProfileData profileData = tenantProfile.getProfileData(); - - List configs = profileData.getQueueConfiguration(); - if (configs == null) { - configs = new ArrayList<>(); - } - configs.add(queueConfiguration); - profileData.setQueueConfiguration(configs); - tenantProfile.setProfileData(profileData); + return queueConfiguration; } private List getQueuesFromConfig(List queueConfiguration, List queues) { diff --git a/application/src/test/java/org/thingsboard/server/controller/BaseTenantProfileControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/BaseTenantProfileControllerTest.java index d4116a873d..e8c3baada9 100644 --- a/application/src/test/java/org/thingsboard/server/controller/BaseTenantProfileControllerTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/BaseTenantProfileControllerTest.java @@ -165,23 +165,6 @@ public abstract class BaseTenantProfileControllerTest extends AbstractController testBroadcastEntityStateChangeEventNeverTenantProfile(); } - @Test - public void testSaveSameTenantProfileWithDifferentIsolatedTbRuleEngine() throws Exception { - loginSysAdmin(); - TenantProfile tenantProfile = this.createTenantProfile("Tenant Profile"); - TenantProfile savedTenantProfile = doPost("/api/tenantProfile", tenantProfile, TenantProfile.class); - savedTenantProfile.setIsolatedTbRuleEngine(true); - addMainQueueConfig(savedTenantProfile); - - Mockito.reset(tbClusterService); - - doPost("/api/tenantProfile", savedTenantProfile) - .andExpect(status().isBadRequest()) - .andExpect(statusReason(containsString("Can't update isolatedTbRuleEngine property"))); - - testBroadcastEntityStateChangeEventNeverTenantProfile(); - } - @Test public void testDeleteTenantProfileWithExistingTenant() throws Exception { loginSysAdmin(); diff --git a/application/src/test/java/org/thingsboard/server/service/notification/NotificationRuleApiTest.java b/application/src/test/java/org/thingsboard/server/service/notification/NotificationRuleApiTest.java index b03a914d1f..f0ece07cdc 100644 --- a/application/src/test/java/org/thingsboard/server/service/notification/NotificationRuleApiTest.java +++ b/application/src/test/java/org/thingsboard/server/service/notification/NotificationRuleApiTest.java @@ -301,7 +301,7 @@ public class NotificationRuleApiTest extends AbstractNotificationApiTest { @Test public void testNotificationRuleProcessing_entitiesLimit() throws Exception { int limit = 5; - updateDefaultTenantProfile(profileConfiguration -> { + updateDefaultTenantProfileConfig(profileConfiguration -> { profileConfiguration.setMaxDevices(limit); profileConfiguration.setMaxAssets(limit); profileConfiguration.setMaxCustomers(limit); @@ -396,7 +396,7 @@ public class NotificationRuleApiTest extends AbstractNotificationApiTest { @Test public void testNotificationRequestsPerRuleRateLimits() throws Exception { int notificationRequestsLimit = 10; - updateDefaultTenantProfile(profileConfiguration -> { + updateDefaultTenantProfileConfig(profileConfiguration -> { profileConfiguration.setTenantNotificationRequestsPerRuleRateLimit(notificationRequestsLimit + ":300"); }); diff --git a/application/src/test/java/org/thingsboard/server/service/queue/QueueServiceTest.java b/application/src/test/java/org/thingsboard/server/service/queue/QueueServiceTest.java deleted file mode 100644 index 4d2c50add3..0000000000 --- a/application/src/test/java/org/thingsboard/server/service/queue/QueueServiceTest.java +++ /dev/null @@ -1,225 +0,0 @@ -/** - * Copyright © 2016-2023 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.queue; - -import org.junit.Before; -import org.junit.Test; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.boot.test.mock.mockito.SpyBean; -import org.thingsboard.server.actors.ActorSystemContext; -import org.thingsboard.server.common.data.DataConstants; -import org.thingsboard.server.common.data.Device; -import org.thingsboard.server.common.data.DeviceProfile; -import org.thingsboard.server.common.data.id.EntityId; -import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.common.data.queue.ProcessingStrategy; -import org.thingsboard.server.common.data.queue.ProcessingStrategyType; -import org.thingsboard.server.common.data.queue.Queue; -import org.thingsboard.server.common.data.queue.SubmitStrategy; -import org.thingsboard.server.common.data.queue.SubmitStrategyType; -import org.thingsboard.server.common.data.tenant.profile.TenantProfileQueueConfiguration; -import org.thingsboard.server.common.msg.TbMsg; -import org.thingsboard.server.common.msg.queue.QueueToRuleEngineMsg; -import org.thingsboard.server.common.msg.queue.ServiceType; -import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; -import org.thingsboard.server.controller.AbstractControllerTest; -import org.thingsboard.server.dao.queue.QueueService; -import org.thingsboard.server.dao.service.DaoSqlTest; -import org.thingsboard.server.queue.discovery.PartitionService; -import org.thingsboard.server.service.entitiy.queue.TbQueueService; -import org.thingsboard.server.service.profile.TbDeviceProfileCache; - -import java.util.List; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicReference; -import java.util.function.Consumer; -import java.util.function.Predicate; - -import static org.assertj.core.api.Assertions.assertThat; -import static org.awaitility.Awaitility.await; -import static org.mockito.ArgumentMatchers.argThat; -import static org.mockito.Mockito.timeout; -import static org.mockito.Mockito.verify; - -@DaoSqlTest -public class QueueServiceTest extends AbstractControllerTest { - - @SpyBean - private ActorSystemContext actorContext; - @Autowired - private TbQueueService tbQueueService; - @Autowired - private QueueService queueService; - @SpyBean - private PartitionService partitionService; - @Autowired - private TbDeviceProfileCache deviceProfileCache; - - @Before - public void beforeEach() { - Queue mainQueue = queueService.findQueueByTenantIdAndName(TenantId.SYS_TENANT_ID, DataConstants.MAIN_QUEUE_NAME); - if (mainQueue == null) { - mainQueue = new Queue(TenantId.SYS_TENANT_ID, getMainQueueConfig()); - tbQueueService.saveQueue(mainQueue); - } - - Queue hpQueue = queueService.findQueueByTenantIdAndName(TenantId.SYS_TENANT_ID, DataConstants.HP_QUEUE_NAME); - if (hpQueue == null) { - hpQueue = new Queue(TenantId.SYS_TENANT_ID, getHighPriorityQueueConfig()); - tbQueueService.saveQueue(hpQueue); - } - } - - @Test - public void testQueuesUpdateOnTenantProfileUpdate() throws Exception { - loginTenantAdmin(); - DeviceProfile hpQueueProfile = createDeviceProfile("HighPriority profile"); - hpQueueProfile.setDefaultQueueName(DataConstants.HP_QUEUE_NAME); - hpQueueProfile = doPost("/api/deviceProfile", hpQueueProfile, DeviceProfile.class); - Device hpQueueDevice = createDevice("HP", hpQueueProfile.getName(), "HP"); - deviceProfileCache.evict(tenantId, hpQueueProfile.getId()); - - DeviceProfile mainQueueProfile = createDeviceProfile("Main profile"); - mainQueueProfile.setDefaultQueueName(DataConstants.MAIN_QUEUE_NAME); - mainQueueProfile = doPost("/api/deviceProfile", mainQueueProfile, DeviceProfile.class); - Device mainQueueDevice = createDevice("Main", mainQueueProfile.getName(), "Main"); - - verifyUsedQueueAndMessage(DataConstants.HP_QUEUE_NAME, hpQueueDevice.getId(), DataConstants.ATTRIBUTES_UPDATED, () -> { - doPost("/api/plugins/telemetry/DEVICE/" + hpQueueDevice.getId() + "/attributes/SERVER_SCOPE", "{\"test\":123}", String.class); - }, usedTpi -> { - assertThat(usedTpi.getTenantId()).get().isEqualTo(TenantId.SYS_TENANT_ID); - }); - verifyUsedQueueAndMessage(DataConstants.MAIN_QUEUE_NAME, mainQueueDevice.getId(), DataConstants.ATTRIBUTES_UPDATED, () -> { - doPost("/api/plugins/telemetry/DEVICE/" + mainQueueDevice.getId() + "/attributes/SERVER_SCOPE", "{\"test\":123}", String.class); - }, usedTpi -> { - assertThat(usedTpi.getTenantId()).get().isEqualTo(TenantId.SYS_TENANT_ID); - }); - - updateDefaultTenantProfile(tenantProfile -> { - tenantProfile.setIsolatedTbRuleEngine(true); - tenantProfile.getProfileData().setQueueConfiguration(List.of( - getMainQueueConfig() - )); - }); - - verifyUsedQueueAndMessage(DataConstants.MAIN_QUEUE_NAME, mainQueueDevice.getId(), DataConstants.ATTRIBUTES_UPDATED, () -> { - doPost("/api/plugins/telemetry/DEVICE/" + mainQueueDevice.getId() + "/attributes/SERVER_SCOPE", "{\"test\":123}", String.class); - }, usedTpi -> { - assertThat(usedTpi.getTenantId()).get().isEqualTo(tenantId); - }); - verifyUsedQueueAndMessage(DataConstants.HP_QUEUE_NAME, hpQueueDevice.getId(), DataConstants.ATTRIBUTES_UPDATED, () -> { - doPost("/api/plugins/telemetry/DEVICE/" + hpQueueDevice.getId() + "/attributes/SERVER_SCOPE", "{\"test\":123}", String.class); - }, usedTpi -> { - assertThat(usedTpi.getTopic()).endsWith("main"); - assertThat(usedTpi.getTenantId()).get().isEqualTo(tenantId); - }); - - updateDefaultTenantProfile(tenantProfile -> { - tenantProfile.setIsolatedTbRuleEngine(true); - tenantProfile.getProfileData().setQueueConfiguration(List.of( - getMainQueueConfig(), getHighPriorityQueueConfig() - )); - }); - - verifyUsedQueueAndMessage(DataConstants.HP_QUEUE_NAME, hpQueueDevice.getId(), DataConstants.ATTRIBUTES_UPDATED, () -> { - doPost("/api/plugins/telemetry/DEVICE/" + hpQueueDevice.getId() + "/attributes/SERVER_SCOPE", "{\"test\":123}", String.class); - }, usedTpi -> { - assertThat(usedTpi.getTopic()).endsWith("hp"); - assertThat(usedTpi.getTenantId()).get().isEqualTo(tenantId); - }); - verifyUsedQueueAndMessage(DataConstants.MAIN_QUEUE_NAME, mainQueueDevice.getId(), DataConstants.ATTRIBUTES_UPDATED, () -> { - doPost("/api/plugins/telemetry/DEVICE/" + mainQueueDevice.getId() + "/attributes/SERVER_SCOPE", "{\"test\":123}", String.class); - }, usedTpi -> { - assertThat(usedTpi.getTenantId()).get().isEqualTo(tenantId); - }); - } - - private void verifyUsedQueueAndMessage(String queue, EntityId entityId, String msgType, Runnable action, Consumer tpiAssert) { - await().atMost(15, TimeUnit.SECONDS) - .untilAsserted(() -> { - TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_RULE_ENGINE, queue, tenantId, entityId); - tpiAssert.accept(tpi); - }); - action.run(); - TbMsg tbMsg = awaitTbMsg(msg -> msg.getOriginator().equals(entityId) - && msg.getType().equals(msgType), 10000); - assertThat(tbMsg.getQueueName()).isEqualTo(queue); - - TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_RULE_ENGINE, queue, tenantId, entityId); - tpiAssert.accept(tpi); - } - - protected TbMsg awaitTbMsg(Predicate predicate, int timeoutMillis) { - AtomicReference tbMsgCaptor = new AtomicReference<>(); - verify(actorContext, timeout(timeoutMillis).atLeastOnce()).tell(argThat(actorMsg -> { - if (!(actorMsg instanceof QueueToRuleEngineMsg)) { - return false; - } - TbMsg tbMsg = ((QueueToRuleEngineMsg) actorMsg).getMsg(); - if (predicate.test(tbMsg)) { - tbMsgCaptor.set(tbMsg); - return true; - } - return false; - })); - return tbMsgCaptor.get(); - } - - private TenantProfileQueueConfiguration getHighPriorityQueueConfig() { - TenantProfileQueueConfiguration hpQueueConfig = new TenantProfileQueueConfiguration(); - hpQueueConfig.setName(DataConstants.HP_QUEUE_NAME); - hpQueueConfig.setTopic(DataConstants.HP_QUEUE_TOPIC); - hpQueueConfig.setPollInterval(25); - hpQueueConfig.setPartitions(10); - hpQueueConfig.setConsumerPerPartition(true); - hpQueueConfig.setPackProcessingTimeout(2000); - SubmitStrategy highPriorityQueueSubmitStrategy = new SubmitStrategy(); - highPriorityQueueSubmitStrategy.setType(SubmitStrategyType.BURST); - highPriorityQueueSubmitStrategy.setBatchSize(100); - hpQueueConfig.setSubmitStrategy(highPriorityQueueSubmitStrategy); - ProcessingStrategy highPriorityQueueProcessingStrategy = new ProcessingStrategy(); - highPriorityQueueProcessingStrategy.setType(ProcessingStrategyType.RETRY_FAILED_AND_TIMED_OUT); - highPriorityQueueProcessingStrategy.setRetries(0); - highPriorityQueueProcessingStrategy.setFailurePercentage(0); - highPriorityQueueProcessingStrategy.setPauseBetweenRetries(5); - highPriorityQueueProcessingStrategy.setMaxPauseBetweenRetries(5); - hpQueueConfig.setProcessingStrategy(highPriorityQueueProcessingStrategy); - return hpQueueConfig; - } - - private TenantProfileQueueConfiguration getMainQueueConfig() { - TenantProfileQueueConfiguration mainQueue = new TenantProfileQueueConfiguration(); - mainQueue.setName(DataConstants.MAIN_QUEUE_NAME); - mainQueue.setTopic(DataConstants.MAIN_QUEUE_TOPIC); - mainQueue.setPollInterval(25); - mainQueue.setPartitions(10); - mainQueue.setConsumerPerPartition(true); - mainQueue.setPackProcessingTimeout(2000); - SubmitStrategy mainQueueSubmitStrategy = new SubmitStrategy(); - mainQueueSubmitStrategy.setType(SubmitStrategyType.BURST); - mainQueueSubmitStrategy.setBatchSize(1000); - mainQueue.setSubmitStrategy(mainQueueSubmitStrategy); - ProcessingStrategy mainQueueProcessingStrategy = new ProcessingStrategy(); - mainQueueProcessingStrategy.setType(ProcessingStrategyType.SKIP_ALL_FAILURES); - mainQueueProcessingStrategy.setRetries(3); - mainQueueProcessingStrategy.setFailurePercentage(0); - mainQueueProcessingStrategy.setPauseBetweenRetries(3); - mainQueueProcessingStrategy.setMaxPauseBetweenRetries(3); - mainQueue.setProcessingStrategy(mainQueueProcessingStrategy); - return mainQueue; - } - -} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/HashPartitionService.java b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/HashPartitionService.java index 31f517c236..a8954b2b33 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/HashPartitionService.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/HashPartitionService.java @@ -198,7 +198,7 @@ public class HashPartitionService implements PartitionService { if (!MAIN_QUEUE_NAME.equals(queueName) && !partitionSizesMap.containsKey(queueKey)) { queueKey = new QueueKey(serviceType, TenantId.SYS_TENANT_ID); } - log.warn("Using queue {} instead of isolated {}", queueKey, queueName); + log.warn("Using queue {} instead of isolated {} for tenant {}", queueKey, queueName, isolatedOrSystemTenantId); } } return resolve(queueKey, entityId); diff --git a/ui-ngx/src/app/modules/home/components/profile/tenant-profile.component.ts b/ui-ngx/src/app/modules/home/components/profile/tenant-profile.component.ts index e96a237033..fa8b1636ff 100644 --- a/ui-ngx/src/app/modules/home/components/profile/tenant-profile.component.ts +++ b/ui-ngx/src/app/modules/home/components/profile/tenant-profile.component.ts @@ -56,10 +56,10 @@ export class TenantProfileComponent extends EntityComponent { const mainQueue = [ { id: guid(), - consumerPerPartition: false, + consumerPerPartition: true, name: 'Main', packProcessingTimeout: 2000, - partitions: 1, + partitions: 2, pollInterval: 25, processingStrategy: { failurePercentage: 0, @@ -82,8 +82,8 @@ export class TenantProfileComponent extends EntityComponent { name: 'HighPriority', topic: 'tb_rule_engine.hp', pollInterval: 25, - partitions: 1, - consumerPerPartition: false, + partitions: 2, + consumerPerPartition: true, packProcessingTimeout: 2000, submitStrategy: { type: 'BURST', @@ -105,8 +105,8 @@ export class TenantProfileComponent extends EntityComponent { name: 'SequentialByOriginator', topic: 'tb_rule_engine.sq', pollInterval: 25, - partitions: 1, - consumerPerPartition: false, + partitions: 2, + consumerPerPartition: true, packProcessingTimeout: 2000, submitStrategy: { type: 'SEQUENTIAL_BY_ORIGINATOR', From 55775f2815db89d78596c6a99a74ebb3cffaa830 Mon Sep 17 00:00:00 2001 From: ViacheslavKlimov Date: Thu, 27 Jul 2023 18:33:03 +0300 Subject: [PATCH 11/26] Consumer group per isolated tenant's queue; remove sleeping for Kafka consumer --- .../queue/common/AbstractTbQueueConsumerTemplate.java | 8 +++++++- .../server/queue/kafka/TbKafkaConsumerTemplate.java | 6 ++++++ .../server/queue/provider/KafkaMonolithQueueFactory.java | 2 +- .../queue/provider/KafkaTbRuleEngineQueueFactory.java | 2 +- 4 files changed, 15 insertions(+), 3 deletions(-) diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/common/AbstractTbQueueConsumerTemplate.java b/common/queue/src/main/java/org/thingsboard/server/queue/common/AbstractTbQueueConsumerTemplate.java index a152e0c466..5b0e84c2ed 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/common/AbstractTbQueueConsumerTemplate.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/common/AbstractTbQueueConsumerTemplate.java @@ -103,7 +103,9 @@ public abstract class AbstractTbQueueConsumerTemplate i consumerLock.unlock(); } - if (records.isEmpty()) { return sleepAndReturnEmpty(startNanos, durationInMillis); } + if (records.isEmpty() && !isLongPollingSupported()) { + return sleepAndReturnEmpty(startNanos, durationInMillis); + } return decodeRecords(records); } @@ -189,4 +191,8 @@ public abstract class AbstractTbQueueConsumerTemplate i abstract protected void doUnsubscribe(); + protected boolean isLongPollingSupported() { + return false; + } + } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaConsumerTemplate.java b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaConsumerTemplate.java index 00bb7aa541..9f58446966 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaConsumerTemplate.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaConsumerTemplate.java @@ -122,4 +122,10 @@ public class TbKafkaConsumerTemplate extends AbstractTbQue statsService.unregisterClientGroup(groupId); } } + + @Override + public boolean isLongPollingSupported() { + return true; + } + } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaMonolithQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaMonolithQueueFactory.java index 364abcff4c..779bf9fae3 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaMonolithQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaMonolithQueueFactory.java @@ -187,7 +187,7 @@ public class KafkaMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEngi consumerBuilder.settings(kafkaSettings); consumerBuilder.topic(configuration.getTopic()); consumerBuilder.clientId("re-" + queueName + "-consumer-" + serviceInfoProvider.getServiceId() + "-" + consumerCount.incrementAndGet()); - consumerBuilder.groupId("re-" + queueName + "-consumer"); + consumerBuilder.groupId("re-" + queueName + (!configuration.getTenantId().isSysTenantId() ? "-" + configuration.getTenantId() : "") + "-consumer"); consumerBuilder.decoder(msg -> new TbProtoQueueMsg<>(msg.getKey(), ToRuleEngineMsg.parseFrom(msg.getData()), msg.getHeaders())); consumerBuilder.admin(ruleEngineAdmin); consumerBuilder.statsService(consumerStatsService); diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbRuleEngineQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbRuleEngineQueueFactory.java index b8e07a45f7..eb387a84f4 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbRuleEngineQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbRuleEngineQueueFactory.java @@ -166,7 +166,7 @@ public class KafkaTbRuleEngineQueueFactory implements TbRuleEngineQueueFactory { consumerBuilder.settings(kafkaSettings); consumerBuilder.topic(configuration.getTopic()); consumerBuilder.clientId("re-" + queueName + "-consumer-" + serviceInfoProvider.getServiceId() + "-" + consumerCount.incrementAndGet()); - consumerBuilder.groupId("re-" + queueName + "-consumer"); + consumerBuilder.groupId("re-" + queueName + (!configuration.getTenantId().isSysTenantId() ? "-" + configuration.getTenantId() : "") + "-consumer"); consumerBuilder.decoder(msg -> new TbProtoQueueMsg<>(msg.getKey(), ToRuleEngineMsg.parseFrom(msg.getData()), msg.getHeaders())); consumerBuilder.admin(ruleEngineAdmin); consumerBuilder.statsService(consumerStatsService); From 137cc1809919c4535d0d64b8d8ebda311521f455 Mon Sep 17 00:00:00 2001 From: ViacheslavKlimov Date: Thu, 27 Jul 2023 19:58:31 +0300 Subject: [PATCH 12/26] Auto offset reset config for Kafka --- application/src/main/resources/thingsboard.yml | 1 + .../org/thingsboard/server/queue/kafka/TbKafkaSettings.java | 4 ++++ msa/vc-executor/src/main/resources/tb-vc-executor.yml | 1 + transport/coap/src/main/resources/tb-coap-transport.yml | 1 + transport/http/src/main/resources/tb-http-transport.yml | 1 + transport/lwm2m/src/main/resources/tb-lwm2m-transport.yml | 1 + transport/mqtt/src/main/resources/tb-mqtt-transport.yml | 1 + transport/snmp/src/main/resources/tb-snmp-transport.yml | 1 + 8 files changed, 11 insertions(+) diff --git a/application/src/main/resources/thingsboard.yml b/application/src/main/resources/thingsboard.yml index 163c325f52..96ee2793ed 100644 --- a/application/src/main/resources/thingsboard.yml +++ b/application/src/main/resources/thingsboard.yml @@ -1036,6 +1036,7 @@ queue: fetch_max_bytes: "${TB_QUEUE_KAFKA_FETCH_MAX_BYTES:134217728}" request.timeout.ms: "${TB_QUEUE_KAFKA_REQUEST_TIMEOUT_MS:30000}" # (30 seconds) # refer to https://docs.confluent.io/platform/current/installation/configuration/producer-configs.html#producerconfigs_request.timeout.ms session.timeout.ms: "${TB_QUEUE_KAFKA_SESSION_TIMEOUT_MS:10000}" # (10 seconds) # refer to https://docs.confluent.io/platform/current/installation/configuration/consumer-configs.html#consumerconfigs_session.timeout.ms + auto_offset_reset: "${TB_QUEUE_KAFKA_AUTO_OFFSET_RESET:earliest}" # earliest, latest or none use_confluent_cloud: "${TB_QUEUE_KAFKA_USE_CONFLUENT_CLOUD:false}" confluent: ssl.algorithm: "${TB_QUEUE_KAFKA_CONFLUENT_SSL_ALGORITHM:https}" diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaSettings.java b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaSettings.java index 41ae0d5ea3..d2a54128e3 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaSettings.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaSettings.java @@ -115,6 +115,9 @@ public class TbKafkaSettings { @Value("${queue.kafka.session.timeout.ms:10000}") private int sessionTimeoutMs; + @Value("${queue.kafka.auto_offset_reset:earliest}") + private String autoOffsetReset; + @Value("${queue.kafka.use_confluent_cloud:false}") private boolean useConfluent; @@ -155,6 +158,7 @@ public class TbKafkaSettings { props.put(ConsumerConfig.MAX_PARTITION_FETCH_BYTES_CONFIG, maxPartitionFetchBytes); props.put(ConsumerConfig.FETCH_MAX_BYTES_CONFIG, fetchMaxBytes); props.put(ConsumerConfig.MAX_POLL_INTERVAL_MS_CONFIG, maxPollIntervalMs); + props.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, autoOffsetReset); props.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class); props.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, ByteArrayDeserializer.class); diff --git a/msa/vc-executor/src/main/resources/tb-vc-executor.yml b/msa/vc-executor/src/main/resources/tb-vc-executor.yml index 352f94e091..9aa149280a 100644 --- a/msa/vc-executor/src/main/resources/tb-vc-executor.yml +++ b/msa/vc-executor/src/main/resources/tb-vc-executor.yml @@ -72,6 +72,7 @@ queue: fetch_max_bytes: "${TB_QUEUE_KAFKA_FETCH_MAX_BYTES:134217728}" request.timeout.ms: "${TB_QUEUE_KAFKA_REQUEST_TIMEOUT_MS:30000}" # (30 seconds) # refer to https://docs.confluent.io/platform/current/installation/configuration/producer-configs.html#producerconfigs_request.timeout.ms session.timeout.ms: "${TB_QUEUE_KAFKA_SESSION_TIMEOUT_MS:10000}" # (10 seconds) # refer to https://docs.confluent.io/platform/current/installation/configuration/consumer-configs.html#consumerconfigs_session.timeout.ms + auto_offset_reset: "${TB_QUEUE_KAFKA_AUTO_OFFSET_RESET:earliest}" # earliest, latest or none use_confluent_cloud: "${TB_QUEUE_KAFKA_USE_CONFLUENT_CLOUD:false}" confluent: ssl.algorithm: "${TB_QUEUE_KAFKA_CONFLUENT_SSL_ALGORITHM:https}" diff --git a/transport/coap/src/main/resources/tb-coap-transport.yml b/transport/coap/src/main/resources/tb-coap-transport.yml index 7ea553fe5c..5e0a7f2a9d 100644 --- a/transport/coap/src/main/resources/tb-coap-transport.yml +++ b/transport/coap/src/main/resources/tb-coap-transport.yml @@ -173,6 +173,7 @@ queue: fetch_max_bytes: "${TB_QUEUE_KAFKA_FETCH_MAX_BYTES:134217728}" request.timeout.ms: "${TB_QUEUE_KAFKA_REQUEST_TIMEOUT_MS:30000}" # (30 seconds) # refer to https://docs.confluent.io/platform/current/installation/configuration/producer-configs.html#producerconfigs_request.timeout.ms session.timeout.ms: "${TB_QUEUE_KAFKA_SESSION_TIMEOUT_MS:10000}" # (10 seconds) # refer to https://docs.confluent.io/platform/current/installation/configuration/consumer-configs.html#consumerconfigs_session.timeout.ms + auto_offset_reset: "${TB_QUEUE_KAFKA_AUTO_OFFSET_RESET:earliest}" # earliest, latest or none use_confluent_cloud: "${TB_QUEUE_KAFKA_USE_CONFLUENT_CLOUD:false}" confluent: ssl.algorithm: "${TB_QUEUE_KAFKA_CONFLUENT_SSL_ALGORITHM:https}" diff --git a/transport/http/src/main/resources/tb-http-transport.yml b/transport/http/src/main/resources/tb-http-transport.yml index 346ec48eae..f7af5c979a 100644 --- a/transport/http/src/main/resources/tb-http-transport.yml +++ b/transport/http/src/main/resources/tb-http-transport.yml @@ -158,6 +158,7 @@ queue: fetch_max_bytes: "${TB_QUEUE_KAFKA_FETCH_MAX_BYTES:134217728}" request.timeout.ms: "${TB_QUEUE_KAFKA_REQUEST_TIMEOUT_MS:30000}" # (30 seconds) # refer to https://docs.confluent.io/platform/current/installation/configuration/producer-configs.html#producerconfigs_request.timeout.ms session.timeout.ms: "${TB_QUEUE_KAFKA_SESSION_TIMEOUT_MS:10000}" # (10 seconds) # refer to https://docs.confluent.io/platform/current/installation/configuration/consumer-configs.html#consumerconfigs_session.timeout.ms + auto_offset_reset: "${TB_QUEUE_KAFKA_AUTO_OFFSET_RESET:earliest}" # earliest, latest or none use_confluent_cloud: "${TB_QUEUE_KAFKA_USE_CONFLUENT_CLOUD:false}" confluent: ssl.algorithm: "${TB_QUEUE_KAFKA_CONFLUENT_SSL_ALGORITHM:https}" diff --git a/transport/lwm2m/src/main/resources/tb-lwm2m-transport.yml b/transport/lwm2m/src/main/resources/tb-lwm2m-transport.yml index 4e8167d89d..3d649b3c2b 100644 --- a/transport/lwm2m/src/main/resources/tb-lwm2m-transport.yml +++ b/transport/lwm2m/src/main/resources/tb-lwm2m-transport.yml @@ -239,6 +239,7 @@ queue: fetch_max_bytes: "${TB_QUEUE_KAFKA_FETCH_MAX_BYTES:134217728}" request.timeout.ms: "${TB_QUEUE_KAFKA_REQUEST_TIMEOUT_MS:30000}" # (30 seconds) # refer to https://docs.confluent.io/platform/current/installation/configuration/producer-configs.html#producerconfigs_request.timeout.ms session.timeout.ms: "${TB_QUEUE_KAFKA_SESSION_TIMEOUT_MS:10000}" # (10 seconds) # refer to https://docs.confluent.io/platform/current/installation/configuration/consumer-configs.html#consumerconfigs_session.timeout.ms + auto_offset_reset: "${TB_QUEUE_KAFKA_AUTO_OFFSET_RESET:earliest}" # earliest, latest or none use_confluent_cloud: "${TB_QUEUE_KAFKA_USE_CONFLUENT_CLOUD:false}" confluent: ssl.algorithm: "${TB_QUEUE_KAFKA_CONFLUENT_SSL_ALGORITHM:https}" diff --git a/transport/mqtt/src/main/resources/tb-mqtt-transport.yml b/transport/mqtt/src/main/resources/tb-mqtt-transport.yml index 1e0b1ebcd4..38f569cfd7 100644 --- a/transport/mqtt/src/main/resources/tb-mqtt-transport.yml +++ b/transport/mqtt/src/main/resources/tb-mqtt-transport.yml @@ -188,6 +188,7 @@ queue: fetch_max_bytes: "${TB_QUEUE_KAFKA_FETCH_MAX_BYTES:134217728}" request.timeout.ms: "${TB_QUEUE_KAFKA_REQUEST_TIMEOUT_MS:30000}" # (30 seconds) # refer to https://docs.confluent.io/platform/current/installation/configuration/producer-configs.html#producerconfigs_request.timeout.ms session.timeout.ms: "${TB_QUEUE_KAFKA_SESSION_TIMEOUT_MS:10000}" # (10 seconds) # refer to https://docs.confluent.io/platform/current/installation/configuration/consumer-configs.html#consumerconfigs_session.timeout.ms + auto_offset_reset: "${TB_QUEUE_KAFKA_AUTO_OFFSET_RESET:earliest}" # earliest, latest or none use_confluent_cloud: "${TB_QUEUE_KAFKA_USE_CONFLUENT_CLOUD:false}" confluent: ssl.algorithm: "${TB_QUEUE_KAFKA_CONFLUENT_SSL_ALGORITHM:https}" diff --git a/transport/snmp/src/main/resources/tb-snmp-transport.yml b/transport/snmp/src/main/resources/tb-snmp-transport.yml index 9f086bcbc5..c4d76dbf30 100644 --- a/transport/snmp/src/main/resources/tb-snmp-transport.yml +++ b/transport/snmp/src/main/resources/tb-snmp-transport.yml @@ -134,6 +134,7 @@ queue: fetch_max_bytes: "${TB_QUEUE_KAFKA_FETCH_MAX_BYTES:134217728}" request.timeout.ms: "${TB_QUEUE_KAFKA_REQUEST_TIMEOUT_MS:30000}" # (30 seconds) # refer to https://docs.confluent.io/platform/current/installation/configuration/producer-configs.html#producerconfigs_request.timeout.ms session.timeout.ms: "${TB_QUEUE_KAFKA_SESSION_TIMEOUT_MS:10000}" # (10 seconds) # refer to https://docs.confluent.io/platform/current/installation/configuration/consumer-configs.html#consumerconfigs_session.timeout.ms + auto_offset_reset: "${TB_QUEUE_KAFKA_AUTO_OFFSET_RESET:earliest}" # earliest, latest or none use_confluent_cloud: "${TB_QUEUE_KAFKA_USE_CONFLUENT_CLOUD:false}" confluent: ssl.algorithm: "${TB_QUEUE_KAFKA_CONFLUENT_SSL_ALGORITHM:https}" From 5862b417aa48859c9f8038c927f27ed36f86aae8 Mon Sep 17 00:00:00 2001 From: ViacheslavKlimov Date: Fri, 28 Jul 2023 12:06:04 +0300 Subject: [PATCH 13/26] Add custom topic properties configuration --- .../entitiy/queue/DefaultTbQueueService.java | 8 ++++++-- .../org/thingsboard/server/queue/TbQueueAdmin.java | 6 +++++- .../server/common/data/queue/Queue.java | 14 +++++++++++++- .../queue/RuleEngineTbQueueAdminFactory.java | 2 +- .../queue/azure/servicebus/TbServiceBusAdmin.java | 7 ++++--- .../server/queue/kafka/TbKafkaAdmin.java | 6 +++--- .../provider/InMemoryTbTransportQueueFactory.java | 2 +- .../server/queue/pubsub/TbPubSubAdmin.java | 2 +- .../server/queue/rabbitmq/TbRabbitMqAdmin.java | 9 ++++++++- .../queue/rabbitmq/TbRabbitMqQueueArguments.java | 8 ++++---- .../server/queue/sqs/TbAwsSqsAdmin.java | 4 +++- .../server/queue/sqs/TbAwsSqsQueueAttributes.java | 8 +++++++- .../server/queue/util/PropertyUtils.java | 14 ++++++++++++++ .../queue/tenant-profile-queues.component.ts | 3 ++- .../components/profile/tenant-profile.component.ts | 9 ++++++--- .../components/queue/queue-form.component.html | 5 +++++ .../home/components/queue/queue-form.component.ts | 3 ++- ui-ngx/src/app/shared/models/queue.models.ts | 1 + .../src/assets/locale/locale.constant-en_US.json | 2 ++ 19 files changed, 88 insertions(+), 25 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/entitiy/queue/DefaultTbQueueService.java b/application/src/main/java/org/thingsboard/server/service/entitiy/queue/DefaultTbQueueService.java index 0166a425d3..9e3d38cf32 100644 --- a/application/src/main/java/org/thingsboard/server/service/entitiy/queue/DefaultTbQueueService.java +++ b/application/src/main/java/org/thingsboard/server/service/entitiy/queue/DefaultTbQueueService.java @@ -96,7 +96,9 @@ public class DefaultTbQueueService extends AbstractTbEntityService implements Tb private void onQueueCreated(Queue queue) { for (int i = 0; i < queue.getPartitions(); i++) { tbQueueAdmin.createTopicIfNotExists( - new TopicPartitionInfo(queue.getTopic(), queue.getTenantId(), i, false).getFullTopicName()); + new TopicPartitionInfo(queue.getTopic(), queue.getTenantId(), i, false).getFullTopicName(), + queue.getCustomProperties() + ); } tbClusterService.onQueueChange(queue); @@ -111,7 +113,9 @@ public class DefaultTbQueueService extends AbstractTbEntityService implements Tb log.info("Added [{}] new partitions to [{}] queue", currentPartitions - oldPartitions, queue.getName()); for (int i = oldPartitions; i < currentPartitions; i++) { tbQueueAdmin.createTopicIfNotExists( - new TopicPartitionInfo(queue.getTopic(), queue.getTenantId(), i, false).getFullTopicName()); + new TopicPartitionInfo(queue.getTopic(), queue.getTenantId(), i, false).getFullTopicName(), + queue.getCustomProperties() + ); } tbClusterService.onQueueChange(queue); } else { diff --git a/common/cluster-api/src/main/java/org/thingsboard/server/queue/TbQueueAdmin.java b/common/cluster-api/src/main/java/org/thingsboard/server/queue/TbQueueAdmin.java index 4b2bde733e..19aa0284ea 100644 --- a/common/cluster-api/src/main/java/org/thingsboard/server/queue/TbQueueAdmin.java +++ b/common/cluster-api/src/main/java/org/thingsboard/server/queue/TbQueueAdmin.java @@ -17,7 +17,11 @@ package org.thingsboard.server.queue; public interface TbQueueAdmin { - void createTopicIfNotExists(String topic); + default void createTopicIfNotExists(String topic) { + createTopicIfNotExists(topic, null); + } + + void createTopicIfNotExists(String topic, String properties); void destroy(); diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/queue/Queue.java b/common/data/src/main/java/org/thingsboard/server/common/data/queue/Queue.java index f757998d04..deb0e57f6b 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/queue/Queue.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/queue/Queue.java @@ -15,6 +15,8 @@ */ package org.thingsboard.server.common.data.queue; +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.fasterxml.jackson.databind.JsonNode; import lombok.Data; import org.thingsboard.server.common.data.HasName; import org.thingsboard.server.common.data.HasTenantId; @@ -25,6 +27,8 @@ import org.thingsboard.server.common.data.tenant.profile.TenantProfileQueueConfi import org.thingsboard.server.common.data.validation.Length; import org.thingsboard.server.common.data.validation.NoXss; +import java.util.Optional; + @Data public class Queue extends SearchTextBasedWithAdditionalInfo implements HasName, HasTenantId { private TenantId tenantId; @@ -65,4 +69,12 @@ public class Queue extends SearchTextBasedWithAdditionalInfo implements public String getSearchText() { return getName(); } -} \ No newline at end of file + + @JsonIgnore + public String getCustomProperties() { + return Optional.ofNullable(getAdditionalInfo()) + .map(info -> info.get("customProperties")) + .filter(JsonNode::isTextual).map(JsonNode::asText).orElse(null); + } + +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/RuleEngineTbQueueAdminFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/RuleEngineTbQueueAdminFactory.java index fe29c3a04f..7a8764325f 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/RuleEngineTbQueueAdminFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/RuleEngineTbQueueAdminFactory.java @@ -99,7 +99,7 @@ public class RuleEngineTbQueueAdminFactory { return new TbQueueAdmin() { @Override - public void createTopicIfNotExists(String topic) { + public void createTopicIfNotExists(String topic, String properties) { } @Override diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/azure/servicebus/TbServiceBusAdmin.java b/common/queue/src/main/java/org/thingsboard/server/queue/azure/servicebus/TbServiceBusAdmin.java index e171bb7a31..d95d2064a4 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/azure/servicebus/TbServiceBusAdmin.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/azure/servicebus/TbServiceBusAdmin.java @@ -22,6 +22,7 @@ import com.microsoft.azure.servicebus.primitives.MessagingEntityAlreadyExistsExc import com.microsoft.azure.servicebus.primitives.ServiceBusException; import lombok.extern.slf4j.Slf4j; import org.thingsboard.server.queue.TbQueueAdmin; +import org.thingsboard.server.queue.util.PropertyUtils; import java.io.IOException; import java.time.Duration; @@ -60,7 +61,7 @@ public class TbServiceBusAdmin implements TbQueueAdmin { } @Override - public void createTopicIfNotExists(String topic) { + public void createTopicIfNotExists(String topic, String properties) { if (queues.contains(topic)) { return; } @@ -68,7 +69,7 @@ public class TbServiceBusAdmin implements TbQueueAdmin { try { QueueDescription queueDescription = new QueueDescription(topic); queueDescription.setRequiresDuplicateDetection(false); - setQueueConfigs(queueDescription); + setQueueConfigs(queueDescription, PropertyUtils.getProps(queueConfigs, properties)); client.createQueue(queueDescription); queues.add(topic); @@ -107,7 +108,7 @@ public class TbServiceBusAdmin implements TbQueueAdmin { } } - private void setQueueConfigs(QueueDescription queueDescription) { + private void setQueueConfigs(QueueDescription queueDescription, Map queueConfigs) { queueConfigs.forEach((confKey, confValue) -> { switch (confKey) { case MAX_SIZE: diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaAdmin.java b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaAdmin.java index f15b9258e8..d486d04783 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaAdmin.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaAdmin.java @@ -21,6 +21,7 @@ import org.apache.kafka.clients.admin.CreateTopicsResult; import org.apache.kafka.clients.admin.NewTopic; import org.apache.kafka.common.errors.TopicExistsException; import org.thingsboard.server.queue.TbQueueAdmin; +import org.thingsboard.server.queue.util.PropertyUtils; import java.util.Collections; import java.util.Map; @@ -62,12 +63,12 @@ public class TbKafkaAdmin implements TbQueueAdmin { } @Override - public void createTopicIfNotExists(String topic) { + public void createTopicIfNotExists(String topic, String properties) { if (topics.contains(topic)) { return; } try { - NewTopic newTopic = new NewTopic(topic, numPartitions, replicationFactor).configs(topicConfigs); + NewTopic newTopic = new NewTopic(topic, numPartitions, replicationFactor).configs(PropertyUtils.getProps(topicConfigs, properties)); createTopic(newTopic).values().get(topic).get(); topics.add(topic); } catch (ExecutionException ee) { @@ -81,7 +82,6 @@ public class TbKafkaAdmin implements TbQueueAdmin { log.warn("[{}] Failed to create topic", topic, e); throw new RuntimeException(e); } - } @Override diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/InMemoryTbTransportQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/InMemoryTbTransportQueueFactory.java index 60c464ecab..4d0089457d 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/InMemoryTbTransportQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/InMemoryTbTransportQueueFactory.java @@ -73,7 +73,7 @@ public class InMemoryTbTransportQueueFactory implements TbTransportQueueFactory templateBuilder.queueAdmin(new TbQueueAdmin() { @Override - public void createTopicIfNotExists(String topic) {} + public void createTopicIfNotExists(String topic, String properties) {} @Override public void destroy() {} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/pubsub/TbPubSubAdmin.java b/common/queue/src/main/java/org/thingsboard/server/queue/pubsub/TbPubSubAdmin.java index d1a4942ad3..f9f20c2448 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/pubsub/TbPubSubAdmin.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/pubsub/TbPubSubAdmin.java @@ -103,7 +103,7 @@ public class TbPubSubAdmin implements TbQueueAdmin { } @Override - public void createTopicIfNotExists(String partition) { + public void createTopicIfNotExists(String partition, String properties) { TopicName topicName = TopicName.newBuilder() .setTopic(partition) .setProject(pubSubSettings.getProjectId()) diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/rabbitmq/TbRabbitMqAdmin.java b/common/queue/src/main/java/org/thingsboard/server/queue/rabbitmq/TbRabbitMqAdmin.java index 00a2ee4c6c..fb646f383a 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/rabbitmq/TbRabbitMqAdmin.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/rabbitmq/TbRabbitMqAdmin.java @@ -18,9 +18,11 @@ package org.thingsboard.server.queue.rabbitmq; import com.rabbitmq.client.Channel; import com.rabbitmq.client.Connection; import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.StringUtils; import org.thingsboard.server.queue.TbQueueAdmin; import java.io.IOException; +import java.util.HashMap; import java.util.Map; import java.util.concurrent.TimeoutException; @@ -50,7 +52,12 @@ public class TbRabbitMqAdmin implements TbQueueAdmin { } @Override - public void createTopicIfNotExists(String topic) { + public void createTopicIfNotExists(String topic, String properties) { + Map arguments = this.arguments; + if (StringUtils.isNotBlank(properties)) { + arguments = new HashMap<>(arguments); + arguments.putAll(TbRabbitMqQueueArguments.getArgs(properties)); + } try { channel.queueDeclare(topic, false, false, false, arguments); } catch (IOException e) { diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/rabbitmq/TbRabbitMqQueueArguments.java b/common/queue/src/main/java/org/thingsboard/server/queue/rabbitmq/TbRabbitMqQueueArguments.java index cb96abdf3c..8fa8c537e6 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/rabbitmq/TbRabbitMqQueueArguments.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/rabbitmq/TbRabbitMqQueueArguments.java @@ -65,7 +65,7 @@ public class TbRabbitMqQueueArguments { vcArgs = getArgs(vcProperties); } - private Map getArgs(String properties) { + public static Map getArgs(String properties) { Map configs = new HashMap<>(); if (StringUtils.isNotEmpty(properties)) { for (String property : properties.split(";")) { @@ -78,7 +78,7 @@ public class TbRabbitMqQueueArguments { return configs; } - private Object getObjectValue(String str) { + private static Object getObjectValue(String str) { if (str.equalsIgnoreCase("true") || str.equalsIgnoreCase("false")) { return Boolean.valueOf(str); } else if (isNumeric(str)) { @@ -87,7 +87,7 @@ public class TbRabbitMqQueueArguments { return str; } - private Object getNumericValue(String str) { + private static Object getNumericValue(String str) { if (str.contains(".")) { return Double.valueOf(str); } else { @@ -97,7 +97,7 @@ public class TbRabbitMqQueueArguments { private static final Pattern PATTERN = Pattern.compile("-?\\d+(\\.\\d+)?"); - public boolean isNumeric(String strNum) { + private static boolean isNumeric(String strNum) { if (strNum == null) { return false; } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/sqs/TbAwsSqsAdmin.java b/common/queue/src/main/java/org/thingsboard/server/queue/sqs/TbAwsSqsAdmin.java index f88a34941a..ba4eeb6ca4 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/sqs/TbAwsSqsAdmin.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/sqs/TbAwsSqsAdmin.java @@ -26,6 +26,7 @@ import com.amazonaws.services.sqs.model.CreateQueueRequest; import com.amazonaws.services.sqs.model.GetQueueUrlResult; import lombok.extern.slf4j.Slf4j; import org.thingsboard.server.queue.TbQueueAdmin; +import org.thingsboard.server.queue.util.PropertyUtils; import java.util.Map; import java.util.function.Function; @@ -63,11 +64,12 @@ public class TbAwsSqsAdmin implements TbQueueAdmin { } @Override - public void createTopicIfNotExists(String topic) { + public void createTopicIfNotExists(String topic, String properties) { String queueName = convertTopicToQueueName(topic); if (queues.containsKey(queueName)) { return; } + Map attributes = PropertyUtils.getProps(this.attributes, properties, TbAwsSqsQueueAttributes::toConfigs); final CreateQueueRequest createQueueRequest = new CreateQueueRequest(queueName).withAttributes(attributes); String queueUrl = sqsClient.createQueue(createQueueRequest).getQueueUrl(); queues.put(getQueueNameFromUrl(queueUrl), queueUrl); diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/sqs/TbAwsSqsQueueAttributes.java b/common/queue/src/main/java/org/thingsboard/server/queue/sqs/TbAwsSqsQueueAttributes.java index 66110ade74..faa8eccc90 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/sqs/TbAwsSqsQueueAttributes.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/sqs/TbAwsSqsQueueAttributes.java @@ -76,6 +76,12 @@ public class TbAwsSqsQueueAttributes { private Map getConfigs(String properties) { Map configs = new HashMap<>(defaultAttributes); + configs.putAll(toConfigs(properties)); + return configs; + } + + public static Map toConfigs(String properties) { + Map configs = new HashMap<>(); if (StringUtils.isNotEmpty(properties)) { for (String property : properties.split(";")) { int delimiterPosition = property.indexOf(":"); @@ -88,7 +94,7 @@ public class TbAwsSqsQueueAttributes { return configs; } - private void validateAttributeName(String key) { + private static void validateAttributeName(String key) { QueueAttributeName.fromValue(key); } } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/util/PropertyUtils.java b/common/queue/src/main/java/org/thingsboard/server/queue/util/PropertyUtils.java index 089d7f2219..afee64f382 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/util/PropertyUtils.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/util/PropertyUtils.java @@ -19,6 +19,7 @@ import org.thingsboard.server.common.data.StringUtils; import java.util.HashMap; import java.util.Map; +import java.util.function.Function; public class PropertyUtils { @@ -37,4 +38,17 @@ public class PropertyUtils { return configs; } + public static Map getProps(Map defaultProperties, String propertiesStr) { + return getProps(defaultProperties, propertiesStr, PropertyUtils::getProps); + } + + public static Map getProps(Map defaultProperties, String propertiesStr, Function> parser) { + Map properties = defaultProperties; + if (StringUtils.isNotBlank(propertiesStr)) { + properties = new HashMap<>(properties); + properties.putAll(parser.apply(propertiesStr)); + } + return properties; + } + } diff --git a/ui-ngx/src/app/modules/home/components/profile/queue/tenant-profile-queues.component.ts b/ui-ngx/src/app/modules/home/components/profile/queue/tenant-profile-queues.component.ts index ac128b3ec5..298b4fe575 100644 --- a/ui-ngx/src/app/modules/home/components/profile/queue/tenant-profile-queues.component.ts +++ b/ui-ngx/src/app/modules/home/components/profile/queue/tenant-profile-queues.component.ts @@ -173,7 +173,8 @@ export class TenantProfileQueuesComponent implements ControlValueAccessor, Valid }, topic: '', additionalInfo: { - description: '' + description: '', + customProperties: '' } }; this.idMap.push(queue.id); diff --git a/ui-ngx/src/app/modules/home/components/profile/tenant-profile.component.ts b/ui-ngx/src/app/modules/home/components/profile/tenant-profile.component.ts index fa8b1636ff..bed7531495 100644 --- a/ui-ngx/src/app/modules/home/components/profile/tenant-profile.component.ts +++ b/ui-ngx/src/app/modules/home/components/profile/tenant-profile.component.ts @@ -74,7 +74,8 @@ export class TenantProfileComponent extends EntityComponent { }, topic: 'tb_rule_engine.main', additionalInfo: { - description: '' + description: '', + customProperties: '' } }, { @@ -97,7 +98,8 @@ export class TenantProfileComponent extends EntityComponent { maxPauseBetweenRetries: 5 }, additionalInfo: { - description: '' + description: '', + customProperties: '' } }, { @@ -120,7 +122,8 @@ export class TenantProfileComponent extends EntityComponent { maxPauseBetweenRetries: 5 }, additionalInfo: { - description: '' + description: '', + customProperties: '' } } ]; diff --git a/ui-ngx/src/app/modules/home/components/queue/queue-form.component.html b/ui-ngx/src/app/modules/home/components/queue/queue-form.component.html index 56b20d8aa8..4845f7f25a 100644 --- a/ui-ngx/src/app/modules/home/components/queue/queue-form.component.html +++ b/ui-ngx/src/app/modules/home/components/queue/queue-form.component.html @@ -203,6 +203,11 @@ + + queue.custom-properties + + queue.custom-properties-hint + queue.description diff --git a/ui-ngx/src/app/modules/home/components/queue/queue-form.component.ts b/ui-ngx/src/app/modules/home/components/queue/queue-form.component.ts index b123cf2ad5..e4fbcd031b 100644 --- a/ui-ngx/src/app/modules/home/components/queue/queue-form.component.ts +++ b/ui-ngx/src/app/modules/home/components/queue/queue-form.component.ts @@ -117,7 +117,8 @@ export class QueueFormComponent implements ControlValueAccessor, OnInit, OnDestr }), topic: [''], additionalInfo: this.fb.group({ - description: [''] + description: [''], + customProperties: [''] }) }); this.valueChange$ = this.queueFormGroup.valueChanges.subscribe(() => { diff --git a/ui-ngx/src/app/shared/models/queue.models.ts b/ui-ngx/src/app/shared/models/queue.models.ts index 76ee0bf022..07a57e68c2 100644 --- a/ui-ngx/src/app/shared/models/queue.models.ts +++ b/ui-ngx/src/app/shared/models/queue.models.ts @@ -121,5 +121,6 @@ export interface QueueInfo extends BaseData { topic: string; additionalInfo: { description?: string; + customProperties?: string; }; } diff --git a/ui-ngx/src/assets/locale/locale.constant-en_US.json b/ui-ngx/src/assets/locale/locale.constant-en_US.json index c9d236bf2a..c1092f9dad 100644 --- a/ui-ngx/src/assets/locale/locale.constant-en_US.json +++ b/ui-ngx/src/assets/locale/locale.constant-en_US.json @@ -3465,6 +3465,8 @@ "description": "Description", "description-hint": "This text will be displayed in the Queue description instead of the selected strategy", "alt-description": "Submit Strategy: {{submitStrategy}}, Processing Strategy: {{processingStrategy}}", + "custom-properties": "Custom properties", + "custom-properties-hint": "Custom queue (topic) creation properties, e.g. 'retention.ms:604800000;retention.bytes:1048576000'", "strategies": { "sequential-by-originator-label": "Sequential by originator", "sequential-by-originator-hint": "New message for e.g. device A is not submitted until previous message for device A is acknowledged", From c0873590772c76abe149a2cd06c1727ebefe7461 Mon Sep 17 00:00:00 2001 From: ViacheslavKlimov Date: Tue, 1 Aug 2023 13:57:32 +0300 Subject: [PATCH 14/26] Move messages to other queue on deletion; improvements --- .../entitiy/queue/DefaultTbQueueService.java | 30 +------- .../queue/DefaultTbClusterService.java | 10 +-- .../DefaultTbRuleEngineConsumerService.java | 71 ++++++++++++++++--- .../src/main/resources/thingsboard.yml | 2 - .../server/queue/TbQueueConsumer.java | 6 ++ .../AbstractTbQueueConsumerTemplate.java | 18 ++++- .../server/queue/kafka/TbKafkaSettings.java | 1 + .../queue/memory/InMemoryTbQueueConsumer.java | 14 ++++ 8 files changed, 107 insertions(+), 45 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/entitiy/queue/DefaultTbQueueService.java b/application/src/main/java/org/thingsboard/server/service/entitiy/queue/DefaultTbQueueService.java index 9e3d38cf32..72b8c2252a 100644 --- a/application/src/main/java/org/thingsboard/server/service/entitiy/queue/DefaultTbQueueService.java +++ b/application/src/main/java/org/thingsboard/server/service/entitiy/queue/DefaultTbQueueService.java @@ -17,7 +17,6 @@ package org.thingsboard.server.service.entitiy.queue; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; -import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Service; import org.thingsboard.server.cluster.TbClusterService; import org.thingsboard.server.common.data.TenantProfile; @@ -29,7 +28,6 @@ import org.thingsboard.server.common.data.tenant.profile.TenantProfileQueueConfi import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; import org.thingsboard.server.dao.queue.QueueService; import org.thingsboard.server.queue.TbQueueAdmin; -import org.thingsboard.server.queue.scheduler.SchedulerComponent; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.entitiy.AbstractTbEntityService; @@ -37,7 +35,6 @@ import java.util.ArrayList; import java.util.Collections; import java.util.List; import java.util.Map; -import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; @Slf4j @@ -49,10 +46,6 @@ public class DefaultTbQueueService extends AbstractTbEntityService implements Tb private final QueueService queueService; private final TbClusterService tbClusterService; private final TbQueueAdmin tbQueueAdmin; - private final SchedulerComponent scheduler; - - @Value("${queue.rule-engine.topic_deletion_delay:60}") - private int topicDeletionDelay; @Override public Queue saveQueue(Queue queue) { @@ -121,14 +114,7 @@ public class DefaultTbQueueService extends AbstractTbEntityService implements Tb } else { log.info("Removed [{}] partitions from [{}] queue", oldPartitions - currentPartitions, queue.getName()); tbClusterService.onQueueChange(queue); - - scheduler.schedule(() -> { - for (int i = currentPartitions; i < oldPartitions; i++) { - String fullTopicName = new TopicPartitionInfo(queue.getTopic(), queue.getTenantId(), i, false).getFullTopicName(); - log.info("Removed partition [{}]", fullTopicName); - tbQueueAdmin.deleteTopic(fullTopicName); - } - }, topicDeletionDelay, TimeUnit.SECONDS); + // TODO: move all the messages left in old partitions and delete topics } } else if (!oldQueue.equals(queue)) { tbClusterService.onQueueChange(queue); @@ -137,21 +123,7 @@ public class DefaultTbQueueService extends AbstractTbEntityService implements Tb private void onQueueDeleted(Queue queue) { tbClusterService.onQueueDelete(queue); - // queueStatsService.deleteQueueStatsByQueueId(tenantId, queueId); - - scheduler.schedule(() -> { - for (int i = 0; i < queue.getPartitions(); i++) { - String fullTopicName = new TopicPartitionInfo(queue.getTopic(), queue.getTenantId(), i, false).getFullTopicName(); - log.info("Deleting queue [{}]", fullTopicName); - try { - tbQueueAdmin.deleteTopic(fullTopicName); - } catch (Exception e) { - log.error("Failed to delete queue [{}]", fullTopicName); - } - } - }, topicDeletionDelay, TimeUnit.SECONDS); - notificationEntityService.notifySendMsgToEdgeService(queue.getTenantId(), queue.getId(), EdgeEventActionType.DELETED); } diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java index 76631aaa95..5a2db8019e 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java @@ -591,11 +591,6 @@ public class DefaultTbClusterService implements TbClusterService { tbTransportServices.removeAll(tbCoreServices); tbCoreServices.removeAll(tbRuleEngineServices); - for (String ruleEngineServiceId : tbRuleEngineServices) { - TopicPartitionInfo tpi = notificationsTopicService.getNotificationsTopic(ServiceType.TB_RULE_ENGINE, ruleEngineServiceId); - producerProvider.getRuleEngineNotificationsMsgProducer().send(tpi, new TbProtoQueueMsg<>(UUID.randomUUID(), ruleEngineMsg), null); - toRuleEngineNfs.incrementAndGet(); - } for (String coreServiceId : tbCoreServices) { TopicPartitionInfo tpi = notificationsTopicService.getNotificationsTopic(ServiceType.TB_CORE, coreServiceId); producerProvider.getTbCoreNotificationsMsgProducer().send(tpi, new TbProtoQueueMsg<>(UUID.randomUUID(), coreMsg), null); @@ -606,5 +601,10 @@ public class DefaultTbClusterService implements TbClusterService { producerProvider.getTransportNotificationsMsgProducer().send(tpi, new TbProtoQueueMsg<>(UUID.randomUUID(), transportMsg), null); toTransportNfs.incrementAndGet(); } + for (String ruleEngineServiceId : tbRuleEngineServices) { + TopicPartitionInfo tpi = notificationsTopicService.getNotificationsTopic(ServiceType.TB_RULE_ENGINE, ruleEngineServiceId); + producerProvider.getRuleEngineNotificationsMsgProducer().send(tpi, new TbProtoQueueMsg<>(UUID.randomUUID(), ruleEngineMsg), null); + toRuleEngineNfs.incrementAndGet(); + } } } diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbRuleEngineConsumerService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbRuleEngineConsumerService.java index 598b41035f..ffcbc6ff60 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbRuleEngineConsumerService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbRuleEngineConsumerService.java @@ -23,11 +23,14 @@ import org.springframework.scheduling.annotation.Scheduled; import org.springframework.stereotype.Service; import org.thingsboard.common.util.ThingsBoardThreadFactory; import org.thingsboard.server.actors.ActorSystemContext; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.EntityIdFactory; import org.thingsboard.server.common.data.id.QueueId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.queue.Queue; import org.thingsboard.server.common.data.rpc.RpcError; import org.thingsboard.server.common.msg.TbMsg; +import org.thingsboard.server.common.msg.gen.MsgProtos; import org.thingsboard.server.common.msg.queue.QueueToRuleEngineMsg; import org.thingsboard.server.common.msg.queue.RuleEngineException; import org.thingsboard.server.common.msg.queue.RuleNodeInfo; @@ -42,12 +45,14 @@ import org.thingsboard.server.dao.tenant.TbTenantProfileCache; import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineNotificationMsg; +import org.thingsboard.server.queue.TbQueueAdmin; import org.thingsboard.server.queue.TbQueueConsumer; import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.queue.discovery.PartitionService; import org.thingsboard.server.queue.discovery.QueueKey; import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; import org.thingsboard.server.queue.discovery.event.PartitionChangeEvent; +import org.thingsboard.server.queue.provider.TbQueueProducerProvider; import org.thingsboard.server.queue.provider.TbRuleEngineQueueFactory; import org.thingsboard.server.queue.util.DataDecodingEncodingService; import org.thingsboard.server.queue.util.TbRuleEngineComponent; @@ -107,13 +112,14 @@ public class DefaultTbRuleEngineConsumerService extends AbstractConsumerService< private final TbRuleEngineDeviceRpcService tbDeviceRpcService; private final TbServiceInfoProvider serviceInfoProvider; private final QueueService queueService; - // private final TenantId tenantId; + private final TbQueueProducerProvider producerProvider; + private final TbQueueAdmin queueAdmin; private final ConcurrentMap>> consumers = new ConcurrentHashMap<>(); private final ConcurrentMap consumerConfigurations = new ConcurrentHashMap<>(); private final ConcurrentMap consumerStats = new ConcurrentHashMap<>(); private final ConcurrentMap topicsConsumerPerPartition = new ConcurrentHashMap<>(); final ExecutorService submitExecutor = Executors.newSingleThreadExecutor(ThingsBoardThreadFactory.forName("tb-rule-engine-consumer-submit")); - final ScheduledExecutorService repartitionExecutor = Executors.newScheduledThreadPool(1, ThingsBoardThreadFactory.forName("tb-rule-engine-consumer-repartition")); + final ScheduledExecutorService repartitionExecutor = Executors.newScheduledThreadPool(2, ThingsBoardThreadFactory.forName("tb-rule-engine-consumer-repartition")); public DefaultTbRuleEngineConsumerService(TbRuleEngineProcessingStrategyFactory processingStrategyFactory, TbRuleEngineSubmitStrategyFactory submitStrategyFactory, @@ -128,7 +134,8 @@ public class DefaultTbRuleEngineConsumerService extends AbstractConsumerService< TbTenantProfileCache tenantProfileCache, TbApiUsageStateService apiUsageStateService, PartitionService partitionService, ApplicationEventPublisher eventPublisher, - TbServiceInfoProvider serviceInfoProvider, QueueService queueService) { + TbServiceInfoProvider serviceInfoProvider, QueueService queueService, + TbQueueProducerProvider producerProvider, TbQueueAdmin queueAdmin) { super(actorContext, encodingService, tenantProfileCache, deviceProfileCache, assetProfileCache, apiUsageStateService, partitionService, eventPublisher, tbRuleEngineQueueFactory.createToRuleEngineNotificationsMsgConsumer(), Optional.empty()); this.statisticsService = statisticsService; this.tbRuleEngineQueueFactory = tbRuleEngineQueueFactory; @@ -138,6 +145,8 @@ public class DefaultTbRuleEngineConsumerService extends AbstractConsumerService< this.statsFactory = statsFactory; this.serviceInfoProvider = serviceInfoProvider; this.queueService = queueService; + this.producerProvider = producerProvider; + this.queueAdmin = queueAdmin; } @PostConstruct @@ -230,7 +239,6 @@ public class DefaultTbRuleEngineConsumerService extends AbstractConsumerService< launchConsumer(consumer, consumerConfigurations.get(queueKey), consumerStats.get(queueKey), "" + queueKey + "-" + tpi.getPartition().orElse(-999999)); consumer.subscribe(Collections.singleton(tpi)); }); - } finally { tbTopicWithConsumerPerPartition.getLock().unlock(); } @@ -264,7 +272,7 @@ public class DefaultTbRuleEngineConsumerService extends AbstractConsumerService< void consumerLoop(TbQueueConsumer> consumer, org.thingsboard.server.common.data.queue.Queue configuration, TbRuleEngineConsumerStats stats, String threadSuffix) { updateCurrentThreadName(threadSuffix); - while (!stopped && !consumer.isStopped()) { + while (!stopped && !consumer.isStopped() && !consumer.isDeleted()) { try { List> msgs = consumer.poll(configuration.getPollInterval()); if (msgs.isEmpty()) { @@ -314,6 +322,10 @@ public class DefaultTbRuleEngineConsumerService extends AbstractConsumerService< } } } + + if (consumer.isDeleted()) { + processQueueDeletion(configuration, consumer); + } log.info("TB Rule Engine Consumer stopped."); } @@ -448,22 +460,22 @@ public class DefaultTbRuleEngineConsumerService extends AbstractConsumerService< TenantId tenantId = new TenantId(new UUID(queueDeleteMsg.getTenantIdMSB(), queueDeleteMsg.getTenantIdLSB())); QueueKey queueKey = new QueueKey(ServiceType.TB_RULE_ENGINE, queueDeleteMsg.getQueueName(), tenantId); + partitionService.removeQueue(queueDeleteMsg); Queue queue = consumerConfigurations.remove(queueKey); if (queue != null) { if (queue.isConsumerPerPartition()) { TbTopicWithConsumerPerPartition tbTopicWithConsumerPerPartition = topicsConsumerPerPartition.remove(queueKey); if (tbTopicWithConsumerPerPartition != null) { - tbTopicWithConsumerPerPartition.getConsumers().values().forEach(TbQueueConsumer::unsubscribe); + tbTopicWithConsumerPerPartition.getConsumers().values().forEach(TbQueueConsumer::onQueueDelete); tbTopicWithConsumerPerPartition.getConsumers().clear(); } } else { TbQueueConsumer> consumer = consumers.remove(queueKey); if (consumer != null) { - consumer.unsubscribe(); + consumer.onQueueDelete(); } } } - partitionService.removeQueue(queueDeleteMsg); } private void forwardToRuleEngineActor(String queueName, TenantId tenantId, ToRuleEngineMsg toRuleEngineMsg, TbMsgCallback callback) { @@ -482,6 +494,49 @@ public class DefaultTbRuleEngineConsumerService extends AbstractConsumerService< actorContext.tell(msg); } + private void processQueueDeletion(Queue queue, TbQueueConsumer> consumer) { + long startTs = System.currentTimeMillis(); + long timeout = TimeUnit.SECONDS.toMillis(30); + try { + int n = 0; + while ((System.currentTimeMillis() - startTs <= timeout)) { + List> msgs = consumer.poll(queue.getPollInterval()); + if (!msgs.isEmpty()) { + for (TbProtoQueueMsg msg : msgs) { + try { + MsgProtos.TbMsgProto tbMsgProto = MsgProtos.TbMsgProto.parseFrom(msg.getValue().getTbMsg().toByteArray()); + EntityId originator = EntityIdFactory.getByTypeAndUuid(tbMsgProto.getEntityType(), new UUID(tbMsgProto.getEntityIdMSB(), tbMsgProto.getEntityIdLSB())); + + TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_RULE_ENGINE, queue.getName(), TenantId.SYS_TENANT_ID, originator); + producerProvider.getRuleEngineMsgProducer().send(tpi, msg, null); + n++; + } catch (Throwable e) { + log.debug("Failed to move message to system {}: {}", consumer.getTopic(), msg, e); + } + } + consumer.commit(); + } else { + break; + } + } + if (n > 0) { + log.info("Moved {} messages from {} to system {}", n, consumer.getFullTopicNames(), consumer.getTopic()); + } + + consumer.unsubscribe(); + for (String topic : consumer.getFullTopicNames()) { + try { + queueAdmin.deleteTopic(topic); + log.info("Deleted topic {}", topic); + } catch (Exception e) { + log.error("Failed to delete topic {} after unsubscribing", topic, e); + } + } + } catch (Exception e) { + log.error("Failed to process deletion of {} ({})", consumer.getTopic(), queue.getTenantId(), e); + } + } + @Scheduled(fixedDelayString = "${queue.rule-engine.stats.print-interval-ms}") public void printStats() { if (statsEnabled) { diff --git a/application/src/main/resources/thingsboard.yml b/application/src/main/resources/thingsboard.yml index 96ee2793ed..faf1ebcbea 100644 --- a/application/src/main/resources/thingsboard.yml +++ b/application/src/main/resources/thingsboard.yml @@ -1233,8 +1233,6 @@ queue: failure-percentage: "${TB_QUEUE_RE_SQ_PROCESSING_STRATEGY_FAILURE_PERCENTAGE:0}" # Skip retry if failures or timeouts are less then X percentage of messages; pause-between-retries: "${TB_QUEUE_RE_SQ_PROCESSING_STRATEGY_RETRY_PAUSE:5}" # Time in seconds to wait in consumer thread before retries; max-pause-between-retries: "${TB_QUEUE_RE_SQ_PROCESSING_STRATEGY_MAX_RETRY_PAUSE:5}" # Max allowed time in seconds for pause between retries. - # Delay between Queue update/delete and actual topic deletion. The delay is for Rule Engines to have time to unsubscribe from the topics, and for other services to stop publishing - topic_deletion_delay: "${TB_QUEUE_RULE_ENGINE_TOPIC_DELETION_DELAY_SECS:180}" transport: # For high priority notifications that require minimum latency and processing time notifications_topic: "${TB_QUEUE_TRANSPORT_NOTIFICATIONS_TOPIC:tb_transport.notifications}" diff --git a/common/cluster-api/src/main/java/org/thingsboard/server/queue/TbQueueConsumer.java b/common/cluster-api/src/main/java/org/thingsboard/server/queue/TbQueueConsumer.java index 04439fc85d..73bea7642f 100644 --- a/common/cluster-api/src/main/java/org/thingsboard/server/queue/TbQueueConsumer.java +++ b/common/cluster-api/src/main/java/org/thingsboard/server/queue/TbQueueConsumer.java @@ -36,4 +36,10 @@ public interface TbQueueConsumer { boolean isStopped(); + void onQueueDelete(); + + boolean isDeleted(); + + List getFullTopicNames(); + } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/common/AbstractTbQueueConsumerTemplate.java b/common/queue/src/main/java/org/thingsboard/server/queue/common/AbstractTbQueueConsumerTemplate.java index 5b0e84c2ed..8e91c1d134 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/common/AbstractTbQueueConsumerTemplate.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/common/AbstractTbQueueConsumerTemplate.java @@ -44,6 +44,7 @@ public abstract class AbstractTbQueueConsumerTemplate i protected volatile Set partitions; protected final ReentrantLock consumerLock = new ReentrantLock(); //NonfairSync final Queue> subscribeQueue = new ConcurrentLinkedQueue<>(); + protected volatile boolean deleted = false; @Getter private final String topic; @@ -94,7 +95,7 @@ public abstract class AbstractTbQueueConsumerTemplate i partitions = subscribeQueue.poll(); } if (!subscribed) { - List topicNames = partitions.stream().map(TopicPartitionInfo::getFullTopicName).collect(Collectors.toList()); + List topicNames = getFullTopicNames(); doSubscribe(topicNames); subscribed = true; } @@ -191,6 +192,21 @@ public abstract class AbstractTbQueueConsumerTemplate i abstract protected void doUnsubscribe(); + @Override + public void onQueueDelete() { + deleted = true; + } + + @Override + public boolean isDeleted() { + return deleted; + } + + @Override + public List getFullTopicNames() { + return partitions.stream().map(TopicPartitionInfo::getFullTopicName).collect(Collectors.toList()); + } + protected boolean isLongPollingSupported() { return false; } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaSettings.java b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaSettings.java index d2a54128e3..55f1721c46 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaSettings.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaSettings.java @@ -159,6 +159,7 @@ public class TbKafkaSettings { props.put(ConsumerConfig.FETCH_MAX_BYTES_CONFIG, fetchMaxBytes); props.put(ConsumerConfig.MAX_POLL_INTERVAL_MS_CONFIG, maxPollIntervalMs); props.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, autoOffsetReset); + props.put(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG, false); props.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class); props.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, ByteArrayDeserializer.class); diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/memory/InMemoryTbQueueConsumer.java b/common/queue/src/main/java/org/thingsboard/server/queue/memory/InMemoryTbQueueConsumer.java index 081202315e..a642d7e9cf 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/memory/InMemoryTbQueueConsumer.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/memory/InMemoryTbQueueConsumer.java @@ -103,4 +103,18 @@ public class InMemoryTbQueueConsumer implements TbQueueCon return stopped; } + @Override + public void onQueueDelete() { + } + + @Override + public boolean isDeleted() { + return false; + } + + @Override + public List getFullTopicNames() { + return partitions.stream().map(TopicPartitionInfo::getFullTopicName).collect(Collectors.toList()); + } + } From 0f13d4614473c056d703b7443c50bea404472c70 Mon Sep 17 00:00:00 2001 From: ViacheslavKlimov Date: Thu, 3 Aug 2023 13:54:59 +0300 Subject: [PATCH 15/26] Add more tests --- .../DefaultTbRuleEngineConsumerService.java | 36 +++---- .../src/main/resources/thingsboard.yml | 2 + .../controller/BaseTenantControllerTest.java | 100 +++++++++++++++--- .../queue/memory/InMemoryTbQueueConsumer.java | 4 +- 4 files changed, 110 insertions(+), 32 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbRuleEngineConsumerService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbRuleEngineConsumerService.java index ffcbc6ff60..863f0354c2 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbRuleEngineConsumerService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbRuleEngineConsumerService.java @@ -103,6 +103,8 @@ public class DefaultTbRuleEngineConsumerService extends AbstractConsumerService< private boolean statsEnabled; @Value("${queue.rule-engine.prometheus-stats.enabled:false}") boolean prometheusStatsEnabled; + @Value("${queue.rule-engine.topic-deletion-delay:30}") + private int topicDeletionDelay; private final StatsFactory statsFactory; private final TbRuleEngineSubmitStrategyFactory submitStrategyFactory; @@ -495,29 +497,27 @@ public class DefaultTbRuleEngineConsumerService extends AbstractConsumerService< } private void processQueueDeletion(Queue queue, TbQueueConsumer> consumer) { - long startTs = System.currentTimeMillis(); - long timeout = TimeUnit.SECONDS.toMillis(30); + long finishTs = System.currentTimeMillis() + TimeUnit.SECONDS.toMillis(topicDeletionDelay); try { int n = 0; - while ((System.currentTimeMillis() - startTs <= timeout)) { + while (System.currentTimeMillis() <= finishTs) { List> msgs = consumer.poll(queue.getPollInterval()); - if (!msgs.isEmpty()) { - for (TbProtoQueueMsg msg : msgs) { - try { - MsgProtos.TbMsgProto tbMsgProto = MsgProtos.TbMsgProto.parseFrom(msg.getValue().getTbMsg().toByteArray()); - EntityId originator = EntityIdFactory.getByTypeAndUuid(tbMsgProto.getEntityType(), new UUID(tbMsgProto.getEntityIdMSB(), tbMsgProto.getEntityIdLSB())); - - TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_RULE_ENGINE, queue.getName(), TenantId.SYS_TENANT_ID, originator); - producerProvider.getRuleEngineMsgProducer().send(tpi, msg, null); - n++; - } catch (Throwable e) { - log.debug("Failed to move message to system {}: {}", consumer.getTopic(), msg, e); - } + if (msgs.isEmpty()) { + continue; + } + for (TbProtoQueueMsg msg : msgs) { + try { + MsgProtos.TbMsgProto tbMsgProto = MsgProtos.TbMsgProto.parseFrom(msg.getValue().getTbMsg().toByteArray()); + EntityId originator = EntityIdFactory.getByTypeAndUuid(tbMsgProto.getEntityType(), new UUID(tbMsgProto.getEntityIdMSB(), tbMsgProto.getEntityIdLSB())); + + TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_RULE_ENGINE, queue.getName(), TenantId.SYS_TENANT_ID, originator); + producerProvider.getRuleEngineMsgProducer().send(tpi, msg, null); + n++; + } catch (Throwable e) { + log.debug("Failed to move message to system {}: {}", consumer.getTopic(), msg, e); } - consumer.commit(); - } else { - break; } + consumer.commit(); } if (n > 0) { log.info("Moved {} messages from {} to system {}", n, consumer.getFullTopicNames(), consumer.getTopic()); diff --git a/application/src/main/resources/thingsboard.yml b/application/src/main/resources/thingsboard.yml index faf1ebcbea..7754159992 100644 --- a/application/src/main/resources/thingsboard.yml +++ b/application/src/main/resources/thingsboard.yml @@ -1233,6 +1233,8 @@ queue: failure-percentage: "${TB_QUEUE_RE_SQ_PROCESSING_STRATEGY_FAILURE_PERCENTAGE:0}" # Skip retry if failures or timeouts are less then X percentage of messages; pause-between-retries: "${TB_QUEUE_RE_SQ_PROCESSING_STRATEGY_RETRY_PAUSE:5}" # Time in seconds to wait in consumer thread before retries; max-pause-between-retries: "${TB_QUEUE_RE_SQ_PROCESSING_STRATEGY_MAX_RETRY_PAUSE:5}" # Max allowed time in seconds for pause between retries. + # After a queue is deleted (or profile's isolation option was disabled), Rule Engine will continue reading related topics during this period, before deleting the actual topics + topic-deletion-delay: "${TB_QUEUE_RULE_ENGINE_TOPIC_DELETION_DELAY_SEC:30}" transport: # For high priority notifications that require minimum latency and processing time notifications_topic: "${TB_QUEUE_TRANSPORT_NOTIFICATIONS_TOPIC:tb_transport.notifications}" diff --git a/application/src/test/java/org/thingsboard/server/controller/BaseTenantControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/BaseTenantControllerTest.java index 85e3fa35bd..db5d804a54 100644 --- a/application/src/test/java/org/thingsboard/server/controller/BaseTenantControllerTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/BaseTenantControllerTest.java @@ -55,18 +55,24 @@ import org.thingsboard.server.common.data.tenant.profile.DefaultTenantProfileCon import org.thingsboard.server.common.data.tenant.profile.TenantProfileData; import org.thingsboard.server.common.data.tenant.profile.TenantProfileQueueConfiguration; import org.thingsboard.server.common.msg.TbMsg; +import org.thingsboard.server.common.msg.TbMsgMetaData; import org.thingsboard.server.common.msg.queue.QueueToRuleEngineMsg; import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; -import org.thingsboard.server.dao.service.DaoSqlTest; +import org.thingsboard.server.gen.transport.TransportProtos; +import org.thingsboard.server.queue.TbQueueAdmin; import org.thingsboard.server.queue.discovery.PartitionService; import java.util.ArrayList; +import java.util.Collections; import java.util.Comparator; +import java.util.Deque; import java.util.HashMap; +import java.util.LinkedList; import java.util.List; import java.util.Map; import java.util.Random; +import java.util.UUID; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicReference; import java.util.function.Consumer; @@ -77,14 +83,18 @@ import static org.assertj.core.api.Assertions.assertThat; import static org.awaitility.Awaitility.await; import static org.hamcrest.Matchers.containsString; import static org.mockito.ArgumentMatchers.argThat; +import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.never; import static org.mockito.Mockito.timeout; import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; +import static org.thingsboard.server.common.data.DataConstants.MAIN_QUEUE_NAME; +import static org.thingsboard.server.common.data.DataConstants.MAIN_QUEUE_TOPIC; @TestPropertySource(properties = { "js.evaluator=mock", + "queue.rule-engine.topic-deletion-delay=10" }) @Slf4j public abstract class BaseTenantControllerTest extends AbstractControllerTest { @@ -100,6 +110,8 @@ public abstract class BaseTenantControllerTest extends AbstractControllerTest { private PartitionService partitionService; @SpyBean private ActorSystemContext actorContext; + @SpyBean + private TbQueueAdmin queueAdmin; @Before public void setUp() throws Exception { @@ -110,7 +122,7 @@ public abstract class BaseTenantControllerTest extends AbstractControllerTest { public void tearDown() throws Exception { loginSysAdmin(); for (Queue queue : doGetTypedWithPageLink("/api/queues?serviceType=TB_RULE_ENGINE&", new TypeReference>() {}, new PageLink(100)).getData()) { - if (!queue.getName().equals(DataConstants.MAIN_QUEUE_NAME)) { + if (!queue.getName().equals(MAIN_QUEUE_NAME)) { doDelete("/api/queues/" + queue.getId()).andExpect(status().isOk()); } } @@ -457,7 +469,7 @@ public abstract class BaseTenantControllerTest extends AbstractControllerTest { tenantProfileData.setConfiguration(new DefaultTenantProfileConfiguration()); tenantProfile.setProfileData(tenantProfileData); tenantProfile.setIsolatedTbRuleEngine(true); - addQueueConfig(tenantProfile, DataConstants.MAIN_QUEUE_NAME); + addQueueConfig(tenantProfile, MAIN_QUEUE_NAME); addQueueConfig(tenantProfile, "Test"); tenantProfile = doPost("/api/tenantProfile", tenantProfile, TenantProfile.class); @@ -486,7 +498,7 @@ public abstract class BaseTenantControllerTest extends AbstractControllerTest { tenantProfileData2.setConfiguration(new DefaultTenantProfileConfiguration()); tenantProfile2.setProfileData(tenantProfileData2); tenantProfile2.setIsolatedTbRuleEngine(true); - addQueueConfig(tenantProfile2, DataConstants.MAIN_QUEUE_NAME); + addQueueConfig(tenantProfile2, MAIN_QUEUE_NAME); addQueueConfig(tenantProfile2, "Test"); addQueueConfig(tenantProfile2, "Test2"); tenantProfile2 = doPost("/api/tenantProfile", tenantProfile2, TenantProfile.class); @@ -571,7 +583,7 @@ public abstract class BaseTenantControllerTest extends AbstractControllerTest { Device hpQueueDevice = createDevice("HP", hpQueueProfile.getName(), "HP"); DeviceProfile mainQueueProfile = createDeviceProfile("Main profile"); - mainQueueProfile.setDefaultQueueName(DataConstants.MAIN_QUEUE_NAME); + mainQueueProfile.setDefaultQueueName(MAIN_QUEUE_NAME); mainQueueProfile = doPost("/api/deviceProfile", mainQueueProfile, DeviceProfile.class); Device mainQueueDevice = createDevice("Main", mainQueueProfile.getName(), "Main"); @@ -581,25 +593,25 @@ public abstract class BaseTenantControllerTest extends AbstractControllerTest { assertThat(usedTpi.getTopic()).isEqualTo(DataConstants.HP_QUEUE_TOPIC); assertThat(usedTpi.getTenantId()).get().isEqualTo(TenantId.SYS_TENANT_ID); }); - verifyUsedQueueAndMessage(DataConstants.MAIN_QUEUE_NAME, tenantId, mainQueueDevice.getId(), DataConstants.ATTRIBUTES_UPDATED, () -> { + verifyUsedQueueAndMessage(MAIN_QUEUE_NAME, tenantId, mainQueueDevice.getId(), DataConstants.ATTRIBUTES_UPDATED, () -> { doPost("/api/plugins/telemetry/DEVICE/" + mainQueueDevice.getId() + "/attributes/SERVER_SCOPE", "{\"test\":123}", String.class); }, usedTpi -> { - assertThat(usedTpi.getTopic()).isEqualTo(DataConstants.MAIN_QUEUE_TOPIC); + assertThat(usedTpi.getTopic()).isEqualTo(MAIN_QUEUE_TOPIC); assertThat(usedTpi.getTenantId()).get().isEqualTo(TenantId.SYS_TENANT_ID); }); loginSysAdmin(); tenantProfile.setIsolatedTbRuleEngine(true); tenantProfile.getProfileData().setQueueConfiguration(List.of( - getQueueConfig(DataConstants.MAIN_QUEUE_NAME, DataConstants.MAIN_QUEUE_TOPIC) + getQueueConfig(MAIN_QUEUE_NAME, MAIN_QUEUE_TOPIC) )); tenantProfile = doPost("/api/tenantProfile", tenantProfile, TenantProfile.class); loginDifferentTenant(); - verifyUsedQueueAndMessage(DataConstants.MAIN_QUEUE_NAME, tenantId, mainQueueDevice.getId(), DataConstants.ATTRIBUTES_UPDATED, () -> { + verifyUsedQueueAndMessage(MAIN_QUEUE_NAME, tenantId, mainQueueDevice.getId(), DataConstants.ATTRIBUTES_UPDATED, () -> { doPost("/api/plugins/telemetry/DEVICE/" + mainQueueDevice.getId() + "/attributes/SERVER_SCOPE", "{\"test\":123}", String.class); }, usedTpi -> { - assertThat(usedTpi.getTopic()).isEqualTo(DataConstants.MAIN_QUEUE_TOPIC); + assertThat(usedTpi.getTopic()).isEqualTo(MAIN_QUEUE_TOPIC); assertThat(usedTpi.getTenantId()).get().isEqualTo(tenantId); }); verifyUsedQueueAndMessage(DataConstants.HP_QUEUE_NAME, tenantId, hpQueueDevice.getId(), DataConstants.ATTRIBUTES_UPDATED, () -> { @@ -612,7 +624,7 @@ public abstract class BaseTenantControllerTest extends AbstractControllerTest { loginSysAdmin(); tenantProfile.setIsolatedTbRuleEngine(true); tenantProfile.getProfileData().setQueueConfiguration(List.of( - getQueueConfig(DataConstants.MAIN_QUEUE_NAME, DataConstants.MAIN_QUEUE_TOPIC), + getQueueConfig(MAIN_QUEUE_NAME, MAIN_QUEUE_TOPIC), getQueueConfig(DataConstants.HP_QUEUE_NAME, DataConstants.HP_QUEUE_TOPIC) )); tenantProfile = doPost("/api/tenantProfile", tenantProfile, TenantProfile.class); @@ -624,14 +636,76 @@ public abstract class BaseTenantControllerTest extends AbstractControllerTest { assertThat(usedTpi.getTopic()).isEqualTo(DataConstants.HP_QUEUE_TOPIC); assertThat(usedTpi.getTenantId()).get().isEqualTo(tenantId); }); - verifyUsedQueueAndMessage(DataConstants.MAIN_QUEUE_NAME, tenantId, mainQueueDevice.getId(), DataConstants.ATTRIBUTES_UPDATED, () -> { + verifyUsedQueueAndMessage(MAIN_QUEUE_NAME, tenantId, mainQueueDevice.getId(), DataConstants.ATTRIBUTES_UPDATED, () -> { doPost("/api/plugins/telemetry/DEVICE/" + mainQueueDevice.getId() + "/attributes/SERVER_SCOPE", "{\"test\":123}", String.class); }, usedTpi -> { - assertThat(usedTpi.getTopic()).isEqualTo(DataConstants.MAIN_QUEUE_TOPIC); + assertThat(usedTpi.getTopic()).isEqualTo(MAIN_QUEUE_TOPIC); assertThat(usedTpi.getTenantId()).get().isEqualTo(tenantId); }); } + @Test + public void testIsolatedQueueDeletion() throws Exception { + loginSysAdmin(); + TenantProfile tenantProfile = new TenantProfile(); + tenantProfile.setName("Test profile"); + TenantProfileData tenantProfileData = new TenantProfileData(); + tenantProfileData.setConfiguration(new DefaultTenantProfileConfiguration()); + tenantProfile.setProfileData(tenantProfileData); + tenantProfile.setIsolatedTbRuleEngine(true); + addQueueConfig(tenantProfile, MAIN_QUEUE_NAME); + tenantProfile = doPost("/api/tenantProfile", tenantProfile, TenantProfile.class); + createDifferentTenant(); + loginSysAdmin(); + savedDifferentTenant.setTenantProfileId(tenantProfile.getId()); + savedDifferentTenant = doPost("/api/tenant", savedDifferentTenant, Tenant.class); + TenantId tenantId = differentTenantId; + await().atMost(10, TimeUnit.SECONDS) + .until(() -> { + TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_RULE_ENGINE, MAIN_QUEUE_NAME, tenantId, tenantId); + return !tpi.getTenantId().get().isSysTenantId(); + }); + TopicPartitionInfo tpi = new TopicPartitionInfo(MAIN_QUEUE_TOPIC, tenantId, 0, false); + String isolatedTopic = tpi.getFullTopicName(); + TbMsg expectedMsg = publishTbMsg(tenantId, tpi); + awaitTbMsg(tbMsg -> tbMsg.getId().equals(expectedMsg.getId()), 10000); // to wait for consumer start + + loginSysAdmin(); + tenantProfile.setIsolatedTbRuleEngine(false); + tenantProfile.getProfileData().setQueueConfiguration(Collections.emptyList()); + tenantProfile = doPost("/api/tenantProfile", tenantProfile, TenantProfile.class); + await().atMost(10, TimeUnit.SECONDS) + .until(() -> partitionService.resolve(ServiceType.TB_RULE_ENGINE, MAIN_QUEUE_NAME, tenantId, tenantId) + .getTenantId().get().isSysTenantId()); + + Deque submittedMsgs = new LinkedList<>(); + await().atLeast(8, TimeUnit.SECONDS) // due to topic-deletion-delay + .atMost(20, TimeUnit.SECONDS) + .pollInterval(1, TimeUnit.SECONDS) + .untilAsserted(() -> { + TbMsg tbMsg = publishTbMsg(tenantId, tpi); + submittedMsgs.add(tbMsg.getId()); + + verify(queueAdmin, times(1)).deleteTopic(eq(isolatedTopic)); + }); + submittedMsgs.removeLast(); + for (UUID msgId : submittedMsgs) { + verify(actorContext, timeout(2000)).tell(argThat(msg -> { + return msg instanceof QueueToRuleEngineMsg && ((QueueToRuleEngineMsg) msg).getMsg().getId().equals(msgId); + })); + } + } + + private TbMsg publishTbMsg(TenantId tenantId, TopicPartitionInfo tpi) { + TbMsg tbMsg = TbMsg.newMsg("POST_TELEMETRY_REQUEST", tenantId, TbMsgMetaData.EMPTY, "{\"test\":1}"); + TransportProtos.ToRuleEngineMsg msg = TransportProtos.ToRuleEngineMsg.newBuilder() + .setTenantIdMSB(tenantId.getId().getMostSignificantBits()) + .setTenantIdLSB(tenantId.getId().getLeastSignificantBits()) + .setTbMsg(TbMsg.toByteString(tbMsg)).build(); + tbClusterService.pushMsgToRuleEngine(tpi, tbMsg.getId(), msg, null); + return tbMsg; + } + private void verifyUsedQueueAndMessage(String queue, TenantId tenantId, EntityId entityId, String msgType, Runnable action, Consumer tpiAssert) { await().atMost(15, TimeUnit.SECONDS) .untilAsserted(() -> { diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/memory/InMemoryTbQueueConsumer.java b/common/queue/src/main/java/org/thingsboard/server/queue/memory/InMemoryTbQueueConsumer.java index a642d7e9cf..dba6f6d588 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/memory/InMemoryTbQueueConsumer.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/memory/InMemoryTbQueueConsumer.java @@ -31,6 +31,7 @@ public class InMemoryTbQueueConsumer implements TbQueueCon private volatile Set partitions; private volatile boolean stopped; private volatile boolean subscribed; + private volatile boolean deleted; public InMemoryTbQueueConsumer(InMemoryStorage storage, String topic) { this.storage = storage; @@ -105,11 +106,12 @@ public class InMemoryTbQueueConsumer implements TbQueueCon @Override public void onQueueDelete() { + deleted = true; } @Override public boolean isDeleted() { - return false; + return deleted; } @Override From 12f8e14c0496a3affd02eaf2e07656d11f7348d9 Mon Sep 17 00:00:00 2001 From: ViacheslavKlimov Date: Thu, 3 Aug 2023 16:34:45 +0300 Subject: [PATCH 16/26] Increase default poll interval for isolated queues --- .../home/components/profile/tenant-profile.component.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/ui-ngx/src/app/modules/home/components/profile/tenant-profile.component.ts b/ui-ngx/src/app/modules/home/components/profile/tenant-profile.component.ts index bed7531495..e9142fa1fa 100644 --- a/ui-ngx/src/app/modules/home/components/profile/tenant-profile.component.ts +++ b/ui-ngx/src/app/modules/home/components/profile/tenant-profile.component.ts @@ -60,7 +60,7 @@ export class TenantProfileComponent extends EntityComponent { name: 'Main', packProcessingTimeout: 2000, partitions: 2, - pollInterval: 25, + pollInterval: 2000, processingStrategy: { failurePercentage: 0, maxPauseBetweenRetries: 3, @@ -82,7 +82,7 @@ export class TenantProfileComponent extends EntityComponent { id: guid(), name: 'HighPriority', topic: 'tb_rule_engine.hp', - pollInterval: 25, + pollInterval: 2000, partitions: 2, consumerPerPartition: true, packProcessingTimeout: 2000, @@ -106,7 +106,7 @@ export class TenantProfileComponent extends EntityComponent { id: guid(), name: 'SequentialByOriginator', topic: 'tb_rule_engine.sq', - pollInterval: 25, + pollInterval: 2000, partitions: 2, consumerPerPartition: true, packProcessingTimeout: 2000, From f32e2f6fdefb96cef3b8dc13008e76326b4faf00 Mon Sep 17 00:00:00 2001 From: ViacheslavKlimov Date: Thu, 10 Aug 2023 12:03:15 +0300 Subject: [PATCH 17/26] Refactoring after review --- .../service/entitiy/queue/DefaultTbQueueService.java | 4 ++++ .../queue/DefaultTbRuleEngineConsumerService.java | 6 +++--- .../org/thingsboard/server/queue/TbQueueConsumer.java | 2 +- .../queue/common/AbstractTbQueueConsumerTemplate.java | 9 ++++----- .../server/queue/memory/InMemoryTbQueueConsumer.java | 8 ++++---- 5 files changed, 16 insertions(+), 13 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/entitiy/queue/DefaultTbQueueService.java b/application/src/main/java/org/thingsboard/server/service/entitiy/queue/DefaultTbQueueService.java index 72b8c2252a..9aa9a421c8 100644 --- a/application/src/main/java/org/thingsboard/server/service/entitiy/queue/DefaultTbQueueService.java +++ b/application/src/main/java/org/thingsboard/server/service/entitiy/queue/DefaultTbQueueService.java @@ -176,6 +176,10 @@ public class DefaultTbQueueService extends AbstractTbEntityService implements Tb } } + if (log.isDebugEnabled()) { + log.debug("[{}] Handling profile queue config update: creating queues {}, updating {}, deleting {}. Affected tenants: {}", + newTenantProfile.getUuidId(), toCreate, toUpdate, toRemove, tenantIds); + } tenantIds.forEach(tenantId -> { toCreate.forEach(key -> saveQueue(new Queue(tenantId, newQueues.get(key)))); diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbRuleEngineConsumerService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbRuleEngineConsumerService.java index 863f0354c2..c8bf469a61 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbRuleEngineConsumerService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbRuleEngineConsumerService.java @@ -121,7 +121,7 @@ public class DefaultTbRuleEngineConsumerService extends AbstractConsumerService< private final ConcurrentMap consumerStats = new ConcurrentHashMap<>(); private final ConcurrentMap topicsConsumerPerPartition = new ConcurrentHashMap<>(); final ExecutorService submitExecutor = Executors.newSingleThreadExecutor(ThingsBoardThreadFactory.forName("tb-rule-engine-consumer-submit")); - final ScheduledExecutorService repartitionExecutor = Executors.newScheduledThreadPool(2, ThingsBoardThreadFactory.forName("tb-rule-engine-consumer-repartition")); + final ScheduledExecutorService repartitionExecutor = Executors.newScheduledThreadPool(1, ThingsBoardThreadFactory.forName("tb-rule-engine-consumer-repartition")); public DefaultTbRuleEngineConsumerService(TbRuleEngineProcessingStrategyFactory processingStrategyFactory, TbRuleEngineSubmitStrategyFactory submitStrategyFactory, @@ -274,7 +274,7 @@ public class DefaultTbRuleEngineConsumerService extends AbstractConsumerService< void consumerLoop(TbQueueConsumer> consumer, org.thingsboard.server.common.data.queue.Queue configuration, TbRuleEngineConsumerStats stats, String threadSuffix) { updateCurrentThreadName(threadSuffix); - while (!stopped && !consumer.isStopped() && !consumer.isDeleted()) { + while (!stopped && !consumer.isStopped() && !consumer.isQueueDeleted()) { try { List> msgs = consumer.poll(configuration.getPollInterval()); if (msgs.isEmpty()) { @@ -325,7 +325,7 @@ public class DefaultTbRuleEngineConsumerService extends AbstractConsumerService< } } - if (consumer.isDeleted()) { + if (consumer.isQueueDeleted()) { processQueueDeletion(configuration, consumer); } log.info("TB Rule Engine Consumer stopped."); diff --git a/common/cluster-api/src/main/java/org/thingsboard/server/queue/TbQueueConsumer.java b/common/cluster-api/src/main/java/org/thingsboard/server/queue/TbQueueConsumer.java index 73bea7642f..9c41f9d342 100644 --- a/common/cluster-api/src/main/java/org/thingsboard/server/queue/TbQueueConsumer.java +++ b/common/cluster-api/src/main/java/org/thingsboard/server/queue/TbQueueConsumer.java @@ -38,7 +38,7 @@ public interface TbQueueConsumer { void onQueueDelete(); - boolean isDeleted(); + boolean isQueueDeleted(); List getFullTopicNames(); diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/common/AbstractTbQueueConsumerTemplate.java b/common/queue/src/main/java/org/thingsboard/server/queue/common/AbstractTbQueueConsumerTemplate.java index 8e91c1d134..2ebe41850d 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/common/AbstractTbQueueConsumerTemplate.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/common/AbstractTbQueueConsumerTemplate.java @@ -44,7 +44,7 @@ public abstract class AbstractTbQueueConsumerTemplate i protected volatile Set partitions; protected final ReentrantLock consumerLock = new ReentrantLock(); //NonfairSync final Queue> subscribeQueue = new ConcurrentLinkedQueue<>(); - protected volatile boolean deleted = false; + protected volatile boolean queueDeleted = false; @Getter private final String topic; @@ -194,12 +194,11 @@ public abstract class AbstractTbQueueConsumerTemplate i @Override public void onQueueDelete() { - deleted = true; + queueDeleted = true; } - @Override - public boolean isDeleted() { - return deleted; + public boolean isQueueDeleted() { + return queueDeleted; } @Override diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/memory/InMemoryTbQueueConsumer.java b/common/queue/src/main/java/org/thingsboard/server/queue/memory/InMemoryTbQueueConsumer.java index dba6f6d588..8711cbbcf1 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/memory/InMemoryTbQueueConsumer.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/memory/InMemoryTbQueueConsumer.java @@ -31,7 +31,7 @@ public class InMemoryTbQueueConsumer implements TbQueueCon private volatile Set partitions; private volatile boolean stopped; private volatile boolean subscribed; - private volatile boolean deleted; + private volatile boolean queueDeleted; public InMemoryTbQueueConsumer(InMemoryStorage storage, String topic) { this.storage = storage; @@ -106,12 +106,12 @@ public class InMemoryTbQueueConsumer implements TbQueueCon @Override public void onQueueDelete() { - deleted = true; + queueDeleted = true; } @Override - public boolean isDeleted() { - return deleted; + public boolean isQueueDeleted() { + return queueDeleted; } @Override From 6751820e0a22d1fb3f07844bb758886e273816f0 Mon Sep 17 00:00:00 2001 From: ViacheslavKlimov Date: Mon, 14 Aug 2023 12:57:53 +0300 Subject: [PATCH 18/26] Dedicated Rule Engines for tenant profile --- .../server/actors/app/AppActor.java | 5 + .../DefaultTbRuleEngineConsumerService.java | 2 +- .../DefaultTenantRoutingInfoService.java | 8 +- .../src/main/resources/thingsboard.yml | 4 + .../discovery/HashPartitionServiceTest.java | 127 ++++++++++++++++-- common/cluster-api/src/main/proto/queue.proto | 1 + .../DefaultTbServiceInfoProvider.java | 11 +- .../queue/discovery/HashPartitionService.java | 84 +++++++++--- .../discovery/TbServiceInfoProvider.java | 5 + .../queue/discovery/TenantRoutingInfo.java | 2 + .../discovery/TenantRoutingInfoService.java | 1 + .../TransportTenantRoutingInfoService.java | 4 +- ...ersionControlTenantRoutingInfoService.java | 2 +- .../assets/locale/locale.constant-en_US.json | 2 +- 14 files changed, 223 insertions(+), 35 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/actors/app/AppActor.java b/application/src/main/java/org/thingsboard/server/actors/app/AppActor.java index fb6fbbdff2..e8a56fab16 100644 --- a/application/src/main/java/org/thingsboard/server/actors/app/AppActor.java +++ b/application/src/main/java/org/thingsboard/server/actors/app/AppActor.java @@ -47,6 +47,7 @@ import org.thingsboard.server.service.transport.msg.TransportToDeviceActorMsgWra import java.util.HashSet; import java.util.Set; +import java.util.UUID; @Slf4j public class AppActor extends ContextAwareActor { @@ -123,7 +124,11 @@ public class AppActor extends ContextAwareActor { try { if (systemContext.isTenantComponentsInitEnabled()) { PageDataIterable tenantIterator = new PageDataIterable<>(tenantService::findTenants, ENTITY_PACK_LIMIT); + Set assignedProfiles = systemContext.getServiceInfoProvider().getAssignedTenantProfiles(); for (Tenant tenant : tenantIterator) { + if (!assignedProfiles.isEmpty() && !assignedProfiles.contains(tenant.getTenantProfileId().getId())) { + continue; + } log.debug("[{}] Creating tenant actor", tenant.getId()); getOrCreateTenantActor(tenant.getId()); log.debug("[{}] Tenant actor created.", tenant.getId()); diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbRuleEngineConsumerService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbRuleEngineConsumerService.java index d560cfee1c..a18b5c099c 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbRuleEngineConsumerService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbRuleEngineConsumerService.java @@ -156,7 +156,7 @@ public class DefaultTbRuleEngineConsumerService extends AbstractConsumerService< super.init("tb-rule-engine-consumer", "tb-rule-engine-notifications-consumer"); List queues = queueService.findAllQueues(); for (Queue configuration : queues) { - initConsumer(configuration); + initConsumer(configuration); // TODO: if this Rule Engine is assigned specific profile, don't init other consumers and properly handle queue update events } } diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTenantRoutingInfoService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTenantRoutingInfoService.java index 400586235e..ea90364ef7 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTenantRoutingInfoService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTenantRoutingInfoService.java @@ -22,7 +22,6 @@ import org.thingsboard.server.common.data.TenantProfile; import org.thingsboard.server.common.data.exception.TenantNotFoundException; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.dao.tenant.TbTenantProfileCache; -import org.thingsboard.server.dao.tenant.TenantService; import org.thingsboard.server.queue.discovery.TenantRoutingInfo; import org.thingsboard.server.queue.discovery.TenantRoutingInfoService; @@ -31,12 +30,9 @@ import org.thingsboard.server.queue.discovery.TenantRoutingInfoService; @ConditionalOnExpression("'${service.type:null}'=='monolith' || '${service.type:null}'=='tb-core' || '${service.type:null}'=='tb-rule-engine'") public class DefaultTenantRoutingInfoService implements TenantRoutingInfoService { - private final TenantService tenantService; - private final TbTenantProfileCache tenantProfileCache; - public DefaultTenantRoutingInfoService(TenantService tenantService, TbTenantProfileCache tenantProfileCache) { - this.tenantService = tenantService; + public DefaultTenantRoutingInfoService(TbTenantProfileCache tenantProfileCache) { this.tenantProfileCache = tenantProfileCache; } @@ -44,7 +40,7 @@ public class DefaultTenantRoutingInfoService implements TenantRoutingInfoService public TenantRoutingInfo getRoutingInfo(TenantId tenantId) { TenantProfile tenantProfile = tenantProfileCache.get(tenantId); if (tenantProfile != null) { - return new TenantRoutingInfo(tenantId, tenantProfile.isIsolatedTbRuleEngine()); + return new TenantRoutingInfo(tenantId, tenantProfile.getId(), tenantProfile.isIsolatedTbRuleEngine()); } else { throw new TenantNotFoundException(tenantId); } diff --git a/application/src/main/resources/thingsboard.yml b/application/src/main/resources/thingsboard.yml index 8576f20783..2ce5c3ba21 100644 --- a/application/src/main/resources/thingsboard.yml +++ b/application/src/main/resources/thingsboard.yml @@ -1253,6 +1253,10 @@ service: type: "${TB_SERVICE_TYPE:monolith}" # monolith or tb-core or tb-rule-engine # Unique id for this service (autogenerated if empty) id: "${TB_SERVICE_ID:}" + rule_engine: + # Comma-separated list of tenant profiles ids assigned to this Rule Engine. + # This Rule Engine will only be responsible for tenants with these profiles (in case 'isolation' option is enabled in profile). + assigned_tenant_profiles: "${TB_RULE_ENGINE_ASSIGNED_TENANT_PROFILES:}" metrics: # Enable/disable actuator metrics. diff --git a/application/src/test/java/org/thingsboard/server/queue/discovery/HashPartitionServiceTest.java b/application/src/test/java/org/thingsboard/server/queue/discovery/HashPartitionServiceTest.java index 25b06e7840..84024ecd1e 100644 --- a/application/src/test/java/org/thingsboard/server/queue/discovery/HashPartitionServiceTest.java +++ b/application/src/test/java/org/thingsboard/server/queue/discovery/HashPartitionServiceTest.java @@ -18,6 +18,7 @@ package org.thingsboard.server.queue.discovery; import com.datastax.oss.driver.api.core.uuid.Uuids; import lombok.SneakyThrows; import lombok.extern.slf4j.Slf4j; +import org.apache.commons.collections4.ListUtils; import org.junit.Assert; import org.junit.Before; import org.junit.Test; @@ -25,24 +26,35 @@ import org.junit.runner.RunWith; import org.mockito.junit.MockitoJUnitRunner; import org.springframework.context.ApplicationEventPublisher; import org.springframework.test.util.ReflectionTestUtils; +import org.thingsboard.server.common.data.DataConstants; import org.thingsboard.server.common.data.id.DeviceId; +import org.thingsboard.server.common.data.id.QueueId; import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.id.TenantProfileId; +import org.thingsboard.server.common.data.id.UUIDBased; +import org.thingsboard.server.common.data.queue.Queue; import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; -import org.thingsboard.server.gen.transport.TransportProtos; +import org.thingsboard.server.gen.transport.TransportProtos.ServiceInfo; import java.text.SimpleDateFormat; import java.util.ArrayList; +import java.util.Collection; import java.util.Collections; import java.util.Comparator; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Random; +import java.util.UUID; import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; +import java.util.stream.Stream; +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; @Slf4j @RunWith(MockitoJUnitRunner.class) @@ -57,7 +69,7 @@ public class HashPartitionServiceTest { private ApplicationEventPublisher applicationEventPublisher; private QueueRoutingInfoService queueRoutingInfoService; - private String hashFunctionName = "sha256"; + private String hashFunctionName = "murmur3_128"; @Before public void setup() throws Exception { @@ -74,15 +86,15 @@ public class HashPartitionServiceTest { ReflectionTestUtils.setField(clusterRoutingService, "vcTopic", "tb.vc"); ReflectionTestUtils.setField(clusterRoutingService, "vcPartitions", 10); ReflectionTestUtils.setField(clusterRoutingService, "hashFunctionName", hashFunctionName); - TransportProtos.ServiceInfo currentServer = TransportProtos.ServiceInfo.newBuilder() + ServiceInfo currentServer = ServiceInfo.newBuilder() .setServiceId("tb-core-0") .addAllServiceTypes(Collections.singletonList(ServiceType.TB_CORE.name())) .build(); // when(queueService.resolve(Mockito.any(), Mockito.anyString())).thenAnswer(i -> i.getArguments()[1]); // when(discoveryService.getServiceInfo()).thenReturn(currentServer); - List otherServers = new ArrayList<>(); + List otherServers = new ArrayList<>(); for (int i = 1; i < SERVER_COUNT; i++) { - otherServers.add(TransportProtos.ServiceInfo.newBuilder() + otherServers.add(ServiceInfo.newBuilder() .setServiceId("tb-rule-" + i) .addAllServiceTypes(Collections.singletonList(ServiceType.TB_CORE.name())) .build()); @@ -122,10 +134,10 @@ public class HashPartitionServiceTest { int queueCount = 3; int partitionCount = 3; - List services = new ArrayList<>(); + List services = new ArrayList<>(); for (int i = 0; i < serverCount; i++) { - services.add(TransportProtos.ServiceInfo.newBuilder().setServiceId("RE-" + i).build()); + services.add(ServiceInfo.newBuilder().setServiceId("RE-" + i).build()); } long start = System.currentTimeMillis(); @@ -140,7 +152,7 @@ public class HashPartitionServiceTest { for (int queueIndex = 0; queueIndex < queueCount; queueIndex++) { QueueKey queueKey = new QueueKey(ServiceType.TB_RULE_ENGINE, "queue" + queueIndex, tenantId); for (int partition = 0; partition < partitionCount; partition++) { - TransportProtos.ServiceInfo serviceInfo = clusterRoutingService.resolveByPartitionIdx(services, queueKey, partition); + ServiceInfo serviceInfo = clusterRoutingService.resolveByPartitionIdx(services, queueKey, partition); String serviceId = serviceInfo.getServiceId(); map.put(serviceId, map.get(serviceId) + 1); } @@ -163,4 +175,103 @@ public class HashPartitionServiceTest { Assert.assertTrue(diffPercent < maxDiffPercent); } + @Test + public void testPartitionsAssignmentWithDedicatedServers() { + int isolatedProfilesCount = 5; + int tenantsCountPerProfile = 100; + int dedicatedServerSetsCount = 3; + int serversCountPerSet = 3; + int profilesPerSet = (int) Math.ceil((double) isolatedProfilesCount / dedicatedServerSetsCount); + + List isolatedTenantProfiles = Stream.generate(() -> new TenantProfileId(UUID.randomUUID())) + .limit(isolatedProfilesCount).collect(Collectors.toList()); + Map tenants = new HashMap<>(); + for (TenantProfileId tenantProfileId : isolatedTenantProfiles) { + for (int i = 0; i < tenantsCountPerProfile; i++) { + tenants.put(new TenantId(UUID.randomUUID()), tenantProfileId); + } + } + + List queues = new ArrayList<>(); + Queue systemQueue = new Queue(); + systemQueue.setTenantId(TenantId.SYS_TENANT_ID); + systemQueue.setName("Main"); + systemQueue.setTopic(DataConstants.MAIN_QUEUE_TOPIC); + systemQueue.setPartitions(10); + systemQueue.setId(new QueueId(UUID.randomUUID())); + queues.add(systemQueue); + tenants.forEach((tenantId, profileId) -> { + Queue isolatedQueue = new Queue(); + isolatedQueue.setTenantId(tenantId); + isolatedQueue.setName("Main"); + isolatedQueue.setTopic(DataConstants.MAIN_QUEUE_TOPIC); + isolatedQueue.setPartitions(2); + isolatedQueue.setId(new QueueId(UUID.randomUUID())); + queues.add(isolatedQueue); + when(routingInfoService.getRoutingInfo(eq(tenantId))).thenReturn(new TenantRoutingInfo(tenantId, profileId, true)); + }); + when(queueRoutingInfoService.getAllQueuesRoutingInfo()).thenReturn(queues.stream() + .map(QueueRoutingInfo::new).collect(Collectors.toList())); + + List ruleEngines = new ArrayList<>(); + Map> dedicatedServers = new HashMap<>(); + int serviceId = 0; + for (int i = 0; i < serversCountPerSet; i++) { + ServiceInfo commonServer = ServiceInfo.newBuilder() + .setServiceId("tb-rule-engine-" + serviceId) + .addAllServiceTypes(List.of(ServiceType.TB_RULE_ENGINE.name())) + .build(); + ruleEngines.add(commonServer); + serviceId++; + } + for (int i = 0; i < dedicatedServerSetsCount; i++) { + List assignedProfiles = ListUtils.partition(isolatedTenantProfiles, profilesPerSet).get(i); + for (int j = 0; j < serversCountPerSet; j++) { + ServiceInfo dedicatedServer = ServiceInfo.newBuilder() + .setServiceId("tb-rule-engine-" + serviceId) + .addAllServiceTypes(List.of(ServiceType.TB_RULE_ENGINE.name())) + .addAllAssignedTenantProfiles(assignedProfiles.stream().map(UUIDBased::toString).collect(Collectors.toList())) + .build(); + ruleEngines.add(dedicatedServer); + serviceId++; + + for (TenantProfileId assignedProfileId : assignedProfiles) { + dedicatedServers.computeIfAbsent(assignedProfileId, p -> new ArrayList<>()).add(dedicatedServer); + } + } + } + + Map>> serversPartitions = new HashMap<>(); + clusterRoutingService.init(); + for (ServiceInfo ruleEngine : ruleEngines) { + List other = new ArrayList<>(ruleEngines); + other.removeIf(serviceInfo -> serviceInfo.getServiceId().equals(ruleEngine.getServiceId())); + + clusterRoutingService.recalculatePartitions(ruleEngine, other); + clusterRoutingService.myPartitions.forEach((queueKey, partitions) -> { + serversPartitions.computeIfAbsent(queueKey, k -> new HashMap<>()).put(ruleEngine, partitions); + }); + } + assertThat(serversPartitions.keySet()).containsAll(queues.stream().map(queue -> new QueueKey(ServiceType.TB_RULE_ENGINE, queue)).collect(Collectors.toList())); + + serversPartitions.forEach((queueKey, partitionsPerServer) -> { + if (queueKey.getTenantId().isSysTenantId()) { + partitionsPerServer.forEach((server, partitions) -> { + assertThat(server.getAssignedTenantProfilesCount()).as("system queues are not assigned to dedicated servers").isZero(); + }); + } else { + List responsibleServers = dedicatedServers.get(tenants.get(queueKey.getTenantId())); + partitionsPerServer.forEach((server, partitions) -> { + assertThat(server.getAssignedTenantProfilesCount()).as("isolated queues are only assigned to dedicated servers").isPositive(); + assertThat(responsibleServers).contains(server); + }); + } + + List allPartitions = partitionsPerServer.values().stream() + .flatMap(Collection::stream) + .collect(Collectors.toList()); + assertThat(allPartitions).doesNotHaveDuplicates(); + }); + } + } diff --git a/common/cluster-api/src/main/proto/queue.proto b/common/cluster-api/src/main/proto/queue.proto index 80f44e59be..78614911d4 100644 --- a/common/cluster-api/src/main/proto/queue.proto +++ b/common/cluster-api/src/main/proto/queue.proto @@ -28,6 +28,7 @@ message ServiceInfo { repeated string serviceTypes = 2; repeated string transports = 6; SystemInfoProto systemInfo = 10; + repeated string assignedTenantProfiles = 11; } message SystemInfoProto { diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/DefaultTbServiceInfoProvider.java b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/DefaultTbServiceInfoProvider.java index 3c85aef350..e9013ef345 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/DefaultTbServiceInfoProvider.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/DefaultTbServiceInfoProvider.java @@ -23,6 +23,7 @@ import org.springframework.context.ApplicationContext; import org.springframework.stereotype.Component; import org.thingsboard.server.common.data.StringUtils; import org.thingsboard.server.common.data.TbTransportService; +import org.thingsboard.server.common.data.util.CollectionsUtil; import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.gen.transport.TransportProtos.ServiceInfo; @@ -35,6 +36,8 @@ import java.util.Arrays; import java.util.Collection; import java.util.Collections; import java.util.List; +import java.util.Set; +import java.util.UUID; import java.util.stream.Collectors; import static org.thingsboard.common.util.SystemUtil.getCpuCount; @@ -57,6 +60,10 @@ public class DefaultTbServiceInfoProvider implements TbServiceInfoProvider { @Value("${service.type:monolith}") private String serviceType; + @Getter + @Value("${service.rule_engine.assigned_tenant_profiles:}") + private Set assignedTenantProfiles; + @Autowired private ApplicationContext applicationContext; @@ -111,7 +118,9 @@ public class DefaultTbServiceInfoProvider implements TbServiceInfoProvider { .setServiceId(serviceId) .addAllServiceTypes(serviceTypes.stream().map(ServiceType::name).collect(Collectors.toList())) .setSystemInfo(getCurrentSystemInfoProto()); - + if (CollectionsUtil.isNotEmpty(assignedTenantProfiles)) { + builder.addAllAssignedTenantProfiles(assignedTenantProfiles.stream().map(UUID::toString).collect(Collectors.toList())); + } return serviceInfo = builder.build(); } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/HashPartitionService.java b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/HashPartitionService.java index a8954b2b33..4c1894eae1 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/HashPartitionService.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/HashPartitionService.java @@ -24,6 +24,7 @@ import org.springframework.stereotype.Service; import org.thingsboard.server.common.data.exception.TenantNotFoundException; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.id.TenantProfileId; import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; import org.thingsboard.server.gen.transport.TransportProtos; @@ -36,6 +37,7 @@ import org.thingsboard.server.queue.util.AfterStartUp; import javax.annotation.PostConstruct; import java.nio.charset.StandardCharsets; import java.util.ArrayList; +import java.util.Collection; import java.util.Collections; import java.util.Comparator; import java.util.HashMap; @@ -70,15 +72,16 @@ public class HashPartitionService implements PartitionService { private final TenantRoutingInfoService tenantRoutingInfoService; private final QueueRoutingInfoService queueRoutingInfoService; - private volatile ConcurrentMap> myPartitions = new ConcurrentHashMap<>(); + protected volatile ConcurrentMap> myPartitions = new ConcurrentHashMap<>(); private final ConcurrentMap partitionTopicsMap = new ConcurrentHashMap<>(); private final ConcurrentMap partitionSizesMap = new ConcurrentHashMap<>(); private final ConcurrentMap tenantRoutingInfoMap = new ConcurrentHashMap<>(); - private Map> tbTransportServicesByType = new HashMap<>(); private List currentOtherServices; + private final Map> tbTransportServicesByType = new HashMap<>(); + private final Map> responsibleServices = new HashMap<>(); private HashFunction hashFunction; @@ -215,14 +218,12 @@ public class HashPartitionService implements PartitionService { } private TopicPartitionInfo resolve(QueueKey queueKey, EntityId entityId) { - int hash = hashFunction.newHasher() - .putLong(entityId.getId().getMostSignificantBits()) - .putLong(entityId.getId().getLeastSignificantBits()).hash().asInt(); - Integer partitionSize = partitionSizesMap.get(queueKey); if (partitionSize == null) { throw new IllegalStateException("Partitions info for queue " + queueKey + " is missing"); } + + int hash = hash(entityId.getId()); int partition = Math.abs(hash % partitionSize); return buildTopicPartitionInfo(queueKey, partition); @@ -231,6 +232,7 @@ public class HashPartitionService implements PartitionService { @Override public synchronized void recalculatePartitions(ServiceInfo currentService, List otherServices) { tbTransportServicesByType.clear(); + responsibleServices.clear(); logServiceInfo(currentService); otherServices.forEach(this::logServiceInfo); @@ -240,6 +242,7 @@ public class HashPartitionService implements PartitionService { addNode(queueServicesMap, other); } queueServicesMap.values().forEach(list -> list.sort(Comparator.comparing(ServiceInfo::getServiceId))); + responsibleServices.values().forEach(list -> list.sort(Comparator.comparing(ServiceInfo::getServiceId))); final ConcurrentMap> newPartitions = new ConcurrentHashMap<>(); partitionSizesMap.forEach((queueKey, size) -> { @@ -287,6 +290,9 @@ public class HashPartitionService implements PartitionService { changes.addAll(newMap.keySet()); if (!changes.isEmpty()) { applicationEventPublisher.publishEvent(new ClusterTopologyChangeEvent(this, changes)); + responsibleServices.forEach((profileId, serviceInfos) -> { + log.info("Servers responsible for tenant profile {}: {}", profileId, toServiceIds(serviceInfos)); + }); } } @@ -324,9 +330,7 @@ public class HashPartitionService implements PartitionService { @Override public int resolvePartitionIndex(UUID entityId, int partitions) { - int hash = hashFunction.newHasher() - .putLong(entityId.getMostSignificantBits()) - .putLong(entityId.getLeastSignificantBits()).hash().asInt(); + int hash = hash(entityId); return Math.abs(hash % partitions); } @@ -408,6 +412,19 @@ public class HashPartitionService implements PartitionService { queueServiceList.computeIfAbsent(key, k -> new ArrayList<>()).add(instance); } }); + + if (instance.getAssignedTenantProfilesCount() > 0) { + for (String profileIdStr : instance.getAssignedTenantProfilesList()) { + TenantProfileId profileId; + try { + profileId = new TenantProfileId(UUID.fromString(profileIdStr)); + } catch (IllegalArgumentException e) { + log.warn("Failed to parse '{}' as tenant profile id", profileIdStr); + continue; + } + responsibleServices.computeIfAbsent(profileId, k -> new ArrayList<>()).add(instance); + } + } } else if (ServiceType.TB_CORE.equals(serviceType) || ServiceType.TB_VC_EXECUTOR.equals(serviceType)) { queueServiceList.computeIfAbsent(new QueueKey(serviceType), key -> new ArrayList<>()).add(instance); } @@ -423,18 +440,51 @@ public class HashPartitionService implements PartitionService { return null; } - if (!ServiceType.TB_RULE_ENGINE.equals(queueKey.getType()) || TenantId.SYS_TENANT_ID.equals(queueKey.getTenantId())) { - return servers.get(partition % servers.size()); - } else { - int hash = hashFunction.newHasher().putLong(queueKey.getTenantId().getId().getMostSignificantBits()) - .putLong(queueKey.getTenantId().getId().getLeastSignificantBits()) + TenantId tenantId = queueKey.getTenantId(); + if (queueKey.getType() == ServiceType.TB_RULE_ENGINE) { + if (!responsibleServices.isEmpty()) { // if there are any dedicated servers + TenantProfileId profileId; + if (tenantId != null && !tenantId.isSysTenantId()) { + TenantRoutingInfo routingInfo = tenantRoutingInfoService.getRoutingInfo(tenantId); + profileId = routingInfo.getProfileId(); + } else { + profileId = null; + } + + List responsible = responsibleServices.get(profileId); + if (responsible == null) { + // if there are no dedicated servers for this tenant profile, or for system queues, + // using the servers that are not responsible for any profile + responsible = servers.stream() + .filter(serviceInfo -> serviceInfo.getAssignedTenantProfilesCount() == 0) + .sorted(Comparator.comparing(ServiceInfo::getServiceId)) + .collect(Collectors.toList()); + if (profileId != null) { + log.debug("Using servers {} for profile {}", toServiceIds(responsible), profileId); + } + responsibleServices.put(profileId, responsible); + } + servers = responsible; + } + + int hash = hashFunction.newHasher() + .putLong(tenantId.getId().getMostSignificantBits()) + .putLong(tenantId.getId().getLeastSignificantBits()) .putString(queueKey.getQueueName(), StandardCharsets.UTF_8) .hash().asInt(); - return servers.get(Math.abs((hash + partition) % servers.size())); + } else { + return servers.get(partition % servers.size()); } } + private int hash(UUID key) { + return hashFunction.newHasher() + .putLong(key.getMostSignificantBits()) + .putLong(key.getLeastSignificantBits()) + .hash().asInt(); + } + public static HashFunction forName(String name) { switch (name) { case "murmur3_32": @@ -448,4 +498,8 @@ public class HashPartitionService implements PartitionService { } } + private List toServiceIds(Collection serviceInfos) { + return serviceInfos.stream().map(ServiceInfo::getServiceId).collect(Collectors.toList()); + } + } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/TbServiceInfoProvider.java b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/TbServiceInfoProvider.java index e49cbbcfd9..9c7d1630ec 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/TbServiceInfoProvider.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/TbServiceInfoProvider.java @@ -18,6 +18,9 @@ package org.thingsboard.server.queue.discovery; import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.gen.transport.TransportProtos.ServiceInfo; +import java.util.Set; +import java.util.UUID; + public interface TbServiceInfoProvider { String getServiceId(); @@ -30,4 +33,6 @@ public interface TbServiceInfoProvider { ServiceInfo generateNewServiceInfoWithCurrentSystemInfo(); + Set getAssignedTenantProfiles(); + } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/TenantRoutingInfo.java b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/TenantRoutingInfo.java index c1c0b49dab..8dee68da49 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/TenantRoutingInfo.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/TenantRoutingInfo.java @@ -17,9 +17,11 @@ package org.thingsboard.server.queue.discovery; import lombok.Data; import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.id.TenantProfileId; @Data public class TenantRoutingInfo { private final TenantId tenantId; + private final TenantProfileId profileId; private final boolean isolatedTbRuleEngine; } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/TenantRoutingInfoService.java b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/TenantRoutingInfoService.java index 8dd3ff95e7..e4c0ac8250 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/TenantRoutingInfoService.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/TenantRoutingInfoService.java @@ -20,4 +20,5 @@ import org.thingsboard.server.common.data.id.TenantId; public interface TenantRoutingInfoService { TenantRoutingInfo getRoutingInfo(TenantId tenantId); + } diff --git a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/TransportTenantRoutingInfoService.java b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/TransportTenantRoutingInfoService.java index c9f126b808..e1192391d5 100644 --- a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/TransportTenantRoutingInfoService.java +++ b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/TransportTenantRoutingInfoService.java @@ -29,7 +29,7 @@ import org.thingsboard.server.queue.discovery.TenantRoutingInfoService; @ConditionalOnExpression("'${service.type:null}'=='tb-transport'") public class TransportTenantRoutingInfoService implements TenantRoutingInfoService { - private TransportTenantProfileCache tenantProfileCache; + private final TransportTenantProfileCache tenantProfileCache; public TransportTenantRoutingInfoService(TransportTenantProfileCache tenantProfileCache) { this.tenantProfileCache = tenantProfileCache; @@ -38,7 +38,7 @@ public class TransportTenantRoutingInfoService implements TenantRoutingInfoServi @Override public TenantRoutingInfo getRoutingInfo(TenantId tenantId) { TenantProfile profile = tenantProfileCache.get(tenantId); - return new TenantRoutingInfo(tenantId, profile.isIsolatedTbRuleEngine()); + return new TenantRoutingInfo(tenantId, profile.getId(), profile.isIsolatedTbRuleEngine()); } } diff --git a/msa/vc-executor/src/main/java/org/thingsboard/server/vc/service/VersionControlTenantRoutingInfoService.java b/msa/vc-executor/src/main/java/org/thingsboard/server/vc/service/VersionControlTenantRoutingInfoService.java index fb33ff9931..b343a4791f 100644 --- a/msa/vc-executor/src/main/java/org/thingsboard/server/vc/service/VersionControlTenantRoutingInfoService.java +++ b/msa/vc-executor/src/main/java/org/thingsboard/server/vc/service/VersionControlTenantRoutingInfoService.java @@ -25,6 +25,6 @@ public class VersionControlTenantRoutingInfoService implements TenantRoutingInfo @Override public TenantRoutingInfo getRoutingInfo(TenantId tenantId) { //This dummy implementation is ok since Version Control service does not produce any rule engine messages. - return new TenantRoutingInfo(tenantId, false); + return new TenantRoutingInfo(tenantId, null, false); } } diff --git a/ui-ngx/src/assets/locale/locale.constant-en_US.json b/ui-ngx/src/assets/locale/locale.constant-en_US.json index c1092f9dad..b449cf1ba5 100644 --- a/ui-ngx/src/assets/locale/locale.constant-en_US.json +++ b/ui-ngx/src/assets/locale/locale.constant-en_US.json @@ -3535,7 +3535,7 @@ "search": "Search tenants", "selected-tenants": "{ count, plural, =1 {1 tenant} other {# tenants} } selected", "isolated-tb-rule-engine": "Processing in isolated ThingsBoard Rule Engine container", - "isolated-tb-rule-engine-details": "Requires separate microservice(s) per isolated Tenant" + "isolated-tb-rule-engine-details": "Requires separate microservice(s) for the profile" }, "tenant-profile": { "tenant-profile": "Tenant profile", From 97ee45be24de93e38bd71c054d6904eb6a6efddf Mon Sep 17 00:00:00 2001 From: Sergey Matvienko Date: Tue, 15 Aug 2023 14:15:21 +0200 Subject: [PATCH 19/26] TbMathNode: refactored for easier testing. Semaphores - WEAK reference type. calculateResult method - removed unused args. --- .../rule/engine/math/TbMathNode.java | 21 ++++++++++++------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/math/TbMathNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/math/TbMathNode.java index eff3917c14..f48e723494 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/math/TbMathNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/math/TbMathNode.java @@ -79,7 +79,7 @@ import java.util.stream.Collectors; ) public class TbMathNode implements TbNode { - private static final ConcurrentMap semaphores = new ConcurrentReferenceHashMap<>(); + private static final ConcurrentMap semaphores = new ConcurrentReferenceHashMap<>(16, ConcurrentReferenceHashMap.ReferenceType.WEAK); private final ThreadLocal customExpression = new ThreadLocal<>(); private TbMathNodeConfiguration config; @@ -116,12 +116,7 @@ public class TbMathNode implements TbNode { } try { - var arguments = config.getArguments(); - Optional msgBodyOpt = convertMsgBodyIfRequired(msg); - var argumentValues = Futures.allAsList(arguments.stream() - .map(arg -> resolveArguments(ctx, msg, msgBodyOpt, arg)).collect(Collectors.toList())); - ListenableFuture resultMsgFuture = Futures.transformAsync(argumentValues, args -> - updateMsgAndDb(ctx, msg, msgBodyOpt, calculateResult(ctx, msg, args)), ctx.getDbCallbackExecutor()); + ListenableFuture resultMsgFuture = processMsgAsync(ctx, msg); DonAsynchron.withCallback(resultMsgFuture, resultMsg -> { try { ctx.tellSuccess(resultMsg); @@ -142,6 +137,16 @@ public class TbMathNode implements TbNode { } } + ListenableFuture processMsgAsync(TbContext ctx, TbMsg msg) { + var arguments = config.getArguments(); + Optional msgBodyOpt = convertMsgBodyIfRequired(msg); + var argumentValues = Futures.allAsList(arguments.stream() + .map(arg -> resolveArguments(ctx, msg, msgBodyOpt, arg)).collect(Collectors.toList())); + ListenableFuture resultMsgFuture = Futures.transformAsync(argumentValues, args -> + updateMsgAndDb(ctx, msg, msgBodyOpt, calculateResult(args)), ctx.getDbCallbackExecutor()); + return resultMsgFuture; + } + private boolean tryAcquire(EntityId originator, Semaphore originatorSemaphore) { boolean acquired; try { @@ -248,7 +253,7 @@ public class TbMathNode implements TbNode { return TbMsg.transformMsg(msg, md); } - private double calculateResult(TbContext ctx, TbMsg msg, List args) { + private double calculateResult(List args) { switch (config.getOperation()) { case ADD: return apply(args.get(0), args.get(1), Double::sum); From 16fdfc518d2a4cee00784fdf1633ff31ae1da992 Mon Sep 17 00:00:00 2001 From: Sergey Matvienko Date: Tue, 15 Aug 2023 14:43:56 +0200 Subject: [PATCH 20/26] TbMathNode: test added for concurrent calls by the same originator utilizing the whole rule-dispatcher pool. 1 failed. non-blocking implementation wanted; Additional refactoring: JUnit5 and mock init --- .../rule/engine/math/TbMathNodeTest.java | 124 ++++++++++++++---- 1 file changed, 98 insertions(+), 26 deletions(-) diff --git a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/math/TbMathNodeTest.java b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/math/TbMathNodeTest.java index 69d1b46dbe..4c36d51a7d 100644 --- a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/math/TbMathNodeTest.java +++ b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/math/TbMathNodeTest.java @@ -15,17 +15,18 @@ */ package org.thingsboard.rule.engine.math; -import com.datastax.oss.driver.api.core.uuid.Uuids; import com.google.common.util.concurrent.Futures; -import org.junit.After; +import lombok.extern.slf4j.Slf4j; import org.junit.Assert; -import org.junit.Before; -import org.junit.Test; -import org.junit.runner.RunWith; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.ArgumentCaptor; import org.mockito.Mock; import org.mockito.Mockito; -import org.mockito.junit.MockitoJUnitRunner; +import org.mockito.junit.jupiter.MockitoExtension; +import org.mockito.verification.Timeout; import org.thingsboard.common.util.AbstractListeningExecutor; import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.rule.engine.api.RuleEngineTelemetryService; @@ -47,21 +48,34 @@ import org.thingsboard.server.dao.attributes.AttributesService; import org.thingsboard.server.dao.timeseries.TimeseriesService; import java.util.Arrays; +import java.util.List; import java.util.Optional; +import java.util.UUID; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import static org.assertj.core.api.Assertions.assertThat; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyDouble; import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.ArgumentMatchers.argThat; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.BDDMockito.willAnswer; import static org.mockito.Mockito.lenient; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.spy; import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; -@RunWith(MockitoJUnitRunner.class) +@ExtendWith(MockitoExtension.class) +@Slf4j public class TbMathNodeTest { - private EntityId originator = new DeviceId(Uuids.timeBased()); - private TenantId tenantId = TenantId.fromUUID(Uuids.timeBased()); + static final int RULE_DISPATCHER_POOL_SIZE = 2; + static final int DB_CALLBACK_POOL_SIZE = 3; + private final EntityId originator = DeviceId.fromString("ccd71696-0586-422d-940e-755a41ec3b0d"); + private final TenantId tenantId = TenantId.fromUUID(UUID.fromString("e7f46b23-0c7d-42f5-9b06-fc35ab17af8a")); @Mock private TbContext ctx; @@ -71,35 +85,41 @@ public class TbMathNodeTest { private TimeseriesService tsService; @Mock private RuleEngineTelemetryService telemetryService; - private AbstractListeningExecutor dbExecutor; + private AbstractListeningExecutor dbCallbackExecutor; + private AbstractListeningExecutor ruleEngineDispatcherExecutor; - @Before + @BeforeEach public void before() { - dbExecutor = new AbstractListeningExecutor() { + dbCallbackExecutor = new AbstractListeningExecutor() { @Override protected int getThreadPollSize() { - return 3; + return DB_CALLBACK_POOL_SIZE; } }; - dbExecutor.init(); - initMocks(); + dbCallbackExecutor.init(); + ruleEngineDispatcherExecutor = new AbstractListeningExecutor() { + @Override + protected int getThreadPollSize() { + return RULE_DISPATCHER_POOL_SIZE; + } + }; + ruleEngineDispatcherExecutor.init(); + + lenient().when(ctx.getAttributesService()).thenReturn(attributesService); + lenient().when(ctx.getTelemetryService()).thenReturn(telemetryService); + lenient().when(ctx.getTimeseriesService()).thenReturn(tsService); + lenient().when(ctx.getTenantId()).thenReturn(tenantId); + lenient().when(ctx.getDbCallbackExecutor()).thenReturn(dbCallbackExecutor); } - @After + @AfterEach public void after() { - dbExecutor.destroy(); + ruleEngineDispatcherExecutor.executor().shutdownNow(); + dbCallbackExecutor.executor().shutdownNow(); } private void initMocks() { - Mockito.reset(ctx); - Mockito.reset(attributesService); - Mockito.reset(tsService); - Mockito.reset(telemetryService); - lenient().when(ctx.getAttributesService()).thenReturn(attributesService); - lenient().when(ctx.getTelemetryService()).thenReturn(telemetryService); - lenient().when(ctx.getTimeseriesService()).thenReturn(tsService); - lenient().when(ctx.getTenantId()).thenReturn(tenantId); - lenient().when(ctx.getDbCallbackExecutor()).thenReturn(dbExecutor); + Mockito.clearInvocations(ctx, attributesService, tsService, telemetryService); } private TbMathNode initNode(TbRuleNodeMathFunctionType operation, TbMathResult result, TbMathArgument... arguments) { @@ -496,4 +516,56 @@ public class TbMathNodeTest { }); Assert.assertNotNull(thrown.getMessage()); } + + @Test + public void testExp4j_concurrent() { + TbMathNode node = spy(initNodeWithCustomFunction("2a+3b", + new TbMathResult(TbMathArgumentType.MESSAGE_BODY, "result", 2, false, false, null), + new TbMathArgument(TbMathArgumentType.MESSAGE_BODY, "a"), + new TbMathArgument(TbMathArgumentType.MESSAGE_BODY, "b") + )); + EntityId originatorSlow = DeviceId.fromString("7f01170d-6bba-419c-b95c-2b4c3ba32f30"); + EntityId originatorFast = DeviceId.fromString("c45360ff-7906-4102-a2ae-3495a86168d0"); + CountDownLatch slowProcessingLatch = new CountDownLatch(1); + + List slowMsgList = List.of( + TbMsg.newMsg("TEST", originatorSlow, new TbMsgMetaData(), JacksonUtil.newObjectNode().put("a", 2).put("b", 2).toString()), + TbMsg.newMsg("TEST", originatorSlow, new TbMsgMetaData(), JacksonUtil.newObjectNode().put("a", 2).put("b", 2).toString()) + ); + List fastMsgList = List.of( + TbMsg.newMsg("TEST", originatorFast, new TbMsgMetaData(), JacksonUtil.newObjectNode().put("a", 2).put("b", 2).toString()), + TbMsg.newMsg("TEST", originatorFast, new TbMsgMetaData(), JacksonUtil.newObjectNode().put("a", 2).put("b", 2).toString()) + ); + + log.debug("rule-dispatcher [{}], db-callback [{}], slowMsg [{}], fastMsg [{}]", RULE_DISPATCHER_POOL_SIZE, DB_CALLBACK_POOL_SIZE, slowMsgList.size(), fastMsgList.size()); + + willAnswer(invocation -> { + TbContext ctx = invocation.getArgument(0); + TbMsg msg = invocation.getArgument(1); + log.debug("awaiting on slowProcessingLatch [{}]", msg); + try { + assertThat(slowProcessingLatch.await(30, TimeUnit.SECONDS)).as("await on slowProcessingLatch").isTrue(); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + return invocation.callRealMethod(); + }).given(node).processMsgAsync(eq(ctx), argThat(slowMsgList::contains)); + + // submit slow msg may block all rule engine dispatcher threads + slowMsgList.forEach(msg -> ruleEngineDispatcherExecutor.executeAsync(() -> node.onMsg(ctx, msg))); + // wait until dispatcher threads started with all slowMsg + verify(node, new Timeout(TimeUnit.SECONDS.toMillis(5), times(slowMsgList.size()))).onMsg(eq(ctx), argThat(slowMsgList::contains)); + + // submit fast have to return immediately + fastMsgList.forEach(msg -> ruleEngineDispatcherExecutor.executeAsync(() -> node.onMsg(ctx, msg))); + // wait until all fast messages processed + verify(ctx, new Timeout(TimeUnit.SECONDS.toMillis(5), times(fastMsgList.size()))).tellSuccess(any()); + + slowProcessingLatch.countDown(); + + verify(ctx, new Timeout(TimeUnit.SECONDS.toMillis(5), times(fastMsgList.size() + slowMsgList.size()))).tellSuccess(any()); + + verify(ctx, never()).tellFailure(any(), any()); + } + } From b8a7c6a3cd758508c4cdcb7b811a839f5fe080d9 Mon Sep 17 00:00:00 2001 From: ViacheslavKlimov Date: Tue, 15 Aug 2023 16:40:26 +0300 Subject: [PATCH 21/26] Don't init unneeded actors and consumers for dedicated Rule Engine --- .../server/actors/ActorSystemContext.java | 1 + .../server/actors/app/AppActor.java | 54 ++++++++++-------- .../server/actors/tenant/TenantActor.java | 23 ++++---- .../DefaultTbRuleEngineConsumerService.java | 55 +++++++++++-------- .../discovery/HashPartitionServiceTest.java | 22 ++++++++ .../DefaultTbServiceInfoProvider.java | 3 + .../queue/discovery/HashPartitionService.java | 15 +++++ .../queue/discovery/PartitionService.java | 3 + 8 files changed, 118 insertions(+), 58 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java b/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java index 7e5dfed5f0..1abfe79a76 100644 --- a/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java +++ b/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java @@ -249,6 +249,7 @@ public class ActorSystemContext { private RuleNodeStateService ruleNodeStateService; @Autowired + @Getter private PartitionService partitionService; @Autowired diff --git a/application/src/main/java/org/thingsboard/server/actors/app/AppActor.java b/application/src/main/java/org/thingsboard/server/actors/app/AppActor.java index e8a56fab16..52abeca2c0 100644 --- a/application/src/main/java/org/thingsboard/server/actors/app/AppActor.java +++ b/application/src/main/java/org/thingsboard/server/actors/app/AppActor.java @@ -46,8 +46,8 @@ import org.thingsboard.server.dao.tenant.TenantService; import org.thingsboard.server.service.transport.msg.TransportToDeviceActorMsgWrapper; import java.util.HashSet; +import java.util.Optional; import java.util.Set; -import java.util.UUID; @Slf4j public class AppActor extends ContextAwareActor { @@ -124,14 +124,13 @@ public class AppActor extends ContextAwareActor { try { if (systemContext.isTenantComponentsInitEnabled()) { PageDataIterable tenantIterator = new PageDataIterable<>(tenantService::findTenants, ENTITY_PACK_LIMIT); - Set assignedProfiles = systemContext.getServiceInfoProvider().getAssignedTenantProfiles(); for (Tenant tenant : tenantIterator) { - if (!assignedProfiles.isEmpty() && !assignedProfiles.contains(tenant.getTenantProfileId().getId())) { - continue; - } log.debug("[{}] Creating tenant actor", tenant.getId()); - getOrCreateTenantActor(tenant.getId()); - log.debug("[{}] Tenant actor created.", tenant.getId()); + getOrCreateTenantActor(tenant.getId()).ifPresentOrElse(tenantActor -> { + log.debug("[{}] Tenant actor created.", tenant.getId()); + }, () -> { + log.debug("[{}] Skipped actor creation", tenant.getId()); + }); } } log.info("Main system actor started."); @@ -145,7 +144,9 @@ public class AppActor extends ContextAwareActor { msg.getMsg().getCallback().onFailure(new RuleEngineException("Message has system tenant id!")); } else { if (!deletedTenants.contains(msg.getTenantId())) { - getOrCreateTenantActor(msg.getTenantId()).tell(msg); + getOrCreateTenantActor(msg.getTenantId()).ifPresentOrElse(actor -> { + actor.tell(msg); + }, () -> msg.getMsg().getCallback().onSuccess()); } else { msg.getMsg().getCallback().onSuccess(); } @@ -165,12 +166,13 @@ public class AppActor extends ContextAwareActor { log.info("[{}] Handling tenant deleted notification: {}", msg.getTenantId(), msg); deletedTenants.add(tenantId); ctx.stop(new TbEntityActorId(tenantId)); - } else { - target = getOrCreateTenantActor(msg.getTenantId()); + return; } - } else { - target = getOrCreateTenantActor(msg.getTenantId()); } + target = getOrCreateTenantActor(msg.getTenantId()).orElseGet(() -> { + log.debug("Ignoring component lifecycle msg for tenant {} because it is not managed by this service", msg.getTenantId()); + return null; + }); } if (target != null) { target.tellWithHighPriority(msg); @@ -181,12 +183,13 @@ public class AppActor extends ContextAwareActor { private void onToDeviceActorMsg(TenantAwareMsg msg, boolean priority) { if (!deletedTenants.contains(msg.getTenantId())) { - TbActorRef tenantActor = getOrCreateTenantActor(msg.getTenantId()); - if (priority) { - tenantActor.tellWithHighPriority(msg); - } else { - tenantActor.tell(msg); - } + getOrCreateTenantActor(msg.getTenantId()).ifPresent(tenantActor -> { + if (priority) { + tenantActor.tellWithHighPriority(msg); + } else { + tenantActor.tell(msg); + } + }); } else { if (msg instanceof TransportToDeviceActorMsgWrapper) { ((TransportToDeviceActorMsgWrapper) msg).getCallback().onSuccess(); @@ -194,10 +197,15 @@ public class AppActor extends ContextAwareActor { } } - private TbActorRef getOrCreateTenantActor(TenantId tenantId) { - return ctx.getOrCreateChildActor(new TbEntityActorId(tenantId), - () -> DefaultActorService.TENANT_DISPATCHER_NAME, - () -> new TenantActor.ActorCreator(systemContext, tenantId)); + private Optional getOrCreateTenantActor(TenantId tenantId) { + if (systemContext.getServiceInfoProvider().isService(ServiceType.TB_CORE) || + systemContext.getPartitionService().isManagedByCurrentService(tenantId)) { + return Optional.of(ctx.getOrCreateChildActor(new TbEntityActorId(tenantId), + () -> DefaultActorService.TENANT_DISPATCHER_NAME, + () -> new TenantActor.ActorCreator(systemContext, tenantId))); + } else { + return Optional.empty(); + } } private void onToEdgeSessionMsg(EdgeSessionMsg msg) { @@ -205,7 +213,7 @@ public class AppActor extends ContextAwareActor { if (ModelConstants.SYSTEM_TENANT.equals(msg.getTenantId())) { log.warn("Message has system tenant id: {}", msg); } else { - target = getOrCreateTenantActor(msg.getTenantId()); + target = getOrCreateTenantActor(msg.getTenantId()).orElse(null); } if (target != null) { target.tellWithHighPriority(msg); diff --git a/application/src/main/java/org/thingsboard/server/actors/tenant/TenantActor.java b/application/src/main/java/org/thingsboard/server/actors/tenant/TenantActor.java index 84b7757846..6cc249c87c 100644 --- a/application/src/main/java/org/thingsboard/server/actors/tenant/TenantActor.java +++ b/application/src/main/java/org/thingsboard/server/actors/tenant/TenantActor.java @@ -32,7 +32,6 @@ import org.thingsboard.server.actors.service.DefaultActorService; import org.thingsboard.server.common.data.ApiUsageState; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.Tenant; -import org.thingsboard.server.common.data.TenantProfile; import org.thingsboard.server.common.data.edge.Edge; import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.EdgeId; @@ -83,21 +82,21 @@ public class TenantActor extends RuleChainManagerActor { cantFindTenant = true; log.info("[{}] Started tenant actor for missing tenant.", tenantId); } else { - TenantProfile tenantProfile = systemContext.getTenantProfileCache().get(tenant.getTenantProfileId()); - isCore = systemContext.getServiceInfoProvider().isService(ServiceType.TB_CORE); isRuleEngine = systemContext.getServiceInfoProvider().isService(ServiceType.TB_RULE_ENGINE); if (isRuleEngine) { - try { - if (getApiUsageState().isReExecEnabled()) { - log.debug("[{}] Going to init rule chains", tenantId); - initRuleChains(); - } else { - log.info("[{}] Skip init of the rule chains due to API limits", tenantId); + if (systemContext.getPartitionService().isManagedByCurrentService(tenantId)) { + try { + if (getApiUsageState().isReExecEnabled()) { + log.debug("[{}] Going to init rule chains", tenantId); + initRuleChains(); + } else { + log.info("[{}] Skip init of the rule chains due to API limits", tenantId); + } + } catch (Exception e) { + log.info("Failed to check ApiUsage \"ReExecEnabled\"!!!", e); + cantFindTenant = true; } - } catch (Exception e) { - log.info("Failed to check ApiUsage \"ReExecEnabled\"!!!", e); - cantFindTenant = true; } } log.debug("[{}] Tenant actor started.", tenantId); diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbRuleEngineConsumerService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbRuleEngineConsumerService.java index a18b5c099c..19dea11665 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbRuleEngineConsumerService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbRuleEngineConsumerService.java @@ -156,7 +156,9 @@ public class DefaultTbRuleEngineConsumerService extends AbstractConsumerService< super.init("tb-rule-engine-consumer", "tb-rule-engine-notifications-consumer"); List queues = queueService.findAllQueues(); for (Queue configuration : queues) { - initConsumer(configuration); // TODO: if this Rule Engine is assigned specific profile, don't init other consumers and properly handle queue update events + if (partitionService.isManagedByCurrentService(configuration.getTenantId())) { + initConsumer(configuration); + } } } @@ -183,7 +185,12 @@ public class DefaultTbRuleEngineConsumerService extends AbstractConsumerService< if (event.getServiceType().equals(getServiceType())) { String serviceQueue = event.getQueueKey().getQueueName(); log.info("[{}] Subscribing to partitions: {}", serviceQueue, event.getPartitions()); - if (!consumerConfigurations.get(event.getQueueKey()).isConsumerPerPartition()) { + Queue configuration = consumerConfigurations.get(event.getQueueKey()); + if (configuration == null) { + log.warn("Received invalid partition change event for {} that is not managed by this service", event.getQueueKey()); + return; + } + if (!configuration.isConsumerPerPartition()) { consumers.get(event.getQueueKey()).subscribe(event.getPartitions()); } else { log.info("[{}] Subscribing consumer per partition: {}", serviceQueue, event.getPartitions()); @@ -425,32 +432,34 @@ public class DefaultTbRuleEngineConsumerService extends AbstractConsumerService< private void updateQueue(TransportProtos.QueueUpdateMsg queueUpdateMsg) { log.info("Received queue update msg: [{}]", queueUpdateMsg); - String queueName = queueUpdateMsg.getQueueName(); TenantId tenantId = new TenantId(new UUID(queueUpdateMsg.getTenantIdMSB(), queueUpdateMsg.getTenantIdLSB())); - QueueId queueId = new QueueId(new UUID(queueUpdateMsg.getQueueIdMSB(), queueUpdateMsg.getQueueIdLSB())); - QueueKey queueKey = new QueueKey(ServiceType.TB_RULE_ENGINE, queueUpdateMsg.getQueueName(), tenantId); - Queue queue = queueService.findQueueById(tenantId, queueId); - Queue oldQueue = consumerConfigurations.remove(queueKey); - if (oldQueue != null) { - if (oldQueue.isConsumerPerPartition()) { - TbTopicWithConsumerPerPartition consumerPerPartition = topicsConsumerPerPartition.remove(queueKey); - ReentrantLock lock = consumerPerPartition.getLock(); - try { - lock.lock(); - consumerPerPartition.getConsumers().values().forEach(TbQueueConsumer::unsubscribe); - } finally { - lock.unlock(); + if (partitionService.isManagedByCurrentService(tenantId)) { + QueueId queueId = new QueueId(new UUID(queueUpdateMsg.getQueueIdMSB(), queueUpdateMsg.getQueueIdLSB())); + String queueName = queueUpdateMsg.getQueueName(); + QueueKey queueKey = new QueueKey(ServiceType.TB_RULE_ENGINE, queueName, tenantId); + Queue queue = queueService.findQueueById(tenantId, queueId); + Queue oldQueue = consumerConfigurations.remove(queueKey); + if (oldQueue != null) { + if (oldQueue.isConsumerPerPartition()) { + TbTopicWithConsumerPerPartition consumerPerPartition = topicsConsumerPerPartition.remove(queueKey); + ReentrantLock lock = consumerPerPartition.getLock(); + try { + lock.lock(); + consumerPerPartition.getConsumers().values().forEach(TbQueueConsumer::unsubscribe); + } finally { + lock.unlock(); + } + } else { + TbQueueConsumer> consumer = consumers.remove(queueKey); + consumer.unsubscribe(); } - } else { - TbQueueConsumer> consumer = consumers.remove(queueKey); - consumer.unsubscribe(); } - } - initConsumer(queue); + initConsumer(queue); - if (!queue.isConsumerPerPartition()) { - launchConsumer(consumers.get(queueKey), consumerConfigurations.get(queueKey), consumerStats.get(queueKey), queueName); + if (!queue.isConsumerPerPartition()) { + launchConsumer(consumers.get(queueKey), consumerConfigurations.get(queueKey), consumerStats.get(queueKey), queueName); + } } partitionService.updateQueue(queueUpdateMsg); diff --git a/application/src/test/java/org/thingsboard/server/queue/discovery/HashPartitionServiceTest.java b/application/src/test/java/org/thingsboard/server/queue/discovery/HashPartitionServiceTest.java index 84024ecd1e..7bd9ec576f 100644 --- a/application/src/test/java/org/thingsboard/server/queue/discovery/HashPartitionServiceTest.java +++ b/application/src/test/java/org/thingsboard/server/queue/discovery/HashPartitionServiceTest.java @@ -46,6 +46,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Random; +import java.util.Set; import java.util.UUID; import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; @@ -274,4 +275,25 @@ public class HashPartitionServiceTest { }); } + @Test + public void testIsManagedByCurrentServiceCheck() { + TenantProfileId isolatedProfileId = new TenantProfileId(UUID.randomUUID()); + when(discoveryService.getAssignedTenantProfiles()).thenReturn(Set.of(isolatedProfileId.getId())); // dedicated server + TenantProfileId regularProfileId = new TenantProfileId(UUID.randomUUID()); + + TenantId isolatedTenantId = new TenantId(UUID.randomUUID()); + when(routingInfoService.getRoutingInfo(eq(isolatedTenantId))).thenReturn(new TenantRoutingInfo(isolatedTenantId, isolatedProfileId, true)); + TenantId regularTenantId = new TenantId(UUID.randomUUID()); + when(routingInfoService.getRoutingInfo(eq(regularTenantId))).thenReturn(new TenantRoutingInfo(regularTenantId, regularProfileId, false)); + + assertThat(clusterRoutingService.isManagedByCurrentService(isolatedTenantId)).isTrue(); + assertThat(clusterRoutingService.isManagedByCurrentService(regularTenantId)).isFalse(); + + + when(discoveryService.getAssignedTenantProfiles()).thenReturn(Collections.emptySet()); // common server + + assertThat(clusterRoutingService.isManagedByCurrentService(isolatedTenantId)).isTrue(); + assertThat(clusterRoutingService.isManagedByCurrentService(regularTenantId)).isTrue(); + } + } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/DefaultTbServiceInfoProvider.java b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/DefaultTbServiceInfoProvider.java index e9013ef345..64a8b70a1f 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/DefaultTbServiceInfoProvider.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/DefaultTbServiceInfoProvider.java @@ -85,6 +85,9 @@ public class DefaultTbServiceInfoProvider implements TbServiceInfoProvider { } else { serviceTypes = Collections.singletonList(ServiceType.of(serviceType)); } + if (!serviceTypes.contains(ServiceType.TB_RULE_ENGINE) || assignedTenantProfiles == null) { + assignedTenantProfiles = Collections.emptySet(); + } generateNewServiceInfoWithCurrentSystemInfo(); } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/HashPartitionService.java b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/HashPartitionService.java index 4c1894eae1..2db342d4e6 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/HashPartitionService.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/HashPartitionService.java @@ -186,6 +186,21 @@ public class HashPartitionService implements PartitionService { removeTenant(tenantId); } + @Override + public boolean isManagedByCurrentService(TenantId tenantId) { + Set assignedTenantProfiles = serviceInfoProvider.getAssignedTenantProfiles(); + if (assignedTenantProfiles.isEmpty()) { + // TODO: refactor this for common servers + return true; + } else { + if (tenantId.isSysTenantId()) { + return false; + } + TenantProfileId profileId = tenantRoutingInfoService.getRoutingInfo(tenantId).getProfileId(); + return assignedTenantProfiles.contains(profileId.getId()); + } + } + @Override public TopicPartitionInfo resolve(ServiceType serviceType, String queueName, TenantId tenantId, EntityId entityId) { TenantId isolatedOrSystemTenantId = getIsolatedOrSystemTenantId(serviceType, tenantId); diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/PartitionService.java b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/PartitionService.java index faa4d956a8..b55ba79f67 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/PartitionService.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/PartitionService.java @@ -64,4 +64,7 @@ public interface PartitionService { void updateQueue(TransportProtos.QueueUpdateMsg queueUpdateMsg); void removeQueue(TransportProtos.QueueDeleteMsg queueDeleteMsg); + + boolean isManagedByCurrentService(TenantId tenantId); + } From 97e9f43f65173e2813a11e0e952592bdebe2ce55 Mon Sep 17 00:00:00 2001 From: ViacheslavKlimov Date: Tue, 15 Aug 2023 17:20:52 +0300 Subject: [PATCH 22/26] Add createCondition to TbActorCtx --- .../org/thingsboard/server/actors/app/AppActor.java | 13 +++++-------- .../ruleChain/RuleChainActorMessageProcessor.java | 3 ++- .../actors/ruleChain/RuleChainManagerActor.java | 3 ++- .../server/actors/tenant/TenantActor.java | 3 ++- .../org/thingsboard/server/actors/TbActorCtx.java | 2 +- .../thingsboard/server/actors/TbActorMailbox.java | 4 ++-- 6 files changed, 14 insertions(+), 14 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/actors/app/AppActor.java b/application/src/main/java/org/thingsboard/server/actors/app/AppActor.java index 52abeca2c0..89a000594b 100644 --- a/application/src/main/java/org/thingsboard/server/actors/app/AppActor.java +++ b/application/src/main/java/org/thingsboard/server/actors/app/AppActor.java @@ -198,14 +198,11 @@ public class AppActor extends ContextAwareActor { } private Optional getOrCreateTenantActor(TenantId tenantId) { - if (systemContext.getServiceInfoProvider().isService(ServiceType.TB_CORE) || - systemContext.getPartitionService().isManagedByCurrentService(tenantId)) { - return Optional.of(ctx.getOrCreateChildActor(new TbEntityActorId(tenantId), - () -> DefaultActorService.TENANT_DISPATCHER_NAME, - () -> new TenantActor.ActorCreator(systemContext, tenantId))); - } else { - return Optional.empty(); - } + return Optional.ofNullable(ctx.getOrCreateChildActor(new TbEntityActorId(tenantId), + () -> DefaultActorService.TENANT_DISPATCHER_NAME, + () -> new TenantActor.ActorCreator(systemContext, tenantId), + () -> systemContext.getServiceInfoProvider().isService(ServiceType.TB_CORE) || + systemContext.getPartitionService().isManagedByCurrentService(tenantId))); } private void onToEdgeSessionMsg(EdgeSessionMsg msg) { diff --git a/application/src/main/java/org/thingsboard/server/actors/ruleChain/RuleChainActorMessageProcessor.java b/application/src/main/java/org/thingsboard/server/actors/ruleChain/RuleChainActorMessageProcessor.java index 4b868dfcbf..b3d7120969 100644 --- a/application/src/main/java/org/thingsboard/server/actors/ruleChain/RuleChainActorMessageProcessor.java +++ b/application/src/main/java/org/thingsboard/server/actors/ruleChain/RuleChainActorMessageProcessor.java @@ -167,7 +167,8 @@ public class RuleChainActorMessageProcessor extends ComponentMsgProcessor DefaultActorService.RULE_DISPATCHER_NAME, - () -> new RuleNodeActor.ActorCreator(systemContext, tenantId, entityId, ruleChainName, ruleNode.getId())); + () -> new RuleNodeActor.ActorCreator(systemContext, tenantId, entityId, ruleChainName, ruleNode.getId()), + () -> true); } private void initRoutes(RuleChain ruleChain, List ruleNodeList) { diff --git a/application/src/main/java/org/thingsboard/server/actors/ruleChain/RuleChainManagerActor.java b/application/src/main/java/org/thingsboard/server/actors/ruleChain/RuleChainManagerActor.java index 7f919754fc..987554c683 100644 --- a/application/src/main/java/org/thingsboard/server/actors/ruleChain/RuleChainManagerActor.java +++ b/application/src/main/java/org/thingsboard/server/actors/ruleChain/RuleChainManagerActor.java @@ -94,7 +94,8 @@ public abstract class RuleChainManagerActor extends ContextAwareActor { } else { return new RuleChainActor.ActorCreator(systemContext, tenantId, ruleChain); } - }); + }, + () -> true); } protected TbActorRef getEntityActorRef(EntityId entityId) { diff --git a/application/src/main/java/org/thingsboard/server/actors/tenant/TenantActor.java b/application/src/main/java/org/thingsboard/server/actors/tenant/TenantActor.java index 6cc249c87c..eddb932084 100644 --- a/application/src/main/java/org/thingsboard/server/actors/tenant/TenantActor.java +++ b/application/src/main/java/org/thingsboard/server/actors/tenant/TenantActor.java @@ -274,7 +274,8 @@ public class TenantActor extends RuleChainManagerActor { private TbActorRef getOrCreateDeviceActor(DeviceId deviceId) { return ctx.getOrCreateChildActor(new TbEntityActorId(deviceId), () -> DefaultActorService.DEVICE_DISPATCHER_NAME, - () -> new DeviceActorCreator(systemContext, tenantId, deviceId)); + () -> new DeviceActorCreator(systemContext, tenantId, deviceId), + () -> true); } private void onToEdgeSessionMsg(EdgeSessionMsg msg) { diff --git a/common/actor/src/main/java/org/thingsboard/server/actors/TbActorCtx.java b/common/actor/src/main/java/org/thingsboard/server/actors/TbActorCtx.java index 2a8f641c4e..bbe19fb3dd 100644 --- a/common/actor/src/main/java/org/thingsboard/server/actors/TbActorCtx.java +++ b/common/actor/src/main/java/org/thingsboard/server/actors/TbActorCtx.java @@ -32,7 +32,7 @@ public interface TbActorCtx extends TbActorRef { void stop(TbActorId target); - TbActorRef getOrCreateChildActor(TbActorId actorId, Supplier dispatcher, Supplier creator); + TbActorRef getOrCreateChildActor(TbActorId actorId, Supplier dispatcher, Supplier creator, Supplier createCondition); void broadcastToChildren(TbActorMsg msg); diff --git a/common/actor/src/main/java/org/thingsboard/server/actors/TbActorMailbox.java b/common/actor/src/main/java/org/thingsboard/server/actors/TbActorMailbox.java index 857c45ad84..34537c143c 100644 --- a/common/actor/src/main/java/org/thingsboard/server/actors/TbActorMailbox.java +++ b/common/actor/src/main/java/org/thingsboard/server/actors/TbActorMailbox.java @@ -214,9 +214,9 @@ public final class TbActorMailbox implements TbActorCtx { } @Override - public TbActorRef getOrCreateChildActor(TbActorId actorId, Supplier dispatcher, Supplier creator) { + public TbActorRef getOrCreateChildActor(TbActorId actorId, Supplier dispatcher, Supplier creator, Supplier createCondition) { TbActorRef actorRef = system.getActor(actorId); - if (actorRef == null) { + if (actorRef == null && createCondition.get()) { return system.createChildActor(dispatcher.get(), creator.get(), selfId); } else { return actorRef; From 44ea477b7bb185d7eca65761d7131bedb2c40aba Mon Sep 17 00:00:00 2001 From: Sergey Matvienko Date: Tue, 15 Aug 2023 22:40:01 +0200 Subject: [PATCH 23/26] TbMathNode: refactored to act in non-blocking style. All messages go through queue by originator with single semaphore and never wait on tryAcquire. Test refactored to provide more details on how slaw and fast messages being submitted and processed --- .../rule/engine/math/TbMathNode.java | 95 ++++++++++++++----- .../rule/engine/math/TbMathNodeTest.java | 67 ++++++++----- 2 files changed, 114 insertions(+), 48 deletions(-) diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/math/TbMathNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/math/TbMathNode.java index f48e723494..0c34f4fdcc 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/math/TbMathNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/math/TbMathNode.java @@ -19,6 +19,8 @@ import com.fasterxml.jackson.databind.node.ObjectNode; import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; import com.google.common.util.concurrent.MoreExecutors; +import lombok.Data; +import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import net.objecthunter.exp4j.Expression; import net.objecthunter.exp4j.ExpressionBuilder; @@ -44,6 +46,8 @@ import java.math.BigDecimal; import java.math.RoundingMode; import java.util.List; import java.util.Optional; +import java.util.Queue; +import java.util.concurrent.ConcurrentLinkedQueue; import java.util.concurrent.ConcurrentMap; import java.util.concurrent.Semaphore; import java.util.concurrent.TimeUnit; @@ -79,9 +83,8 @@ import java.util.stream.Collectors; ) public class TbMathNode implements TbNode { - private static final ConcurrentMap semaphores = new ConcurrentReferenceHashMap<>(16, ConcurrentReferenceHashMap.ReferenceType.WEAK); + private static final ConcurrentMap> locks = new ConcurrentReferenceHashMap<>(16, ConcurrentReferenceHashMap.ReferenceType.WEAK); private final ThreadLocal customExpression = new ThreadLocal<>(); - private TbMathNodeConfiguration config; private boolean msgBodyToJsonConversionRequired; @@ -106,34 +109,58 @@ public class TbMathNode implements TbNode { @Override public void onMsg(TbContext ctx, TbMsg msg) { - var originator = msg.getOriginator(); - var originatorSemaphore = semaphores.computeIfAbsent(originator, tmp -> new Semaphore(1, true)); - boolean acquired = tryAcquire(originator, originatorSemaphore); + var semaphoreWithQueue = locks.computeIfAbsent(msg.getOriginator(), SemaphoreWithQueue::new); + semaphoreWithQueue.getQueue().add(new TbMsgTbContext(msg, ctx)); - if (!acquired) { - ctx.tellFailure(msg, new RuntimeException("Failed to process message for originator synchronously")); - return; - } + tryProcessQueue(semaphoreWithQueue); + } - try { - ListenableFuture resultMsgFuture = processMsgAsync(ctx, msg); - DonAsynchron.withCallback(resultMsgFuture, resultMsg -> { - try { - ctx.tellSuccess(resultMsg); - } finally { - originatorSemaphore.release(); + void tryProcessQueue(SemaphoreWithQueue lockAndQueue) { + final Semaphore semaphore = lockAndQueue.getSemaphore(); + final Queue queue = lockAndQueue.getQueue(); + while (!queue.isEmpty()) { + // The semaphore have to be acquired before EACH poll and released before NEXT poll. + // Otherwise, some message will remain unprocessed in queue + if (!semaphore.tryAcquire()) { + return; + } + TbMsgTbContext tbMsgTbContext = null; + try { + tbMsgTbContext = queue.poll(); + if (tbMsgTbContext == null) { + semaphore.release(); + continue; } - }, t -> { - try { - ctx.tellFailure(msg, t); - } finally { - originatorSemaphore.release(); + final TbMsg msg = tbMsgTbContext.getMsg(); + if (!msg.getCallback().isMsgValid()) { + log.trace("[{}] Skipping non-valid message [{}]", lockAndQueue.getEntityId(), msg); + semaphore.release(); + continue; } - }, ctx.getDbCallbackExecutor()); - } catch (Throwable e) { - originatorSemaphore.release(); - log.warn("[{}] Failed to process message: {}", originator, msg, e); - throw e; + //DO PROCESSING + final TbContext ctx = tbMsgTbContext.getCtx(); + final ListenableFuture resultMsgFuture = processMsgAsync(ctx, msg); + DonAsynchron.withCallback(resultMsgFuture, resultMsg -> { + try { + ctx.tellSuccess(resultMsg); + } finally { + lockAndQueue.getSemaphore().release(); + tryProcessQueue(lockAndQueue); + } + }, t -> { + try { + ctx.tellFailure(msg, t); + } finally { + lockAndQueue.getSemaphore().release(); + tryProcessQueue(lockAndQueue); + } + }, ctx.getDbCallbackExecutor()); + } catch (Throwable e) { + semaphore.release(); + log.warn("[{}] Failed to process message: {}", lockAndQueue.getEntityId(), tbMsgTbContext == null ? null : tbMsgTbContext.getMsg(), e); + throw e; + } + break; //submitted async exact one task. next poll will try on callback } } @@ -399,4 +426,20 @@ public class TbMathNode implements TbNode { @Override public void destroy() { } + + @Data + @RequiredArgsConstructor + static public class SemaphoreWithQueue { + final EntityId entityId; + final Semaphore semaphore = new Semaphore(1); + final Queue queue = new ConcurrentLinkedQueue<>(); + } + + @Data + @RequiredArgsConstructor + static public class TbMsgTbContext { + final TbMsg msg; + final TbContext ctx; + } + } diff --git a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/math/TbMathNodeTest.java b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/math/TbMathNodeTest.java index 4c36d51a7d..50424a1521 100644 --- a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/math/TbMathNodeTest.java +++ b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/math/TbMathNodeTest.java @@ -53,6 +53,8 @@ import java.util.Optional; import java.util.UUID; import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; +import java.util.stream.IntStream; import static org.assertj.core.api.Assertions.assertThat; import static org.junit.jupiter.api.Assertions.assertThrows; @@ -90,19 +92,9 @@ public class TbMathNodeTest { @BeforeEach public void before() { - dbCallbackExecutor = new AbstractListeningExecutor() { - @Override - protected int getThreadPollSize() { - return DB_CALLBACK_POOL_SIZE; - } - }; + dbCallbackExecutor = new DBCallbackExecutor(); dbCallbackExecutor.init(); - ruleEngineDispatcherExecutor = new AbstractListeningExecutor() { - @Override - protected int getThreadPollSize() { - return RULE_DISPATCHER_POOL_SIZE; - } - }; + ruleEngineDispatcherExecutor = new RuleDispatcherExecutor(); ruleEngineDispatcherExecutor.init(); lenient().when(ctx.getAttributesService()).thenReturn(attributesService); @@ -528,21 +520,20 @@ public class TbMathNodeTest { EntityId originatorFast = DeviceId.fromString("c45360ff-7906-4102-a2ae-3495a86168d0"); CountDownLatch slowProcessingLatch = new CountDownLatch(1); - List slowMsgList = List.of( - TbMsg.newMsg("TEST", originatorSlow, new TbMsgMetaData(), JacksonUtil.newObjectNode().put("a", 2).put("b", 2).toString()), - TbMsg.newMsg("TEST", originatorSlow, new TbMsgMetaData(), JacksonUtil.newObjectNode().put("a", 2).put("b", 2).toString()) - ); - List fastMsgList = List.of( - TbMsg.newMsg("TEST", originatorFast, new TbMsgMetaData(), JacksonUtil.newObjectNode().put("a", 2).put("b", 2).toString()), - TbMsg.newMsg("TEST", originatorFast, new TbMsgMetaData(), JacksonUtil.newObjectNode().put("a", 2).put("b", 2).toString()) - ); + List slowMsgList = IntStream.range(0, 5) + .mapToObj(x -> TbMsg.newMsg("TEST", originatorSlow, new TbMsgMetaData(), JacksonUtil.newObjectNode().put("a", 2).put("b", 2).toString())) + .collect(Collectors.toList()); + List fastMsgList = IntStream.range(0, 2) + .mapToObj(x -> TbMsg.newMsg("TEST", originatorFast, new TbMsgMetaData(), JacksonUtil.newObjectNode().put("a", 2).put("b", 2).toString())) + .collect(Collectors.toList()); + + assertThat(slowMsgList.size()).as("slow msgs >= rule-dispatcher pool size").isGreaterThanOrEqualTo(RULE_DISPATCHER_POOL_SIZE); log.debug("rule-dispatcher [{}], db-callback [{}], slowMsg [{}], fastMsg [{}]", RULE_DISPATCHER_POOL_SIZE, DB_CALLBACK_POOL_SIZE, slowMsgList.size(), fastMsgList.size()); willAnswer(invocation -> { - TbContext ctx = invocation.getArgument(0); TbMsg msg = invocation.getArgument(1); - log.debug("awaiting on slowProcessingLatch [{}]", msg); + log.debug("\uD83D\uDC0C processMsgAsync slow originator [{}][{}]", msg.getOriginator(), msg); try { assertThat(slowProcessingLatch.await(30, TimeUnit.SECONDS)).as("await on slowProcessingLatch").isTrue(); } catch (InterruptedException e) { @@ -551,6 +542,24 @@ public class TbMathNodeTest { return invocation.callRealMethod(); }).given(node).processMsgAsync(eq(ctx), argThat(slowMsgList::contains)); + willAnswer(invocation -> { + TbMsg msg = invocation.getArgument(1); + log.debug("\u26A1\uFE0F processMsgAsync FAST originator [{}][{}]", msg.getOriginator(), msg); + return invocation.callRealMethod(); + }).given(node).processMsgAsync(eq(ctx), argThat(fastMsgList::contains)); + + willAnswer(invocation -> { + TbMsg msg = invocation.getArgument(1); + log.debug("submit slow originator onMsg [{}][{}]", msg.getOriginator(), msg); + return invocation.callRealMethod(); + }).given(node).onMsg(eq(ctx), argThat(slowMsgList::contains)); + + willAnswer(invocation -> { + TbMsg msg = invocation.getArgument(1); + log.debug("submit FAST originator onMsg [{}][{}]", msg.getOriginator(), msg); + return invocation.callRealMethod(); + }).given(node).onMsg(eq(ctx), argThat(fastMsgList::contains)); + // submit slow msg may block all rule engine dispatcher threads slowMsgList.forEach(msg -> ruleEngineDispatcherExecutor.executeAsync(() -> node.onMsg(ctx, msg))); // wait until dispatcher threads started with all slowMsg @@ -568,4 +577,18 @@ public class TbMathNodeTest { verify(ctx, never()).tellFailure(any(), any()); } + static class RuleDispatcherExecutor extends AbstractListeningExecutor { + @Override + protected int getThreadPollSize() { + return RULE_DISPATCHER_POOL_SIZE; + } + } + + static class DBCallbackExecutor extends AbstractListeningExecutor { + @Override + protected int getThreadPollSize() { + return DB_CALLBACK_POOL_SIZE; + } + } + } From 916487105a0277fb76d071d672331456d82bd7c3 Mon Sep 17 00:00:00 2001 From: ViacheslavKlimov Date: Wed, 16 Aug 2023 15:36:01 +0300 Subject: [PATCH 24/26] Update profile isolation option description --- ui-ngx/src/assets/locale/locale.constant-en_US.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ui-ngx/src/assets/locale/locale.constant-en_US.json b/ui-ngx/src/assets/locale/locale.constant-en_US.json index b449cf1ba5..cb046cea48 100644 --- a/ui-ngx/src/assets/locale/locale.constant-en_US.json +++ b/ui-ngx/src/assets/locale/locale.constant-en_US.json @@ -3534,8 +3534,8 @@ "tenant-required": "Tenant is required", "search": "Search tenants", "selected-tenants": "{ count, plural, =1 {1 tenant} other {# tenants} } selected", - "isolated-tb-rule-engine": "Processing in isolated ThingsBoard Rule Engine container", - "isolated-tb-rule-engine-details": "Requires separate microservice(s) for the profile" + "isolated-tb-rule-engine": "Use isolated ThingsBoard Rule Engine queues", + "isolated-tb-rule-engine-details": "Each tenant will have its own processing queues" }, "tenant-profile": { "tenant-profile": "Tenant profile", From 46e0262419f92ce6f168800726fc4a2138c7664e Mon Sep 17 00:00:00 2001 From: ViacheslavKlimov Date: Wed, 16 Aug 2023 15:46:24 +0300 Subject: [PATCH 25/26] Update profile isolation option description --- ui-ngx/src/assets/locale/locale.constant-en_US.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ui-ngx/src/assets/locale/locale.constant-en_US.json b/ui-ngx/src/assets/locale/locale.constant-en_US.json index cb046cea48..513a10dba3 100644 --- a/ui-ngx/src/assets/locale/locale.constant-en_US.json +++ b/ui-ngx/src/assets/locale/locale.constant-en_US.json @@ -3535,7 +3535,7 @@ "search": "Search tenants", "selected-tenants": "{ count, plural, =1 {1 tenant} other {# tenants} } selected", "isolated-tb-rule-engine": "Use isolated ThingsBoard Rule Engine queues", - "isolated-tb-rule-engine-details": "Each tenant will have its own processing queues" + "isolated-tb-rule-engine-details": "Each tenant will have dedicated Rule Engine queues" }, "tenant-profile": { "tenant-profile": "Tenant profile", From d30f8a8352791529f145d8434827401aeeefe0e8 Mon Sep 17 00:00:00 2001 From: ViacheslavKlimov Date: Wed, 16 Aug 2023 17:53:44 +0300 Subject: [PATCH 26/26] Minor refactoring --- .../server/actors/app/AppActor.java | 31 +++++++++---------- .../queue/DefaultTbClusterService.java | 10 +++--- .../DefaultTbRuleEngineConsumerService.java | 4 +-- .../provider/KafkaMonolithQueueFactory.java | 2 +- .../KafkaTbRuleEngineQueueFactory.java | 2 +- .../profile/tenant-profile.component.ts | 12 +++---- 6 files changed, 29 insertions(+), 32 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/actors/app/AppActor.java b/application/src/main/java/org/thingsboard/server/actors/app/AppActor.java index 89a000594b..d4f04486e6 100644 --- a/application/src/main/java/org/thingsboard/server/actors/app/AppActor.java +++ b/application/src/main/java/org/thingsboard/server/actors/app/AppActor.java @@ -143,13 +143,9 @@ public class AppActor extends ContextAwareActor { if (TenantId.SYS_TENANT_ID.equals(msg.getTenantId())) { msg.getMsg().getCallback().onFailure(new RuleEngineException("Message has system tenant id!")); } else { - if (!deletedTenants.contains(msg.getTenantId())) { - getOrCreateTenantActor(msg.getTenantId()).ifPresentOrElse(actor -> { - actor.tell(msg); - }, () -> msg.getMsg().getCallback().onSuccess()); - } else { - msg.getMsg().getCallback().onSuccess(); - } + getOrCreateTenantActor(msg.getTenantId()).ifPresentOrElse(actor -> { + actor.tell(msg); + }, () -> msg.getMsg().getCallback().onSuccess()); } } @@ -182,22 +178,23 @@ public class AppActor extends ContextAwareActor { } private void onToDeviceActorMsg(TenantAwareMsg msg, boolean priority) { - if (!deletedTenants.contains(msg.getTenantId())) { - getOrCreateTenantActor(msg.getTenantId()).ifPresent(tenantActor -> { - if (priority) { - tenantActor.tellWithHighPriority(msg); - } else { - tenantActor.tell(msg); - } - }); - } else { + getOrCreateTenantActor(msg.getTenantId()).ifPresentOrElse(tenantActor -> { + if (priority) { + tenantActor.tellWithHighPriority(msg); + } else { + tenantActor.tell(msg); + } + }, () -> { if (msg instanceof TransportToDeviceActorMsgWrapper) { ((TransportToDeviceActorMsgWrapper) msg).getCallback().onSuccess(); } - } + }); } private Optional getOrCreateTenantActor(TenantId tenantId) { + if (deletedTenants.contains(tenantId)) { + return Optional.empty(); + } return Optional.ofNullable(ctx.getOrCreateChildActor(new TbEntityActorId(tenantId), () -> DefaultActorService.TENANT_DISPATCHER_NAME, () -> new TenantActor.ActorCreator(systemContext, tenantId), diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java index 5a2db8019e..76631aaa95 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java @@ -591,6 +591,11 @@ public class DefaultTbClusterService implements TbClusterService { tbTransportServices.removeAll(tbCoreServices); tbCoreServices.removeAll(tbRuleEngineServices); + for (String ruleEngineServiceId : tbRuleEngineServices) { + TopicPartitionInfo tpi = notificationsTopicService.getNotificationsTopic(ServiceType.TB_RULE_ENGINE, ruleEngineServiceId); + producerProvider.getRuleEngineNotificationsMsgProducer().send(tpi, new TbProtoQueueMsg<>(UUID.randomUUID(), ruleEngineMsg), null); + toRuleEngineNfs.incrementAndGet(); + } for (String coreServiceId : tbCoreServices) { TopicPartitionInfo tpi = notificationsTopicService.getNotificationsTopic(ServiceType.TB_CORE, coreServiceId); producerProvider.getTbCoreNotificationsMsgProducer().send(tpi, new TbProtoQueueMsg<>(UUID.randomUUID(), coreMsg), null); @@ -601,10 +606,5 @@ public class DefaultTbClusterService implements TbClusterService { producerProvider.getTransportNotificationsMsgProducer().send(tpi, new TbProtoQueueMsg<>(UUID.randomUUID(), transportMsg), null); toTransportNfs.incrementAndGet(); } - for (String ruleEngineServiceId : tbRuleEngineServices) { - TopicPartitionInfo tpi = notificationsTopicService.getNotificationsTopic(ServiceType.TB_RULE_ENGINE, ruleEngineServiceId); - producerProvider.getRuleEngineNotificationsMsgProducer().send(tpi, new TbProtoQueueMsg<>(UUID.randomUUID(), ruleEngineMsg), null); - toRuleEngineNfs.incrementAndGet(); - } } } diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbRuleEngineConsumerService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbRuleEngineConsumerService.java index 19dea11665..f4716b925f 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbRuleEngineConsumerService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbRuleEngineConsumerService.java @@ -104,7 +104,7 @@ public class DefaultTbRuleEngineConsumerService extends AbstractConsumerService< @Value("${queue.rule-engine.prometheus-stats.enabled:false}") boolean prometheusStatsEnabled; @Value("${queue.rule-engine.topic-deletion-delay:30}") - private int topicDeletionDelay; + private int topicDeletionDelayInSec; private final StatsFactory statsFactory; private final TbRuleEngineSubmitStrategyFactory submitStrategyFactory; @@ -506,7 +506,7 @@ public class DefaultTbRuleEngineConsumerService extends AbstractConsumerService< } private void processQueueDeletion(Queue queue, TbQueueConsumer> consumer) { - long finishTs = System.currentTimeMillis() + TimeUnit.SECONDS.toMillis(topicDeletionDelay); + long finishTs = System.currentTimeMillis() + TimeUnit.SECONDS.toMillis(topicDeletionDelayInSec); try { int n = 0; while (System.currentTimeMillis() <= finishTs) { diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaMonolithQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaMonolithQueueFactory.java index 779bf9fae3..22a16de64f 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaMonolithQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaMonolithQueueFactory.java @@ -187,7 +187,7 @@ public class KafkaMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEngi consumerBuilder.settings(kafkaSettings); consumerBuilder.topic(configuration.getTopic()); consumerBuilder.clientId("re-" + queueName + "-consumer-" + serviceInfoProvider.getServiceId() + "-" + consumerCount.incrementAndGet()); - consumerBuilder.groupId("re-" + queueName + (!configuration.getTenantId().isSysTenantId() ? "-" + configuration.getTenantId() : "") + "-consumer"); + consumerBuilder.groupId("re-" + queueName + (configuration.getTenantId().isSysTenantId() ? "" : ("-" + configuration.getTenantId())) + "-consumer"); consumerBuilder.decoder(msg -> new TbProtoQueueMsg<>(msg.getKey(), ToRuleEngineMsg.parseFrom(msg.getData()), msg.getHeaders())); consumerBuilder.admin(ruleEngineAdmin); consumerBuilder.statsService(consumerStatsService); diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbRuleEngineQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbRuleEngineQueueFactory.java index eb387a84f4..2e3bf784d7 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbRuleEngineQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbRuleEngineQueueFactory.java @@ -166,7 +166,7 @@ public class KafkaTbRuleEngineQueueFactory implements TbRuleEngineQueueFactory { consumerBuilder.settings(kafkaSettings); consumerBuilder.topic(configuration.getTopic()); consumerBuilder.clientId("re-" + queueName + "-consumer-" + serviceInfoProvider.getServiceId() + "-" + consumerCount.incrementAndGet()); - consumerBuilder.groupId("re-" + queueName + (!configuration.getTenantId().isSysTenantId() ? "-" + configuration.getTenantId() : "") + "-consumer"); + consumerBuilder.groupId("re-" + queueName + (configuration.getTenantId().isSysTenantId() ? "" : ("-" + configuration.getTenantId())) + "-consumer"); consumerBuilder.decoder(msg -> new TbProtoQueueMsg<>(msg.getKey(), ToRuleEngineMsg.parseFrom(msg.getData()), msg.getHeaders())); consumerBuilder.admin(ruleEngineAdmin); consumerBuilder.statsService(consumerStatsService); diff --git a/ui-ngx/src/app/modules/home/components/profile/tenant-profile.component.ts b/ui-ngx/src/app/modules/home/components/profile/tenant-profile.component.ts index e9142fa1fa..26d6611a3c 100644 --- a/ui-ngx/src/app/modules/home/components/profile/tenant-profile.component.ts +++ b/ui-ngx/src/app/modules/home/components/profile/tenant-profile.component.ts @@ -58,8 +58,8 @@ export class TenantProfileComponent extends EntityComponent { id: guid(), consumerPerPartition: true, name: 'Main', - packProcessingTimeout: 2000, - partitions: 2, + packProcessingTimeout: 10000, + partitions: 1, pollInterval: 2000, processingStrategy: { failurePercentage: 0, @@ -83,9 +83,9 @@ export class TenantProfileComponent extends EntityComponent { name: 'HighPriority', topic: 'tb_rule_engine.hp', pollInterval: 2000, - partitions: 2, + partitions: 1, consumerPerPartition: true, - packProcessingTimeout: 2000, + packProcessingTimeout: 10000, submitStrategy: { type: 'BURST', batchSize: 100 @@ -107,9 +107,9 @@ export class TenantProfileComponent extends EntityComponent { name: 'SequentialByOriginator', topic: 'tb_rule_engine.sq', pollInterval: 2000, - partitions: 2, + partitions: 1, consumerPerPartition: true, - packProcessingTimeout: 2000, + packProcessingTimeout: 10000, submitStrategy: { type: 'SEQUENTIAL_BY_ORIGINATOR', batchSize: 100