From 6751820e0a22d1fb3f07844bb758886e273816f0 Mon Sep 17 00:00:00 2001 From: ViacheslavKlimov Date: Mon, 14 Aug 2023 12:57:53 +0300 Subject: [PATCH] Dedicated Rule Engines for tenant profile --- .../server/actors/app/AppActor.java | 5 + .../DefaultTbRuleEngineConsumerService.java | 2 +- .../DefaultTenantRoutingInfoService.java | 8 +- .../src/main/resources/thingsboard.yml | 4 + .../discovery/HashPartitionServiceTest.java | 127 ++++++++++++++++-- common/cluster-api/src/main/proto/queue.proto | 1 + .../DefaultTbServiceInfoProvider.java | 11 +- .../queue/discovery/HashPartitionService.java | 84 +++++++++--- .../discovery/TbServiceInfoProvider.java | 5 + .../queue/discovery/TenantRoutingInfo.java | 2 + .../discovery/TenantRoutingInfoService.java | 1 + .../TransportTenantRoutingInfoService.java | 4 +- ...ersionControlTenantRoutingInfoService.java | 2 +- .../assets/locale/locale.constant-en_US.json | 2 +- 14 files changed, 223 insertions(+), 35 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/actors/app/AppActor.java b/application/src/main/java/org/thingsboard/server/actors/app/AppActor.java index fb6fbbdff2..e8a56fab16 100644 --- a/application/src/main/java/org/thingsboard/server/actors/app/AppActor.java +++ b/application/src/main/java/org/thingsboard/server/actors/app/AppActor.java @@ -47,6 +47,7 @@ import org.thingsboard.server.service.transport.msg.TransportToDeviceActorMsgWra import java.util.HashSet; import java.util.Set; +import java.util.UUID; @Slf4j public class AppActor extends ContextAwareActor { @@ -123,7 +124,11 @@ public class AppActor extends ContextAwareActor { try { if (systemContext.isTenantComponentsInitEnabled()) { PageDataIterable tenantIterator = new PageDataIterable<>(tenantService::findTenants, ENTITY_PACK_LIMIT); + Set assignedProfiles = systemContext.getServiceInfoProvider().getAssignedTenantProfiles(); for (Tenant tenant : tenantIterator) { + if (!assignedProfiles.isEmpty() && !assignedProfiles.contains(tenant.getTenantProfileId().getId())) { + continue; + } log.debug("[{}] Creating tenant actor", tenant.getId()); getOrCreateTenantActor(tenant.getId()); log.debug("[{}] Tenant actor created.", tenant.getId()); diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbRuleEngineConsumerService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbRuleEngineConsumerService.java index d560cfee1c..a18b5c099c 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbRuleEngineConsumerService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbRuleEngineConsumerService.java @@ -156,7 +156,7 @@ public class DefaultTbRuleEngineConsumerService extends AbstractConsumerService< super.init("tb-rule-engine-consumer", "tb-rule-engine-notifications-consumer"); List queues = queueService.findAllQueues(); for (Queue configuration : queues) { - initConsumer(configuration); + initConsumer(configuration); // TODO: if this Rule Engine is assigned specific profile, don't init other consumers and properly handle queue update events } } diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTenantRoutingInfoService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTenantRoutingInfoService.java index 400586235e..ea90364ef7 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTenantRoutingInfoService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTenantRoutingInfoService.java @@ -22,7 +22,6 @@ import org.thingsboard.server.common.data.TenantProfile; import org.thingsboard.server.common.data.exception.TenantNotFoundException; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.dao.tenant.TbTenantProfileCache; -import org.thingsboard.server.dao.tenant.TenantService; import org.thingsboard.server.queue.discovery.TenantRoutingInfo; import org.thingsboard.server.queue.discovery.TenantRoutingInfoService; @@ -31,12 +30,9 @@ import org.thingsboard.server.queue.discovery.TenantRoutingInfoService; @ConditionalOnExpression("'${service.type:null}'=='monolith' || '${service.type:null}'=='tb-core' || '${service.type:null}'=='tb-rule-engine'") public class DefaultTenantRoutingInfoService implements TenantRoutingInfoService { - private final TenantService tenantService; - private final TbTenantProfileCache tenantProfileCache; - public DefaultTenantRoutingInfoService(TenantService tenantService, TbTenantProfileCache tenantProfileCache) { - this.tenantService = tenantService; + public DefaultTenantRoutingInfoService(TbTenantProfileCache tenantProfileCache) { this.tenantProfileCache = tenantProfileCache; } @@ -44,7 +40,7 @@ public class DefaultTenantRoutingInfoService implements TenantRoutingInfoService public TenantRoutingInfo getRoutingInfo(TenantId tenantId) { TenantProfile tenantProfile = tenantProfileCache.get(tenantId); if (tenantProfile != null) { - return new TenantRoutingInfo(tenantId, tenantProfile.isIsolatedTbRuleEngine()); + return new TenantRoutingInfo(tenantId, tenantProfile.getId(), tenantProfile.isIsolatedTbRuleEngine()); } else { throw new TenantNotFoundException(tenantId); } diff --git a/application/src/main/resources/thingsboard.yml b/application/src/main/resources/thingsboard.yml index 8576f20783..2ce5c3ba21 100644 --- a/application/src/main/resources/thingsboard.yml +++ b/application/src/main/resources/thingsboard.yml @@ -1253,6 +1253,10 @@ service: type: "${TB_SERVICE_TYPE:monolith}" # monolith or tb-core or tb-rule-engine # Unique id for this service (autogenerated if empty) id: "${TB_SERVICE_ID:}" + rule_engine: + # Comma-separated list of tenant profiles ids assigned to this Rule Engine. + # This Rule Engine will only be responsible for tenants with these profiles (in case 'isolation' option is enabled in profile). + assigned_tenant_profiles: "${TB_RULE_ENGINE_ASSIGNED_TENANT_PROFILES:}" metrics: # Enable/disable actuator metrics. diff --git a/application/src/test/java/org/thingsboard/server/queue/discovery/HashPartitionServiceTest.java b/application/src/test/java/org/thingsboard/server/queue/discovery/HashPartitionServiceTest.java index 25b06e7840..84024ecd1e 100644 --- a/application/src/test/java/org/thingsboard/server/queue/discovery/HashPartitionServiceTest.java +++ b/application/src/test/java/org/thingsboard/server/queue/discovery/HashPartitionServiceTest.java @@ -18,6 +18,7 @@ package org.thingsboard.server.queue.discovery; import com.datastax.oss.driver.api.core.uuid.Uuids; import lombok.SneakyThrows; import lombok.extern.slf4j.Slf4j; +import org.apache.commons.collections4.ListUtils; import org.junit.Assert; import org.junit.Before; import org.junit.Test; @@ -25,24 +26,35 @@ import org.junit.runner.RunWith; import org.mockito.junit.MockitoJUnitRunner; import org.springframework.context.ApplicationEventPublisher; import org.springframework.test.util.ReflectionTestUtils; +import org.thingsboard.server.common.data.DataConstants; import org.thingsboard.server.common.data.id.DeviceId; +import org.thingsboard.server.common.data.id.QueueId; import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.id.TenantProfileId; +import org.thingsboard.server.common.data.id.UUIDBased; +import org.thingsboard.server.common.data.queue.Queue; import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; -import org.thingsboard.server.gen.transport.TransportProtos; +import org.thingsboard.server.gen.transport.TransportProtos.ServiceInfo; import java.text.SimpleDateFormat; import java.util.ArrayList; +import java.util.Collection; import java.util.Collections; import java.util.Comparator; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Random; +import java.util.UUID; import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; +import java.util.stream.Stream; +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; @Slf4j @RunWith(MockitoJUnitRunner.class) @@ -57,7 +69,7 @@ public class HashPartitionServiceTest { private ApplicationEventPublisher applicationEventPublisher; private QueueRoutingInfoService queueRoutingInfoService; - private String hashFunctionName = "sha256"; + private String hashFunctionName = "murmur3_128"; @Before public void setup() throws Exception { @@ -74,15 +86,15 @@ public class HashPartitionServiceTest { ReflectionTestUtils.setField(clusterRoutingService, "vcTopic", "tb.vc"); ReflectionTestUtils.setField(clusterRoutingService, "vcPartitions", 10); ReflectionTestUtils.setField(clusterRoutingService, "hashFunctionName", hashFunctionName); - TransportProtos.ServiceInfo currentServer = TransportProtos.ServiceInfo.newBuilder() + ServiceInfo currentServer = ServiceInfo.newBuilder() .setServiceId("tb-core-0") .addAllServiceTypes(Collections.singletonList(ServiceType.TB_CORE.name())) .build(); // when(queueService.resolve(Mockito.any(), Mockito.anyString())).thenAnswer(i -> i.getArguments()[1]); // when(discoveryService.getServiceInfo()).thenReturn(currentServer); - List otherServers = new ArrayList<>(); + List otherServers = new ArrayList<>(); for (int i = 1; i < SERVER_COUNT; i++) { - otherServers.add(TransportProtos.ServiceInfo.newBuilder() + otherServers.add(ServiceInfo.newBuilder() .setServiceId("tb-rule-" + i) .addAllServiceTypes(Collections.singletonList(ServiceType.TB_CORE.name())) .build()); @@ -122,10 +134,10 @@ public class HashPartitionServiceTest { int queueCount = 3; int partitionCount = 3; - List services = new ArrayList<>(); + List services = new ArrayList<>(); for (int i = 0; i < serverCount; i++) { - services.add(TransportProtos.ServiceInfo.newBuilder().setServiceId("RE-" + i).build()); + services.add(ServiceInfo.newBuilder().setServiceId("RE-" + i).build()); } long start = System.currentTimeMillis(); @@ -140,7 +152,7 @@ public class HashPartitionServiceTest { for (int queueIndex = 0; queueIndex < queueCount; queueIndex++) { QueueKey queueKey = new QueueKey(ServiceType.TB_RULE_ENGINE, "queue" + queueIndex, tenantId); for (int partition = 0; partition < partitionCount; partition++) { - TransportProtos.ServiceInfo serviceInfo = clusterRoutingService.resolveByPartitionIdx(services, queueKey, partition); + ServiceInfo serviceInfo = clusterRoutingService.resolveByPartitionIdx(services, queueKey, partition); String serviceId = serviceInfo.getServiceId(); map.put(serviceId, map.get(serviceId) + 1); } @@ -163,4 +175,103 @@ public class HashPartitionServiceTest { Assert.assertTrue(diffPercent < maxDiffPercent); } + @Test + public void testPartitionsAssignmentWithDedicatedServers() { + int isolatedProfilesCount = 5; + int tenantsCountPerProfile = 100; + int dedicatedServerSetsCount = 3; + int serversCountPerSet = 3; + int profilesPerSet = (int) Math.ceil((double) isolatedProfilesCount / dedicatedServerSetsCount); + + List isolatedTenantProfiles = Stream.generate(() -> new TenantProfileId(UUID.randomUUID())) + .limit(isolatedProfilesCount).collect(Collectors.toList()); + Map tenants = new HashMap<>(); + for (TenantProfileId tenantProfileId : isolatedTenantProfiles) { + for (int i = 0; i < tenantsCountPerProfile; i++) { + tenants.put(new TenantId(UUID.randomUUID()), tenantProfileId); + } + } + + List queues = new ArrayList<>(); + Queue systemQueue = new Queue(); + systemQueue.setTenantId(TenantId.SYS_TENANT_ID); + systemQueue.setName("Main"); + systemQueue.setTopic(DataConstants.MAIN_QUEUE_TOPIC); + systemQueue.setPartitions(10); + systemQueue.setId(new QueueId(UUID.randomUUID())); + queues.add(systemQueue); + tenants.forEach((tenantId, profileId) -> { + Queue isolatedQueue = new Queue(); + isolatedQueue.setTenantId(tenantId); + isolatedQueue.setName("Main"); + isolatedQueue.setTopic(DataConstants.MAIN_QUEUE_TOPIC); + isolatedQueue.setPartitions(2); + isolatedQueue.setId(new QueueId(UUID.randomUUID())); + queues.add(isolatedQueue); + when(routingInfoService.getRoutingInfo(eq(tenantId))).thenReturn(new TenantRoutingInfo(tenantId, profileId, true)); + }); + when(queueRoutingInfoService.getAllQueuesRoutingInfo()).thenReturn(queues.stream() + .map(QueueRoutingInfo::new).collect(Collectors.toList())); + + List ruleEngines = new ArrayList<>(); + Map> dedicatedServers = new HashMap<>(); + int serviceId = 0; + for (int i = 0; i < serversCountPerSet; i++) { + ServiceInfo commonServer = ServiceInfo.newBuilder() + .setServiceId("tb-rule-engine-" + serviceId) + .addAllServiceTypes(List.of(ServiceType.TB_RULE_ENGINE.name())) + .build(); + ruleEngines.add(commonServer); + serviceId++; + } + for (int i = 0; i < dedicatedServerSetsCount; i++) { + List assignedProfiles = ListUtils.partition(isolatedTenantProfiles, profilesPerSet).get(i); + for (int j = 0; j < serversCountPerSet; j++) { + ServiceInfo dedicatedServer = ServiceInfo.newBuilder() + .setServiceId("tb-rule-engine-" + serviceId) + .addAllServiceTypes(List.of(ServiceType.TB_RULE_ENGINE.name())) + .addAllAssignedTenantProfiles(assignedProfiles.stream().map(UUIDBased::toString).collect(Collectors.toList())) + .build(); + ruleEngines.add(dedicatedServer); + serviceId++; + + for (TenantProfileId assignedProfileId : assignedProfiles) { + dedicatedServers.computeIfAbsent(assignedProfileId, p -> new ArrayList<>()).add(dedicatedServer); + } + } + } + + Map>> serversPartitions = new HashMap<>(); + clusterRoutingService.init(); + for (ServiceInfo ruleEngine : ruleEngines) { + List other = new ArrayList<>(ruleEngines); + other.removeIf(serviceInfo -> serviceInfo.getServiceId().equals(ruleEngine.getServiceId())); + + clusterRoutingService.recalculatePartitions(ruleEngine, other); + clusterRoutingService.myPartitions.forEach((queueKey, partitions) -> { + serversPartitions.computeIfAbsent(queueKey, k -> new HashMap<>()).put(ruleEngine, partitions); + }); + } + assertThat(serversPartitions.keySet()).containsAll(queues.stream().map(queue -> new QueueKey(ServiceType.TB_RULE_ENGINE, queue)).collect(Collectors.toList())); + + serversPartitions.forEach((queueKey, partitionsPerServer) -> { + if (queueKey.getTenantId().isSysTenantId()) { + partitionsPerServer.forEach((server, partitions) -> { + assertThat(server.getAssignedTenantProfilesCount()).as("system queues are not assigned to dedicated servers").isZero(); + }); + } else { + List responsibleServers = dedicatedServers.get(tenants.get(queueKey.getTenantId())); + partitionsPerServer.forEach((server, partitions) -> { + assertThat(server.getAssignedTenantProfilesCount()).as("isolated queues are only assigned to dedicated servers").isPositive(); + assertThat(responsibleServers).contains(server); + }); + } + + List allPartitions = partitionsPerServer.values().stream() + .flatMap(Collection::stream) + .collect(Collectors.toList()); + assertThat(allPartitions).doesNotHaveDuplicates(); + }); + } + } diff --git a/common/cluster-api/src/main/proto/queue.proto b/common/cluster-api/src/main/proto/queue.proto index 80f44e59be..78614911d4 100644 --- a/common/cluster-api/src/main/proto/queue.proto +++ b/common/cluster-api/src/main/proto/queue.proto @@ -28,6 +28,7 @@ message ServiceInfo { repeated string serviceTypes = 2; repeated string transports = 6; SystemInfoProto systemInfo = 10; + repeated string assignedTenantProfiles = 11; } message SystemInfoProto { diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/DefaultTbServiceInfoProvider.java b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/DefaultTbServiceInfoProvider.java index 3c85aef350..e9013ef345 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/DefaultTbServiceInfoProvider.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/DefaultTbServiceInfoProvider.java @@ -23,6 +23,7 @@ import org.springframework.context.ApplicationContext; import org.springframework.stereotype.Component; import org.thingsboard.server.common.data.StringUtils; import org.thingsboard.server.common.data.TbTransportService; +import org.thingsboard.server.common.data.util.CollectionsUtil; import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.gen.transport.TransportProtos.ServiceInfo; @@ -35,6 +36,8 @@ import java.util.Arrays; import java.util.Collection; import java.util.Collections; import java.util.List; +import java.util.Set; +import java.util.UUID; import java.util.stream.Collectors; import static org.thingsboard.common.util.SystemUtil.getCpuCount; @@ -57,6 +60,10 @@ public class DefaultTbServiceInfoProvider implements TbServiceInfoProvider { @Value("${service.type:monolith}") private String serviceType; + @Getter + @Value("${service.rule_engine.assigned_tenant_profiles:}") + private Set assignedTenantProfiles; + @Autowired private ApplicationContext applicationContext; @@ -111,7 +118,9 @@ public class DefaultTbServiceInfoProvider implements TbServiceInfoProvider { .setServiceId(serviceId) .addAllServiceTypes(serviceTypes.stream().map(ServiceType::name).collect(Collectors.toList())) .setSystemInfo(getCurrentSystemInfoProto()); - + if (CollectionsUtil.isNotEmpty(assignedTenantProfiles)) { + builder.addAllAssignedTenantProfiles(assignedTenantProfiles.stream().map(UUID::toString).collect(Collectors.toList())); + } return serviceInfo = builder.build(); } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/HashPartitionService.java b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/HashPartitionService.java index a8954b2b33..4c1894eae1 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/HashPartitionService.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/HashPartitionService.java @@ -24,6 +24,7 @@ import org.springframework.stereotype.Service; import org.thingsboard.server.common.data.exception.TenantNotFoundException; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.id.TenantProfileId; import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; import org.thingsboard.server.gen.transport.TransportProtos; @@ -36,6 +37,7 @@ import org.thingsboard.server.queue.util.AfterStartUp; import javax.annotation.PostConstruct; import java.nio.charset.StandardCharsets; import java.util.ArrayList; +import java.util.Collection; import java.util.Collections; import java.util.Comparator; import java.util.HashMap; @@ -70,15 +72,16 @@ public class HashPartitionService implements PartitionService { private final TenantRoutingInfoService tenantRoutingInfoService; private final QueueRoutingInfoService queueRoutingInfoService; - private volatile ConcurrentMap> myPartitions = new ConcurrentHashMap<>(); + protected volatile ConcurrentMap> myPartitions = new ConcurrentHashMap<>(); private final ConcurrentMap partitionTopicsMap = new ConcurrentHashMap<>(); private final ConcurrentMap partitionSizesMap = new ConcurrentHashMap<>(); private final ConcurrentMap tenantRoutingInfoMap = new ConcurrentHashMap<>(); - private Map> tbTransportServicesByType = new HashMap<>(); private List currentOtherServices; + private final Map> tbTransportServicesByType = new HashMap<>(); + private final Map> responsibleServices = new HashMap<>(); private HashFunction hashFunction; @@ -215,14 +218,12 @@ public class HashPartitionService implements PartitionService { } private TopicPartitionInfo resolve(QueueKey queueKey, EntityId entityId) { - int hash = hashFunction.newHasher() - .putLong(entityId.getId().getMostSignificantBits()) - .putLong(entityId.getId().getLeastSignificantBits()).hash().asInt(); - Integer partitionSize = partitionSizesMap.get(queueKey); if (partitionSize == null) { throw new IllegalStateException("Partitions info for queue " + queueKey + " is missing"); } + + int hash = hash(entityId.getId()); int partition = Math.abs(hash % partitionSize); return buildTopicPartitionInfo(queueKey, partition); @@ -231,6 +232,7 @@ public class HashPartitionService implements PartitionService { @Override public synchronized void recalculatePartitions(ServiceInfo currentService, List otherServices) { tbTransportServicesByType.clear(); + responsibleServices.clear(); logServiceInfo(currentService); otherServices.forEach(this::logServiceInfo); @@ -240,6 +242,7 @@ public class HashPartitionService implements PartitionService { addNode(queueServicesMap, other); } queueServicesMap.values().forEach(list -> list.sort(Comparator.comparing(ServiceInfo::getServiceId))); + responsibleServices.values().forEach(list -> list.sort(Comparator.comparing(ServiceInfo::getServiceId))); final ConcurrentMap> newPartitions = new ConcurrentHashMap<>(); partitionSizesMap.forEach((queueKey, size) -> { @@ -287,6 +290,9 @@ public class HashPartitionService implements PartitionService { changes.addAll(newMap.keySet()); if (!changes.isEmpty()) { applicationEventPublisher.publishEvent(new ClusterTopologyChangeEvent(this, changes)); + responsibleServices.forEach((profileId, serviceInfos) -> { + log.info("Servers responsible for tenant profile {}: {}", profileId, toServiceIds(serviceInfos)); + }); } } @@ -324,9 +330,7 @@ public class HashPartitionService implements PartitionService { @Override public int resolvePartitionIndex(UUID entityId, int partitions) { - int hash = hashFunction.newHasher() - .putLong(entityId.getMostSignificantBits()) - .putLong(entityId.getLeastSignificantBits()).hash().asInt(); + int hash = hash(entityId); return Math.abs(hash % partitions); } @@ -408,6 +412,19 @@ public class HashPartitionService implements PartitionService { queueServiceList.computeIfAbsent(key, k -> new ArrayList<>()).add(instance); } }); + + if (instance.getAssignedTenantProfilesCount() > 0) { + for (String profileIdStr : instance.getAssignedTenantProfilesList()) { + TenantProfileId profileId; + try { + profileId = new TenantProfileId(UUID.fromString(profileIdStr)); + } catch (IllegalArgumentException e) { + log.warn("Failed to parse '{}' as tenant profile id", profileIdStr); + continue; + } + responsibleServices.computeIfAbsent(profileId, k -> new ArrayList<>()).add(instance); + } + } } else if (ServiceType.TB_CORE.equals(serviceType) || ServiceType.TB_VC_EXECUTOR.equals(serviceType)) { queueServiceList.computeIfAbsent(new QueueKey(serviceType), key -> new ArrayList<>()).add(instance); } @@ -423,18 +440,51 @@ public class HashPartitionService implements PartitionService { return null; } - if (!ServiceType.TB_RULE_ENGINE.equals(queueKey.getType()) || TenantId.SYS_TENANT_ID.equals(queueKey.getTenantId())) { - return servers.get(partition % servers.size()); - } else { - int hash = hashFunction.newHasher().putLong(queueKey.getTenantId().getId().getMostSignificantBits()) - .putLong(queueKey.getTenantId().getId().getLeastSignificantBits()) + TenantId tenantId = queueKey.getTenantId(); + if (queueKey.getType() == ServiceType.TB_RULE_ENGINE) { + if (!responsibleServices.isEmpty()) { // if there are any dedicated servers + TenantProfileId profileId; + if (tenantId != null && !tenantId.isSysTenantId()) { + TenantRoutingInfo routingInfo = tenantRoutingInfoService.getRoutingInfo(tenantId); + profileId = routingInfo.getProfileId(); + } else { + profileId = null; + } + + List responsible = responsibleServices.get(profileId); + if (responsible == null) { + // if there are no dedicated servers for this tenant profile, or for system queues, + // using the servers that are not responsible for any profile + responsible = servers.stream() + .filter(serviceInfo -> serviceInfo.getAssignedTenantProfilesCount() == 0) + .sorted(Comparator.comparing(ServiceInfo::getServiceId)) + .collect(Collectors.toList()); + if (profileId != null) { + log.debug("Using servers {} for profile {}", toServiceIds(responsible), profileId); + } + responsibleServices.put(profileId, responsible); + } + servers = responsible; + } + + int hash = hashFunction.newHasher() + .putLong(tenantId.getId().getMostSignificantBits()) + .putLong(tenantId.getId().getLeastSignificantBits()) .putString(queueKey.getQueueName(), StandardCharsets.UTF_8) .hash().asInt(); - return servers.get(Math.abs((hash + partition) % servers.size())); + } else { + return servers.get(partition % servers.size()); } } + private int hash(UUID key) { + return hashFunction.newHasher() + .putLong(key.getMostSignificantBits()) + .putLong(key.getLeastSignificantBits()) + .hash().asInt(); + } + public static HashFunction forName(String name) { switch (name) { case "murmur3_32": @@ -448,4 +498,8 @@ public class HashPartitionService implements PartitionService { } } + private List toServiceIds(Collection serviceInfos) { + return serviceInfos.stream().map(ServiceInfo::getServiceId).collect(Collectors.toList()); + } + } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/TbServiceInfoProvider.java b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/TbServiceInfoProvider.java index e49cbbcfd9..9c7d1630ec 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/TbServiceInfoProvider.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/TbServiceInfoProvider.java @@ -18,6 +18,9 @@ package org.thingsboard.server.queue.discovery; import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.gen.transport.TransportProtos.ServiceInfo; +import java.util.Set; +import java.util.UUID; + public interface TbServiceInfoProvider { String getServiceId(); @@ -30,4 +33,6 @@ public interface TbServiceInfoProvider { ServiceInfo generateNewServiceInfoWithCurrentSystemInfo(); + Set getAssignedTenantProfiles(); + } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/TenantRoutingInfo.java b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/TenantRoutingInfo.java index c1c0b49dab..8dee68da49 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/TenantRoutingInfo.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/TenantRoutingInfo.java @@ -17,9 +17,11 @@ package org.thingsboard.server.queue.discovery; import lombok.Data; import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.id.TenantProfileId; @Data public class TenantRoutingInfo { private final TenantId tenantId; + private final TenantProfileId profileId; private final boolean isolatedTbRuleEngine; } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/TenantRoutingInfoService.java b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/TenantRoutingInfoService.java index 8dd3ff95e7..e4c0ac8250 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/TenantRoutingInfoService.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/TenantRoutingInfoService.java @@ -20,4 +20,5 @@ import org.thingsboard.server.common.data.id.TenantId; public interface TenantRoutingInfoService { TenantRoutingInfo getRoutingInfo(TenantId tenantId); + } diff --git a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/TransportTenantRoutingInfoService.java b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/TransportTenantRoutingInfoService.java index c9f126b808..e1192391d5 100644 --- a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/TransportTenantRoutingInfoService.java +++ b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/TransportTenantRoutingInfoService.java @@ -29,7 +29,7 @@ import org.thingsboard.server.queue.discovery.TenantRoutingInfoService; @ConditionalOnExpression("'${service.type:null}'=='tb-transport'") public class TransportTenantRoutingInfoService implements TenantRoutingInfoService { - private TransportTenantProfileCache tenantProfileCache; + private final TransportTenantProfileCache tenantProfileCache; public TransportTenantRoutingInfoService(TransportTenantProfileCache tenantProfileCache) { this.tenantProfileCache = tenantProfileCache; @@ -38,7 +38,7 @@ public class TransportTenantRoutingInfoService implements TenantRoutingInfoServi @Override public TenantRoutingInfo getRoutingInfo(TenantId tenantId) { TenantProfile profile = tenantProfileCache.get(tenantId); - return new TenantRoutingInfo(tenantId, profile.isIsolatedTbRuleEngine()); + return new TenantRoutingInfo(tenantId, profile.getId(), profile.isIsolatedTbRuleEngine()); } } diff --git a/msa/vc-executor/src/main/java/org/thingsboard/server/vc/service/VersionControlTenantRoutingInfoService.java b/msa/vc-executor/src/main/java/org/thingsboard/server/vc/service/VersionControlTenantRoutingInfoService.java index fb33ff9931..b343a4791f 100644 --- a/msa/vc-executor/src/main/java/org/thingsboard/server/vc/service/VersionControlTenantRoutingInfoService.java +++ b/msa/vc-executor/src/main/java/org/thingsboard/server/vc/service/VersionControlTenantRoutingInfoService.java @@ -25,6 +25,6 @@ public class VersionControlTenantRoutingInfoService implements TenantRoutingInfo @Override public TenantRoutingInfo getRoutingInfo(TenantId tenantId) { //This dummy implementation is ok since Version Control service does not produce any rule engine messages. - return new TenantRoutingInfo(tenantId, false); + return new TenantRoutingInfo(tenantId, null, false); } } diff --git a/ui-ngx/src/assets/locale/locale.constant-en_US.json b/ui-ngx/src/assets/locale/locale.constant-en_US.json index c1092f9dad..b449cf1ba5 100644 --- a/ui-ngx/src/assets/locale/locale.constant-en_US.json +++ b/ui-ngx/src/assets/locale/locale.constant-en_US.json @@ -3535,7 +3535,7 @@ "search": "Search tenants", "selected-tenants": "{ count, plural, =1 {1 tenant} other {# tenants} } selected", "isolated-tb-rule-engine": "Processing in isolated ThingsBoard Rule Engine container", - "isolated-tb-rule-engine-details": "Requires separate microservice(s) per isolated Tenant" + "isolated-tb-rule-engine-details": "Requires separate microservice(s) for the profile" }, "tenant-profile": { "tenant-profile": "Tenant profile",