From 27e54c3ae07d355299ec082967e72e338f59b953 Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Wed, 26 Mar 2025 16:36:03 +0200 Subject: [PATCH 01/47] fixed tenant cache computing when partition change event --- ...aultCalculatedFieldEntityProfileCache.java | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/cf/cache/DefaultCalculatedFieldEntityProfileCache.java b/application/src/main/java/org/thingsboard/server/service/cf/cache/DefaultCalculatedFieldEntityProfileCache.java index 2f5772ae50..8a0c67ce29 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/cache/DefaultCalculatedFieldEntityProfileCache.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/cache/DefaultCalculatedFieldEntityProfileCache.java @@ -24,14 +24,11 @@ import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; import org.thingsboard.server.queue.discovery.PartitionService; -import org.thingsboard.server.queue.discovery.QueueKey; import org.thingsboard.server.queue.discovery.TbApplicationEventListener; import org.thingsboard.server.queue.discovery.event.PartitionChangeEvent; import org.thingsboard.server.queue.util.TbRuleEngineComponent; import java.util.Collection; -import java.util.Collections; -import java.util.List; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; import java.util.stream.Collectors; @@ -46,15 +43,21 @@ public class DefaultCalculatedFieldEntityProfileCache extends TbApplicationEvent private static final Integer UNKNOWN = 0; private final ConcurrentMap tenantCache = new ConcurrentHashMap<>(); private final PartitionService partitionService; - private volatile List myPartitions = Collections.emptyList(); @Override protected void onTbApplicationEvent(PartitionChangeEvent event) { - myPartitions = event.getCfPartitions().stream() + var tenantPartitions = event.getCfPartitions().stream() .filter(TopicPartitionInfo::isMyPartition) - .map(tpi -> tpi.getPartition().orElse(UNKNOWN)).collect(Collectors.toList()); - //Naive approach that need to be improved. - tenantCache.values().forEach(cache -> cache.setMyPartitions(myPartitions)); + .filter(tpi -> tpi.getTenantId().isPresent()) + .collect(Collectors.groupingBy( + tpi -> tpi.getTenantId().get(), + Collectors.mapping(tpi -> tpi.getPartition().orElse(UNKNOWN), Collectors.toList()) + )); + + tenantPartitions.forEach((tenantId, partitions) -> { + var cache = tenantCache.computeIfAbsent(tenantId, id -> new TenantEntityProfileCache()); + cache.setMyPartitions(partitions); + }); } @Override From 2d95b7ad5827738fdcdf299c85a1b0d7bf1bdc90 Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Wed, 26 Mar 2025 17:27:58 +0200 Subject: [PATCH 02/47] added system partitons handling --- ...aultCalculatedFieldEntityProfileCache.java | 32 +++++++++++++++---- 1 file changed, 25 insertions(+), 7 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/cf/cache/DefaultCalculatedFieldEntityProfileCache.java b/application/src/main/java/org/thingsboard/server/service/cf/cache/DefaultCalculatedFieldEntityProfileCache.java index 8a0c67ce29..080907587e 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/cache/DefaultCalculatedFieldEntityProfileCache.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/cache/DefaultCalculatedFieldEntityProfileCache.java @@ -28,10 +28,14 @@ import org.thingsboard.server.queue.discovery.TbApplicationEventListener; import org.thingsboard.server.queue.discovery.event.PartitionChangeEvent; import org.thingsboard.server.queue.util.TbRuleEngineComponent; +import java.util.ArrayList; import java.util.Collection; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; -import java.util.stream.Collectors; @TbRuleEngineComponent @Service @@ -46,18 +50,32 @@ public class DefaultCalculatedFieldEntityProfileCache extends TbApplicationEvent @Override protected void onTbApplicationEvent(PartitionChangeEvent event) { - var tenantPartitions = event.getCfPartitions().stream() + Map> tenantPartitions = new HashMap<>(); + List systemPartitions = new ArrayList<>(); + + event.getCfPartitions().stream() .filter(TopicPartitionInfo::isMyPartition) - .filter(tpi -> tpi.getTenantId().isPresent()) - .collect(Collectors.groupingBy( - tpi -> tpi.getTenantId().get(), - Collectors.mapping(tpi -> tpi.getPartition().orElse(UNKNOWN), Collectors.toList()) - )); + .forEach(tpi -> { + Integer partition = tpi.getPartition().orElse(UNKNOWN); + Optional tenantIdOpt = tpi.getTenantId(); + if (tenantIdOpt.isPresent()) { + tenantPartitions.computeIfAbsent(tenantIdOpt.get(), id -> new ArrayList<>()).add(partition); + } else { + systemPartitions.add(partition); + } + }); tenantPartitions.forEach((tenantId, partitions) -> { var cache = tenantCache.computeIfAbsent(tenantId, id -> new TenantEntityProfileCache()); cache.setMyPartitions(partitions); }); + + tenantCache.keySet().stream() + .filter(tenantId -> !tenantPartitions.containsKey(tenantId)) + .forEach(tenantId -> { + var cache = tenantCache.get(tenantId); + cache.setMyPartitions(systemPartitions); + }); } @Override From b715ce2e4366c63b466bfcd954a77f890f06fd3f Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Thu, 27 Mar 2025 13:08:47 +0200 Subject: [PATCH 03/47] removed myEntityIds map and handled tenant and asset/device profile deletion --- ...alculatedFieldManagerMessageProcessor.java | 41 +++++++++----- .../CalculatedFieldEntityProfileCache.java | 6 +- ...aultCalculatedFieldEntityProfileCache.java | 56 ++++++------------- .../cf/cache/TenantEntityProfileCache.java | 45 ++++----------- ...faultTbCalculatedFieldConsumerService.java | 15 ++++- 5 files changed, 73 insertions(+), 90 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java index e109e83e4d..27d3ce315a 100644 --- a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java +++ b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java @@ -322,11 +322,15 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware EntityId entityId = cfCtx.getEntityId(); EntityType entityType = cfCtx.getEntityId().getEntityType(); if (isProfileEntity(entityType)) { - var entityIds = cfEntityCache.getMyEntityIdsByProfileId(tenantId, entityId); + var entityIds = cfEntityCache.getEntityIdsByProfileId(tenantId, entityId); if (!entityIds.isEmpty()) { //TODO: no need to do this if we cache all created actors and know which one belong to us; var multiCallback = new MultipleTbCallback(entityIds.size(), callback); - entityIds.forEach(id -> deleteCfForEntity(id, cfId, multiCallback)); + entityIds.forEach(id -> { + if (isMyPartition(id, multiCallback)) { + deleteCfForEntity(id, cfId, multiCallback); + } + }); } else { callback.onSuccess(); } @@ -366,10 +370,11 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware EntityId sourceEntityId = msg.getEntityId(); log.debug("Received linked telemetry msg from entity [{}]", sourceEntityId); var proto = msg.getProto(); + var callback = msg.getCallback(); var linksList = proto.getLinksList(); if (linksList.isEmpty()) { log.debug("[{}] No CF links to process new telemetry.", msg.getTenantId()); - msg.getCallback().onSuccess(); + callback.onSuccess(); } for (var linkProto : linksList) { var link = fromProto(linkProto); @@ -378,21 +383,25 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware var cf = calculatedFields.get(link.cfId()); if (EntityType.DEVICE_PROFILE.equals(targetEntityType) || EntityType.ASSET_PROFILE.equals(targetEntityType)) { // iterate over all entities that belong to profile and push the message for corresponding CF - var entityIds = cfEntityCache.getMyEntityIdsByProfileId(tenantId, targetEntityId); + var entityIds = cfEntityCache.getEntityIdsByProfileId(tenantId, targetEntityId); if (!entityIds.isEmpty()) { - MultipleTbCallback callback = new MultipleTbCallback(entityIds.size(), msg.getCallback()); - var newMsg = new EntityCalculatedFieldLinkedTelemetryMsg(tenantId, sourceEntityId, proto.getMsg(), cf, callback); + MultipleTbCallback multipleCallback = new MultipleTbCallback(entityIds.size(), callback); + var newMsg = new EntityCalculatedFieldLinkedTelemetryMsg(tenantId, sourceEntityId, proto.getMsg(), cf, multipleCallback); entityIds.forEach(entityId -> { - log.debug("Pushing linked telemetry msg to specific actor [{}]", entityId); - getOrCreateActor(entityId).tell(newMsg); + if (isMyPartition(entityId, multipleCallback)) { + log.debug("Pushing linked telemetry msg to specific actor [{}]", entityId); + getOrCreateActor(entityId).tell(newMsg); + } }); } else { - msg.getCallback().onSuccess(); + callback.onSuccess(); } } else { - log.debug("Pushing linked telemetry msg to specific actor [{}]", targetEntityId); - var newMsg = new EntityCalculatedFieldLinkedTelemetryMsg(tenantId, sourceEntityId, proto.getMsg(), cf, msg.getCallback()); - getOrCreateActor(targetEntityId).tell(newMsg); + if (isMyPartition(targetEntityId, callback)) { + log.debug("Pushing linked telemetry msg to specific actor [{}]", targetEntityId); + var newMsg = new EntityCalculatedFieldLinkedTelemetryMsg(tenantId, sourceEntityId, proto.getMsg(), cf, callback); + getOrCreateActor(targetEntityId).tell(newMsg); + } } } } @@ -436,10 +445,14 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware EntityId entityId = cfCtx.getEntityId(); EntityType entityType = cfCtx.getEntityId().getEntityType(); if (isProfileEntity(entityType)) { - var entityIds = cfEntityCache.getMyEntityIdsByProfileId(tenantId, entityId); + var entityIds = cfEntityCache.getEntityIdsByProfileId(tenantId, entityId); if (!entityIds.isEmpty()) { var multiCallback = new MultipleTbCallback(entityIds.size(), callback); - entityIds.forEach(id -> initCfForEntity(id, cfCtx, forceStateReinit, multiCallback)); + entityIds.forEach(id -> { + if (isMyPartition(id, multiCallback)) { + initCfForEntity(id, cfCtx, forceStateReinit, multiCallback); + } + }); } else { callback.onSuccess(); } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/cache/CalculatedFieldEntityProfileCache.java b/application/src/main/java/org/thingsboard/server/service/cf/cache/CalculatedFieldEntityProfileCache.java index bb5ef91974..df2dc88f6b 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/cache/CalculatedFieldEntityProfileCache.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/cache/CalculatedFieldEntityProfileCache.java @@ -30,7 +30,11 @@ public interface CalculatedFieldEntityProfileCache extends ApplicationListener

getMyEntityIdsByProfileId(TenantId tenantId, EntityId profileId); + void evictProfile(TenantId tenantId, EntityId profileId); + + void removeTenant(TenantId tenantId); + + Collection getEntityIdsByProfileId(TenantId tenantId, EntityId profileId); int getEntityIdPartition(TenantId tenantId, EntityId entityId); } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/cache/DefaultCalculatedFieldEntityProfileCache.java b/application/src/main/java/org/thingsboard/server/service/cf/cache/DefaultCalculatedFieldEntityProfileCache.java index 080907587e..8059b3a825 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/cache/DefaultCalculatedFieldEntityProfileCache.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/cache/DefaultCalculatedFieldEntityProfileCache.java @@ -22,17 +22,12 @@ import org.thingsboard.server.common.data.DataConstants; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.msg.queue.ServiceType; -import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; import org.thingsboard.server.queue.discovery.PartitionService; import org.thingsboard.server.queue.discovery.TbApplicationEventListener; import org.thingsboard.server.queue.discovery.event.PartitionChangeEvent; import org.thingsboard.server.queue.util.TbRuleEngineComponent; -import java.util.ArrayList; import java.util.Collection; -import java.util.HashMap; -import java.util.List; -import java.util.Map; import java.util.Optional; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; @@ -50,50 +45,23 @@ public class DefaultCalculatedFieldEntityProfileCache extends TbApplicationEvent @Override protected void onTbApplicationEvent(PartitionChangeEvent event) { - Map> tenantPartitions = new HashMap<>(); - List systemPartitions = new ArrayList<>(); - - event.getCfPartitions().stream() - .filter(TopicPartitionInfo::isMyPartition) - .forEach(tpi -> { - Integer partition = tpi.getPartition().orElse(UNKNOWN); - Optional tenantIdOpt = tpi.getTenantId(); - if (tenantIdOpt.isPresent()) { - tenantPartitions.computeIfAbsent(tenantIdOpt.get(), id -> new ArrayList<>()).add(partition); - } else { - systemPartitions.add(partition); - } - }); - - tenantPartitions.forEach((tenantId, partitions) -> { - var cache = tenantCache.computeIfAbsent(tenantId, id -> new TenantEntityProfileCache()); - cache.setMyPartitions(partitions); + event.getCfPartitions().forEach(tpi -> { + Optional tenantIdOpt = tpi.getTenantId(); + tenantIdOpt.ifPresent(tenantId -> tenantCache.computeIfAbsent(tenantId, id -> new TenantEntityProfileCache())); }); - - tenantCache.keySet().stream() - .filter(tenantId -> !tenantPartitions.containsKey(tenantId)) - .forEach(tenantId -> { - var cache = tenantCache.get(tenantId); - cache.setMyPartitions(systemPartitions); - }); } @Override public void add(TenantId tenantId, EntityId profileId, EntityId entityId) { - var tpi = partitionService.resolve(ServiceType.TB_RULE_ENGINE, DataConstants.CF_QUEUE_NAME, tenantId, entityId); - var partition = tpi.getPartition().orElse(UNKNOWN); - tenantCache.computeIfAbsent(tenantId, id -> new TenantEntityProfileCache()) - .add(profileId, entityId, partition, tpi.isMyPartition()); + tenantCache.computeIfAbsent(tenantId, id -> new TenantEntityProfileCache()).add(profileId, entityId); } @Override public void update(TenantId tenantId, EntityId oldProfileId, EntityId newProfileId, EntityId entityId) { - var tpi = partitionService.resolve(ServiceType.TB_RULE_ENGINE, DataConstants.CF_QUEUE_NAME, tenantId, entityId); - var partition = tpi.getPartition().orElse(UNKNOWN); var cache = tenantCache.computeIfAbsent(tenantId, id -> new TenantEntityProfileCache()); //TODO: make this method atomic; cache.remove(oldProfileId, entityId); - cache.add(newProfileId, entityId, partition, tpi.isMyPartition()); + cache.add(newProfileId, entityId); } @Override @@ -103,8 +71,18 @@ public class DefaultCalculatedFieldEntityProfileCache extends TbApplicationEvent } @Override - public Collection getMyEntityIdsByProfileId(TenantId tenantId, EntityId profileId) { - return tenantCache.computeIfAbsent(tenantId, id -> new TenantEntityProfileCache()).getMyEntityIdsByProfileId(profileId); + public void evictProfile(TenantId tenantId, EntityId profileId) { + tenantCache.computeIfAbsent(tenantId, id -> new TenantEntityProfileCache()).removeProfileId(profileId); + } + + @Override + public void removeTenant(TenantId tenantId) { + tenantCache.remove(tenantId); + } + + @Override + public Collection getEntityIdsByProfileId(TenantId tenantId, EntityId profileId) { + return tenantCache.computeIfAbsent(tenantId, id -> new TenantEntityProfileCache()).getEntityIdsByProfileId(profileId); } @Override diff --git a/application/src/main/java/org/thingsboard/server/service/cf/cache/TenantEntityProfileCache.java b/application/src/main/java/org/thingsboard/server/service/cf/cache/TenantEntityProfileCache.java index 1a17b9b8be..27c2bdd6d5 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/cache/TenantEntityProfileCache.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/cache/TenantEntityProfileCache.java @@ -32,31 +32,13 @@ import java.util.concurrent.locks.ReentrantReadWriteLock; public class TenantEntityProfileCache { private final ReadWriteLock lock = new ReentrantReadWriteLock(); - private final Map>> allEntities = new HashMap<>(); - private final Map> myEntities = new HashMap<>(); - - public void setMyPartitions(List myPartitions) { - lock.writeLock().lock(); - try { - myEntities.clear(); - myPartitions.forEach(partitionId -> { - var map = allEntities.get(partitionId); - if (map != null) { - map.forEach((profileId, entityIds) -> myEntities.computeIfAbsent(profileId, k -> new HashSet<>()).addAll(entityIds)); - } - }); - } finally { - lock.writeLock().unlock(); - } - } + private final Map> allEntities = new HashMap<>(); public void removeProfileId(EntityId profileId) { lock.writeLock().lock(); try { // Remove from allEntities - allEntities.values().forEach(map -> map.remove(profileId)); - // Remove from myEntities - myEntities.remove(profileId); + allEntities.remove(profileId); } finally { lock.writeLock().unlock(); } @@ -66,9 +48,7 @@ public class TenantEntityProfileCache { lock.writeLock().lock(); try { // Remove from allEntities - allEntities.values().forEach(map -> map.values().forEach(set -> set.remove(entityId))); - // Remove from myEntities - myEntities.values().forEach(set -> set.remove(entityId)); + allEntities.values().forEach(set -> set.remove(entityId)); } finally { lock.writeLock().unlock(); } @@ -78,33 +58,28 @@ public class TenantEntityProfileCache { lock.writeLock().lock(); try { // Remove from allEntities - allEntities.values().forEach(map -> removeSafely(map, profileId, entityId)); - // Remove from myEntities - removeSafely(myEntities, profileId, entityId); + removeSafely(allEntities, profileId, entityId); } finally { lock.writeLock().unlock(); } } - public void add(EntityId profileId, EntityId entityId, Integer partition, boolean mine) { + public void add(EntityId profileId, EntityId entityId) { lock.writeLock().lock(); try { - if(EntityType.DEVICE.equals(profileId.getEntityType())){ - throw new RuntimeException("WTF?"); - } - if (mine) { - myEntities.computeIfAbsent(profileId, k -> new HashSet<>()).add(entityId); + if (EntityType.DEVICE.equals(profileId.getEntityType()) || EntityType.ASSET.equals(profileId.getEntityType())) { + throw new RuntimeException("Entity type '" + profileId.getEntityType() + "' is not a profileId."); } - allEntities.computeIfAbsent(partition, k -> new HashMap<>()).computeIfAbsent(profileId, p -> new HashSet<>()).add(entityId); + allEntities.computeIfAbsent(profileId, k -> new HashSet<>()).add(entityId); } finally { lock.writeLock().unlock(); } } - public Collection getMyEntityIdsByProfileId(EntityId profileId) { + public Collection getEntityIdsByProfileId(EntityId profileId) { lock.readLock().lock(); try { - var entities = myEntities.getOrDefault(profileId, Collections.emptySet()); + var entities = allEntities.getOrDefault(profileId, Collections.emptySet()); List result = new ArrayList<>(entities.size()); result.addAll(entities); return result; diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCalculatedFieldConsumerService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCalculatedFieldConsumerService.java index bce1992932..b543e860f3 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCalculatedFieldConsumerService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCalculatedFieldConsumerService.java @@ -52,6 +52,7 @@ import org.thingsboard.server.queue.util.TbRuleEngineComponent; import org.thingsboard.server.service.apiusage.TbApiUsageStateService; import org.thingsboard.server.service.cf.CalculatedFieldCache; import org.thingsboard.server.service.cf.CalculatedFieldStateService; +import org.thingsboard.server.service.cf.cache.CalculatedFieldEntityProfileCache; import org.thingsboard.server.service.profile.TbAssetProfileCache; import org.thingsboard.server.service.profile.TbDeviceProfileCache; import org.thingsboard.server.service.queue.processing.AbstractConsumerPartitionedService; @@ -80,6 +81,7 @@ public class DefaultTbCalculatedFieldConsumerService extends AbstractConsumerPar private final TbRuleEngineQueueFactory queueFactory; private final CalculatedFieldStateService stateService; + private final CalculatedFieldEntityProfileCache entityProfileCache; public DefaultTbCalculatedFieldConsumerService(TbRuleEngineQueueFactory tbQueueFactory, ActorSystemContext actorContext, @@ -91,11 +93,13 @@ public class DefaultTbCalculatedFieldConsumerService extends AbstractConsumerPar ApplicationEventPublisher eventPublisher, JwtSettingsService jwtSettingsService, CalculatedFieldCache calculatedFieldCache, - CalculatedFieldStateService stateService) { + CalculatedFieldStateService stateService, + CalculatedFieldEntityProfileCache entityProfileCache) { super(actorContext, tenantProfileCache, deviceProfileCache, assetProfileCache, calculatedFieldCache, apiUsageStateService, partitionService, eventPublisher, jwtSettingsService); this.queueFactory = tbQueueFactory; this.stateService = stateService; + this.entityProfileCache = entityProfileCache; } @Override @@ -227,6 +231,7 @@ public class DefaultTbCalculatedFieldConsumerService extends AbstractConsumerPar public void handleComponentLifecycleEvent(ComponentLifecycleMsg event) { if (event.getEntityId().getEntityType() == EntityType.TENANT) { if (event.getEvent() == ComponentLifecycleEvent.DELETED) { + entityProfileCache.removeTenant(event.getTenantId()); Set partitions = stateService.getPartitions(); if (CollectionUtils.isEmpty(partitions)) { return; @@ -235,6 +240,14 @@ public class DefaultTbCalculatedFieldConsumerService extends AbstractConsumerPar .filter(tpi -> tpi.getTenantId().isPresent() && tpi.getTenantId().get().equals(event.getTenantId())) .collect(Collectors.toSet())); } + } else if (event.getEntityId().getEntityType() == EntityType.ASSET_PROFILE) { + if (event.getEvent() == ComponentLifecycleEvent.DELETED) { + entityProfileCache.evictProfile(event.getTenantId(), event.getEntityId()); + } + } else if (event.getEntityId().getEntityType() == EntityType.DEVICE_PROFILE) { + if (event.getEvent() == ComponentLifecycleEvent.DELETED) { + entityProfileCache.evictProfile(event.getTenantId(), event.getEntityId()); + } } } From fbcfe9281d715130525d6eeaf306918bf7735af7 Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Fri, 28 Mar 2025 17:02:41 +0200 Subject: [PATCH 04/47] wip cf consumer groups --- .../AbstractCalculatedFieldStateService.java | 18 +++---- .../cf/CalculatedFieldStateService.java | 6 +-- .../KafkaCalculatedFieldStateService.java | 6 +-- .../RocksDBCalculatedFieldStateService.java | 8 ++-- ...faultTbCalculatedFieldConsumerService.java | 48 ++++++++++++++----- .../InMemoryMonolithQueueFactory.java | 2 +- .../provider/KafkaMonolithQueueFactory.java | 15 ++++-- .../KafkaTbRuleEngineQueueFactory.java | 14 ++++-- .../provider/TbRuleEngineQueueFactory.java | 2 +- 9 files changed, 80 insertions(+), 39 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/cf/AbstractCalculatedFieldStateService.java b/application/src/main/java/org/thingsboard/server/service/cf/AbstractCalculatedFieldStateService.java index 91c08ab6e0..1347705a23 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/AbstractCalculatedFieldStateService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/AbstractCalculatedFieldStateService.java @@ -30,7 +30,9 @@ import org.thingsboard.server.service.cf.ctx.CalculatedFieldEntityCtxId; import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldState; import java.util.Collection; +import java.util.Map; import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; import java.util.stream.Collectors; import static org.thingsboard.server.utils.CalculatedFieldUtils.fromProto; @@ -41,7 +43,7 @@ public abstract class AbstractCalculatedFieldStateService implements CalculatedF @Autowired private ActorSystemContext actorSystemContext; - protected QueueStateService, TbProtoQueueMsg> stateService; + protected Map, TbProtoQueueMsg>> stateServices = new ConcurrentHashMap<>(); @Override public final void persistState(CalculatedFieldEntityCtxId stateId, CalculatedFieldState state, TbCallback callback) { @@ -72,22 +74,22 @@ public abstract class AbstractCalculatedFieldStateService implements CalculatedF @Override public void restore(QueueKey queueKey, Set partitions) { - stateService.update(queueKey, partitions); + stateServices.get(queueKey).update(queueKey, partitions); } @Override - public void delete(Set partitions) { - stateService.delete(partitions); + public void delete(QueueKey queueKey, Set partitions) { + stateServices.get(queueKey).delete(partitions); } @Override - public Set getPartitions() { - return stateService.getPartitions().values().stream().flatMap(Collection::stream).collect(Collectors.toSet()); + public Set getPartitions(QueueKey queueKey) { + return stateServices.get(queueKey).getPartitions().values().stream().flatMap(Collection::stream).collect(Collectors.toSet()); } @Override - public void stop() { - stateService.stop(); + public void stop(QueueKey queueKey) { + stateServices.get(queueKey).stop(); } } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldStateService.java b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldStateService.java index d0b34f18e8..a8b0ccf30e 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldStateService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldStateService.java @@ -37,10 +37,10 @@ public interface CalculatedFieldStateService { void restore(QueueKey queueKey, Set partitions); - void delete(Set partitions); + void delete(QueueKey queueKey, Set partitions); - Set getPartitions(); + Set getPartitions(QueueKey queueKey); - void stop(); + void stop(QueueKey queueKey); } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/KafkaCalculatedFieldStateService.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/KafkaCalculatedFieldStateService.java index 90d4056afc..9d59cbce95 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/KafkaCalculatedFieldStateService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/KafkaCalculatedFieldStateService.java @@ -97,7 +97,7 @@ public class KafkaCalculatedFieldStateService extends AbstractCalculatedFieldSta .scheduler(eventConsumer.getScheduler()) .taskExecutor(eventConsumer.getTaskExecutor()) .build(); - super.stateService = new KafkaQueueStateService<>(eventConsumer, stateConsumer); + super.stateServices.put(queueKey, new KafkaQueueStateService<>(eventConsumer, stateConsumer)); this.stateProducer = (TbKafkaProducerTemplate>) queueFactory.createCalculatedFieldStateProducer(); } @@ -145,8 +145,8 @@ public class KafkaCalculatedFieldStateService extends AbstractCalculatedFieldSta } @Override - public void stop() { - super.stop(); + public void stop(QueueKey queueKey) { + super.stop(queueKey); stateProducer.stop(); } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/RocksDBCalculatedFieldStateService.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/RocksDBCalculatedFieldStateService.java index 0eaa506dfd..889d8fcc01 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/RocksDBCalculatedFieldStateService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/RocksDBCalculatedFieldStateService.java @@ -20,13 +20,15 @@ import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; import org.springframework.stereotype.Service; +import org.thingsboard.server.common.data.DataConstants; +import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.common.msg.queue.TbCallback; import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; -import org.thingsboard.server.queue.common.state.DefaultQueueStateService; import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldStateProto; import org.thingsboard.server.gen.transport.TransportProtos.ToCalculatedFieldMsg; import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.queue.common.consumer.PartitionedQueueConsumerManager; +import org.thingsboard.server.queue.common.state.DefaultQueueStateService; import org.thingsboard.server.queue.discovery.QueueKey; import org.thingsboard.server.service.cf.AbstractCalculatedFieldStateService; import org.thingsboard.server.service.cf.CfRocksDb; @@ -44,7 +46,7 @@ public class RocksDBCalculatedFieldStateService extends AbstractCalculatedFieldS @Override public void init(PartitionedQueueConsumerManager> eventConsumer) { - super.stateService = new DefaultQueueStateService<>(eventConsumer); + super.stateServices.put(new QueueKey(ServiceType.TB_RULE_ENGINE, DataConstants.CF_STATES_QUEUE_NAME), new DefaultQueueStateService<>(eventConsumer)); } @Override @@ -61,7 +63,7 @@ public class RocksDBCalculatedFieldStateService extends AbstractCalculatedFieldS @Override public void restore(QueueKey queueKey, Set partitions) { - if (stateService.getPartitions().isEmpty()) { + if (stateServices.get(queueKey).getPartitions().isEmpty()) { cfRocksDb.forEach((key, value) -> { try { processRestoredState(CalculatedFieldStateProto.parseFrom(value)); diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCalculatedFieldConsumerService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCalculatedFieldConsumerService.java index bce1992932..7149b58c57 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCalculatedFieldConsumerService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCalculatedFieldConsumerService.java @@ -30,12 +30,14 @@ import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.id.EntityIdFactory; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.plugin.ComponentLifecycleEvent; +import org.thingsboard.server.common.data.queue.Queue; import org.thingsboard.server.common.data.queue.QueueConfig; import org.thingsboard.server.common.msg.cf.CalculatedFieldPartitionChangeMsg; import org.thingsboard.server.common.msg.plugin.ComponentLifecycleMsg; import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.common.msg.queue.TbCallback; import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; +import org.thingsboard.server.dao.queue.QueueService; import org.thingsboard.server.dao.tenant.TbTenantProfileCache; import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldLinkedTelemetryMsgProto; import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldTelemetryMsgProto; @@ -79,7 +81,10 @@ public class DefaultTbCalculatedFieldConsumerService extends AbstractConsumerPar private long packProcessingTimeout; private final TbRuleEngineQueueFactory queueFactory; + private final QueueService queueService; + private final CalculatedFieldStateService stateService; + private final ConcurrentMap>> consumers = new ConcurrentHashMap<>(); public DefaultTbCalculatedFieldConsumerService(TbRuleEngineQueueFactory tbQueueFactory, ActorSystemContext actorContext, @@ -91,28 +96,40 @@ public class DefaultTbCalculatedFieldConsumerService extends AbstractConsumerPar ApplicationEventPublisher eventPublisher, JwtSettingsService jwtSettingsService, CalculatedFieldCache calculatedFieldCache, + QueueService queueService, CalculatedFieldStateService stateService) { super(actorContext, tenantProfileCache, deviceProfileCache, assetProfileCache, calculatedFieldCache, apiUsageStateService, partitionService, eventPublisher, jwtSettingsService); this.queueFactory = tbQueueFactory; + this.queueService = queueService; this.stateService = stateService; } @Override protected void doAfterStartUp() { - var queueKey = new QueueKey(ServiceType.TB_RULE_ENGINE, DataConstants.CF_QUEUE_NAME); - PartitionedQueueConsumerManager> eventConsumer = PartitionedQueueConsumerManager.>create() + List queues = queueService.findAllQueues(); + for (Queue configuration : queues) { + if (partitionService.isManagedByCurrentService(configuration.getTenantId())) { + stateService.init(createConsumer(configuration)); + } + } + } + + private PartitionedQueueConsumerManager> createConsumer(Queue queue) { + QueueKey queueKey = new QueueKey(ServiceType.TB_RULE_ENGINE, DataConstants.CF_QUEUE_NAME, queue.getTenantId()); + var eventConsumer = PartitionedQueueConsumerManager.>create() .queueKey(queueKey) .topic(partitionService.getTopic(queueKey)) .pollInterval(pollInterval) .msgPackProcessor(this::processMsgs) - .consumerCreator((config, partitionId) -> queueFactory.createToCalculatedFieldMsgConsumer()) + .consumerCreator((config, partitionId) -> queueFactory.createToCalculatedFieldMsgConsumer(queue, partitionId)) .queueAdmin(queueFactory.getCalculatedFieldQueueAdmin()) .consumerExecutor(consumersExecutor) .scheduler(scheduler) .taskExecutor(mgmtExecutor) .build(); - stateService.init(eventConsumer); + consumers.put(queueKey, eventConsumer); + return eventConsumer; } @PreDestroy @@ -227,13 +244,20 @@ public class DefaultTbCalculatedFieldConsumerService extends AbstractConsumerPar public void handleComponentLifecycleEvent(ComponentLifecycleMsg event) { if (event.getEntityId().getEntityType() == EntityType.TENANT) { if (event.getEvent() == ComponentLifecycleEvent.DELETED) { - Set partitions = stateService.getPartitions(); - if (CollectionUtils.isEmpty(partitions)) { - return; - } - stateService.delete(partitions.stream() - .filter(tpi -> tpi.getTenantId().isPresent() && tpi.getTenantId().get().equals(event.getTenantId())) - .collect(Collectors.toSet())); + consumers.keySet().removeIf(queueKey -> { + boolean toRemove = queueKey.getTenantId().equals(event.getTenantId()); + if (toRemove) { + Set partitions = stateService.getPartitions(queueKey); + if (!CollectionUtils.isEmpty(partitions)) { + stateService.delete(queueKey, partitions); + } + var consumer = consumers.remove(queueKey); + if (consumer != null) { + consumer.stop(); + } + } + return toRemove; + }); } } } @@ -258,7 +282,7 @@ public class DefaultTbCalculatedFieldConsumerService extends AbstractConsumerPar @Override protected void stopConsumers() { super.stopConsumers(); - stateService.stop(); // eventConsumer will be stopped by stateService + consumers.keySet().forEach(stateService::stop); // eventConsumer will be stopped by stateService } } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/InMemoryMonolithQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/InMemoryMonolithQueueFactory.java index bd8b4bd4f8..8e724b9145 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/InMemoryMonolithQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/InMemoryMonolithQueueFactory.java @@ -134,7 +134,7 @@ public class InMemoryMonolithQueueFactory implements TbCoreQueueFactory, TbRuleE } @Override - public TbQueueConsumer> createToCalculatedFieldMsgConsumer() { + public TbQueueConsumer> createToCalculatedFieldMsgConsumer(Queue configuration, Integer partitionId) { return new InMemoryTbQueueConsumer<>(storage, topicService.buildTopicName(calculatedFieldSettings.getEventTopic())); } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaMonolithQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaMonolithQueueFactory.java index c50344a23f..841a5b29d8 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaMonolithQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaMonolithQueueFactory.java @@ -104,7 +104,7 @@ public class KafkaMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEngi private final TbQueueAdmin housekeeperReprocessingAdmin; private final TbQueueAdmin edgeAdmin; private final TbQueueAdmin edgeEventAdmin; - private final TbQueueAdmin cfAdmin; + private final TbKafkaAdmin cfAdmin; private final TbQueueAdmin cfStateAdmin; private final TbQueueAdmin edqsEventsAdmin; private final TbKafkaAdmin edqsRequestsAdmin; @@ -514,15 +514,22 @@ public class KafkaMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEngi } @Override - public TbQueueConsumer> createToCalculatedFieldMsgConsumer() { + public TbQueueConsumer> createToCalculatedFieldMsgConsumer(Queue configuration, Integer partitionId) { + String queueName = configuration.getName(); + String groupId = topicService.buildConsumerGroupId("cf-", configuration.getTenantId(), queueName, partitionId); + + cfAdmin.syncOffsets(topicService.buildConsumerGroupId("cf-", configuration.getTenantId(), queueName, null), // the fat groupId + groupId, partitionId); + TbKafkaConsumerTemplate.TbKafkaConsumerTemplateBuilder> consumerBuilder = TbKafkaConsumerTemplate.builder(); consumerBuilder.settings(kafkaSettings); consumerBuilder.topic(topicService.buildTopicName(calculatedFieldSettings.getEventTopic())); - consumerBuilder.clientId("monolith-calculated-field-consumer-" + serviceInfoProvider.getServiceId() + "-" + consumerCount.incrementAndGet()); - consumerBuilder.groupId(topicService.buildTopicName("monolith-calculated-field-consumer")); + consumerBuilder.clientId("cf-" + queueName + "-consumer-" + serviceInfoProvider.getServiceId() + "-" + consumerCount.incrementAndGet()); + consumerBuilder.groupId(groupId); consumerBuilder.decoder(msg -> new TbProtoQueueMsg<>(msg.getKey(), ToCalculatedFieldMsg.parseFrom(msg.getData()), msg.getHeaders())); consumerBuilder.admin(cfAdmin); consumerBuilder.statsService(consumerStatsService); + return consumerBuilder.build(); } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbRuleEngineQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbRuleEngineQueueFactory.java index 2e342b0dd2..e0f7348d9f 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbRuleEngineQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbRuleEngineQueueFactory.java @@ -89,7 +89,7 @@ public class KafkaTbRuleEngineQueueFactory implements TbRuleEngineQueueFactory { private final TbQueueAdmin housekeeperAdmin; private final TbQueueAdmin edgeAdmin; private final TbQueueAdmin edgeEventAdmin; - private final TbQueueAdmin cfAdmin; + private final TbKafkaAdmin cfAdmin; private final TbQueueAdmin cfStateAdmin; private final TbQueueAdmin edqsEventsAdmin; private final AtomicLong consumerCount = new AtomicLong(); @@ -309,12 +309,18 @@ public class KafkaTbRuleEngineQueueFactory implements TbRuleEngineQueueFactory { } @Override - public TbQueueConsumer> createToCalculatedFieldMsgConsumer() { + public TbQueueConsumer> createToCalculatedFieldMsgConsumer(Queue configuration, Integer partitionId) { + String queueName = configuration.getName(); + String groupId = topicService.buildConsumerGroupId("cf-", configuration.getTenantId(), queueName, partitionId); + + cfAdmin.syncOffsets(topicService.buildConsumerGroupId("cf-", configuration.getTenantId(), queueName, null), // the fat groupId + groupId, partitionId); + TbKafkaConsumerTemplate.TbKafkaConsumerTemplateBuilder> consumerBuilder = TbKafkaConsumerTemplate.builder(); consumerBuilder.settings(kafkaSettings); consumerBuilder.topic(topicService.buildTopicName(calculatedFieldSettings.getEventTopic())); - consumerBuilder.clientId("tb-rule-engine-calculated-field-consumer-" + serviceInfoProvider.getServiceId() + "-" + consumerCount.incrementAndGet()); - consumerBuilder.groupId(topicService.buildTopicName("tb-rule-engine-calculated-field-consumer")); + consumerBuilder.clientId("cf-" + queueName + "-consumer-" + serviceInfoProvider.getServiceId() + "-" + consumerCount.incrementAndGet()); + consumerBuilder.groupId(groupId); consumerBuilder.decoder(msg -> new TbProtoQueueMsg<>(msg.getKey(), ToCalculatedFieldMsg.parseFrom(msg.getData()), msg.getHeaders())); consumerBuilder.admin(cfAdmin); consumerBuilder.statsService(consumerStatsService); diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbRuleEngineQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbRuleEngineQueueFactory.java index 329e3e346a..4a4b36b6ae 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbRuleEngineQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbRuleEngineQueueFactory.java @@ -121,7 +121,7 @@ public interface TbRuleEngineQueueFactory extends TbUsageStatsClientQueueFactory TbQueueRequestTemplate, TbProtoQueueMsg> createRemoteJsRequestTemplate(); - TbQueueConsumer> createToCalculatedFieldMsgConsumer(); + TbQueueConsumer> createToCalculatedFieldMsgConsumer(Queue configuration, Integer partitionId); TbQueueAdmin getCalculatedFieldQueueAdmin(); From f84e5da9df500084fb93c5b88bc2db6116eaf7f8 Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Mon, 31 Mar 2025 16:56:34 +0300 Subject: [PATCH 05/47] consumer groups updates --- ...CalculatedFieldEntityMessageProcessor.java | 10 +++---- ...alculatedFieldManagerMessageProcessor.java | 8 +++--- .../KafkaCalculatedFieldStateService.java | 4 +-- .../RocksDBCalculatedFieldStateService.java | 2 +- ...faultTbCalculatedFieldConsumerService.java | 28 +++++++++---------- .../InMemoryMonolithQueueFactory.java | 3 +- .../provider/KafkaMonolithQueueFactory.java | 9 +++--- .../KafkaTbRuleEngineQueueFactory.java | 10 ++++--- .../provider/TbRuleEngineQueueFactory.java | 3 +- 9 files changed, 41 insertions(+), 36 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java index db65248a8f..5ea79e42a8 100644 --- a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java +++ b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldEntityMessageProcessor.java @@ -107,7 +107,7 @@ public class CalculatedFieldEntityMessageProcessor extends AbstractContextAwareM public void process(CalculatedFieldStateRestoreMsg msg) { CalculatedFieldId cfId = msg.getId().cfId(); - log.info("[{}] [{}] Processing CF state restore msg.", msg.getId().entityId(), cfId); + log.debug("[{}] [{}] Processing CF state restore msg.", msg.getId().entityId(), cfId); if (msg.getState() != null) { states.put(cfId, msg.getState()); } else { @@ -116,10 +116,10 @@ public class CalculatedFieldEntityMessageProcessor extends AbstractContextAwareM } public void process(EntityInitCalculatedFieldMsg msg) throws CalculatedFieldException { - log.info("[{}] Processing entity init CF msg.", msg.getCtx().getCfId()); + log.debug("[{}] Processing entity init CF msg.", msg.getCtx().getCfId()); var ctx = msg.getCtx(); if (msg.isForceReinit()) { - log.info("Force reinitialization of CF: [{}].", ctx.getCfId()); + log.debug("Force reinitialization of CF: [{}].", ctx.getCfId()); states.remove(ctx.getCfId()); } try { @@ -138,7 +138,7 @@ public class CalculatedFieldEntityMessageProcessor extends AbstractContextAwareM } public void process(CalculatedFieldEntityDeleteMsg msg) { - log.info("[{}] Processing CF entity delete msg.", msg.getEntityId()); + log.debug("[{}] Processing CF entity delete msg.", msg.getEntityId()); if (this.entityId.equals(msg.getEntityId())) { if (states.isEmpty()) { msg.getCallback().onSuccess(); @@ -244,7 +244,7 @@ public class CalculatedFieldEntityMessageProcessor extends AbstractContextAwareM private void processArgumentValuesUpdate(CalculatedFieldCtx ctx, List cfIdList, MultipleTbCallback callback, Map newArgValues, UUID tbMsgId, TbMsgType tbMsgType) throws CalculatedFieldException { if (newArgValues.isEmpty()) { - log.info("[{}] No new argument values to process for CF.", ctx.getCfId()); + log.debug("[{}] No new argument values to process for CF.", ctx.getCfId()); callback.onSuccess(CALLBACKS_PER_CF); } CalculatedFieldState state = states.get(ctx.getCfId()); diff --git a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java index e109e83e4d..15a8fbebb9 100644 --- a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java +++ b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java @@ -236,12 +236,12 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware private void onCfCreated(ComponentLifecycleMsg msg, TbCallback callback) throws CalculatedFieldException { var cfId = new CalculatedFieldId(msg.getEntityId().getId()); if (calculatedFields.containsKey(cfId)) { - log.warn("[{}] CF was already initialized [{}]", tenantId, cfId); + log.debug("[{}] CF was already initialized [{}]", tenantId, cfId); callback.onSuccess(); } else { var cf = cfDaoService.findById(msg.getTenantId(), cfId); if (cf == null) { - log.warn("[{}] Failed to lookup CF by id [{}]", tenantId, cfId); + log.debug("[{}] Failed to lookup CF by id [{}]", tenantId, cfId); callback.onSuccess(); } else { var cfCtx = new CalculatedFieldCtx(cf, systemContext.getTbelInvokeService(), systemContext.getApiLimitService()); @@ -268,7 +268,7 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware } else { var newCf = cfDaoService.findById(msg.getTenantId(), cfId); if (newCf == null) { - log.warn("[{}] Failed to lookup CF by id [{}]", tenantId, cfId); + log.debug("[{}] Failed to lookup CF by id [{}]", tenantId, cfId); callback.onSuccess(); } else { var newCfCtx = new CalculatedFieldCtx(newCf, systemContext.getTbelInvokeService(), systemContext.getApiLimitService()); @@ -313,7 +313,7 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware var cfId = new CalculatedFieldId(msg.getEntityId().getId()); var cfCtx = calculatedFields.remove(cfId); if (cfCtx == null) { - log.warn("[{}] CF was already deleted [{}]", tenantId, cfId); + log.debug("[{}] CF was already deleted [{}]", tenantId, cfId); callback.onSuccess(); } else { entityIdCalculatedFields.get(cfCtx.getEntityId()).remove(cfCtx); diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/KafkaCalculatedFieldStateService.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/KafkaCalculatedFieldStateService.java index ffbd0d4b03..37f79edf6e 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/KafkaCalculatedFieldStateService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/KafkaCalculatedFieldStateService.java @@ -68,7 +68,7 @@ public class KafkaCalculatedFieldStateService extends AbstractCalculatedFieldSta @Override public void init(PartitionedQueueConsumerManager> eventConsumer) { - var queueKey = new QueueKey(ServiceType.TB_RULE_ENGINE, DataConstants.CF_STATES_QUEUE_NAME); + var queueKey = new QueueKey(ServiceType.TB_RULE_ENGINE, DataConstants.CF_QUEUE_NAME); PartitionedQueueConsumerManager> stateConsumer = PartitionedQueueConsumerManager.>create() .queueKey(queueKey) .topic(partitionService.getTopic(queueKey)) @@ -106,7 +106,7 @@ public class KafkaCalculatedFieldStateService extends AbstractCalculatedFieldSta @Override protected void doPersist(CalculatedFieldEntityCtxId stateId, CalculatedFieldStateProto stateMsgProto, TbCallback callback) { - TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_RULE_ENGINE, DataConstants.CF_STATES_QUEUE_NAME, stateId.tenantId(), stateId.entityId()); + TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_RULE_ENGINE, DataConstants.CF_QUEUE_NAME, stateId.tenantId(), stateId.entityId()); TbProtoQueueMsg msg = new TbProtoQueueMsg<>(stateId.entityId().getId(), stateMsgProto); if (stateMsgProto == null) { putStateId(msg.getHeaders(), stateId); diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/RocksDBCalculatedFieldStateService.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/RocksDBCalculatedFieldStateService.java index 889d8fcc01..3d2f522440 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/RocksDBCalculatedFieldStateService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/RocksDBCalculatedFieldStateService.java @@ -46,7 +46,7 @@ public class RocksDBCalculatedFieldStateService extends AbstractCalculatedFieldS @Override public void init(PartitionedQueueConsumerManager> eventConsumer) { - super.stateServices.put(new QueueKey(ServiceType.TB_RULE_ENGINE, DataConstants.CF_STATES_QUEUE_NAME), new DefaultQueueStateService<>(eventConsumer)); + super.stateServices.put(new QueueKey(ServiceType.TB_RULE_ENGINE, DataConstants.CF_QUEUE_NAME), new DefaultQueueStateService<>(eventConsumer)); } @Override diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCalculatedFieldConsumerService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCalculatedFieldConsumerService.java index 8b1cc8be1e..1af2116805 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCalculatedFieldConsumerService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCalculatedFieldConsumerService.java @@ -29,16 +29,16 @@ import org.thingsboard.server.common.data.DataConstants; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.id.EntityIdFactory; import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.page.PageDataIterable; import org.thingsboard.server.common.data.plugin.ComponentLifecycleEvent; -import org.thingsboard.server.common.data.queue.Queue; import org.thingsboard.server.common.data.queue.QueueConfig; import org.thingsboard.server.common.msg.cf.CalculatedFieldPartitionChangeMsg; import org.thingsboard.server.common.msg.plugin.ComponentLifecycleMsg; import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.common.msg.queue.TbCallback; import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; -import org.thingsboard.server.dao.queue.QueueService; import org.thingsboard.server.dao.tenant.TbTenantProfileCache; +import org.thingsboard.server.dao.tenant.TenantService; import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldLinkedTelemetryMsgProto; import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldTelemetryMsgProto; import org.thingsboard.server.gen.transport.TransportProtos.ToCalculatedFieldMsg; @@ -81,7 +81,7 @@ public class DefaultTbCalculatedFieldConsumerService extends AbstractPartitionBa private long packProcessingTimeout; private final TbRuleEngineQueueFactory queueFactory; - private final QueueService queueService; + private final TenantService tenantService; private final CalculatedFieldStateService stateService; private final ConcurrentMap>> consumers = new ConcurrentHashMap<>(); @@ -96,33 +96,33 @@ public class DefaultTbCalculatedFieldConsumerService extends AbstractPartitionBa ApplicationEventPublisher eventPublisher, JwtSettingsService jwtSettingsService, CalculatedFieldCache calculatedFieldCache, - QueueService queueService, - CalculatedFieldStateService stateService) { + CalculatedFieldStateService stateService, + TenantService tenantService) { super(actorContext, tenantProfileCache, deviceProfileCache, assetProfileCache, calculatedFieldCache, apiUsageStateService, partitionService, eventPublisher, jwtSettingsService); this.queueFactory = tbQueueFactory; - this.queueService = queueService; this.stateService = stateService; + this.tenantService = tenantService; } @Override protected void onStartUp() { - List queues = queueService.findAllQueues(); - for (Queue configuration : queues) { - if (partitionService.isManagedByCurrentService(configuration.getTenantId())) { - stateService.init(createConsumer(configuration)); + PageDataIterable iterator = new PageDataIterable<>(tenantService::findTenantsIds, 1024); + for (TenantId tenantId : iterator) { + if (partitionService.isManagedByCurrentService(tenantId)) { + stateService.init(createConsumer(tenantId)); } } } - private PartitionedQueueConsumerManager> createConsumer(Queue queue) { - QueueKey queueKey = new QueueKey(ServiceType.TB_RULE_ENGINE, DataConstants.CF_QUEUE_NAME, queue.getTenantId()); + private PartitionedQueueConsumerManager> createConsumer(TenantId tenantId) { + QueueKey queueKey = new QueueKey(ServiceType.TB_RULE_ENGINE, DataConstants.CF_QUEUE_NAME); var eventConsumer = PartitionedQueueConsumerManager.>create() .queueKey(queueKey) .topic(partitionService.getTopic(queueKey)) .pollInterval(pollInterval) .msgPackProcessor(this::processMsgs) - .consumerCreator((config, partitionId) -> queueFactory.createToCalculatedFieldMsgConsumer(queue, partitionId)) + .consumerCreator((config, partitionId) -> queueFactory.createToCalculatedFieldMsgConsumer(tenantId, partitionId)) .queueAdmin(queueFactory.getCalculatedFieldQueueAdmin()) .consumerExecutor(consumersExecutor) .scheduler(scheduler) @@ -251,7 +251,7 @@ public class DefaultTbCalculatedFieldConsumerService extends AbstractPartitionBa if (!CollectionUtils.isEmpty(partitions)) { stateService.delete(queueKey, partitions); } - var consumer = consumers.remove(queueKey); + var consumer = consumers.get(queueKey); if (consumer != null) { consumer.stop(); } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/InMemoryMonolithQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/InMemoryMonolithQueueFactory.java index ec7da1975d..94d53e4864 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/InMemoryMonolithQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/InMemoryMonolithQueueFactory.java @@ -20,6 +20,7 @@ import lombok.extern.slf4j.Slf4j; import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; import org.springframework.scheduling.annotation.Scheduled; import org.springframework.stereotype.Component; +import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.queue.Queue; import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.gen.js.JsInvokeProtos; @@ -133,7 +134,7 @@ public class InMemoryMonolithQueueFactory implements TbCoreQueueFactory, TbRuleE } @Override - public TbQueueConsumer> createToCalculatedFieldMsgConsumer(Queue configuration, Integer partitionId) { + public TbQueueConsumer> createToCalculatedFieldMsgConsumer(TenantId tenantId, Integer partitionId) { return new InMemoryTbQueueConsumer<>(storage, topicService.buildTopicName(calculatedFieldSettings.getEventTopic())); } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaMonolithQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaMonolithQueueFactory.java index 14ee1c212a..4cb8fc5793 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaMonolithQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaMonolithQueueFactory.java @@ -20,6 +20,7 @@ import jakarta.annotation.PreDestroy; import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; import org.springframework.context.annotation.Bean; import org.springframework.stereotype.Component; +import org.thingsboard.server.common.data.DataConstants; import org.thingsboard.server.common.data.id.EdgeId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.queue.Queue; @@ -513,11 +514,11 @@ public class KafkaMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEngi } @Override - public TbQueueConsumer> createToCalculatedFieldMsgConsumer(Queue configuration, Integer partitionId) { - String queueName = configuration.getName(); - String groupId = topicService.buildConsumerGroupId("cf-", configuration.getTenantId(), queueName, partitionId); + public TbQueueConsumer> createToCalculatedFieldMsgConsumer(TenantId tenantId, Integer partitionId) { + String queueName = DataConstants.CF_QUEUE_NAME; + String groupId = topicService.buildConsumerGroupId("cf-", tenantId, queueName, partitionId); - cfAdmin.syncOffsets(topicService.buildConsumerGroupId("cf-", configuration.getTenantId(), queueName, null), // the fat groupId + cfAdmin.syncOffsets(topicService.buildConsumerGroupId("cf-", tenantId, queueName, null), // the fat groupId groupId, partitionId); TbKafkaConsumerTemplate.TbKafkaConsumerTemplateBuilder> consumerBuilder = TbKafkaConsumerTemplate.builder(); diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbRuleEngineQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbRuleEngineQueueFactory.java index 74c78e1aff..ae565a0827 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbRuleEngineQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbRuleEngineQueueFactory.java @@ -20,6 +20,8 @@ import jakarta.annotation.PreDestroy; import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; import org.springframework.context.annotation.Bean; import org.springframework.stereotype.Component; +import org.thingsboard.server.common.data.DataConstants; +import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.queue.Queue; import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.gen.js.JsInvokeProtos; @@ -313,11 +315,11 @@ public class KafkaTbRuleEngineQueueFactory implements TbRuleEngineQueueFactory { } @Override - public TbQueueConsumer> createToCalculatedFieldMsgConsumer(Queue configuration, Integer partitionId) { - String queueName = configuration.getName(); - String groupId = topicService.buildConsumerGroupId("cf-", configuration.getTenantId(), queueName, partitionId); + public TbQueueConsumer> createToCalculatedFieldMsgConsumer(TenantId tenantId, Integer partitionId) { + String queueName = DataConstants.CF_QUEUE_NAME; + String groupId = topicService.buildConsumerGroupId("cf-", tenantId, queueName, partitionId); - cfAdmin.syncOffsets(topicService.buildConsumerGroupId("cf-", configuration.getTenantId(), queueName, null), // the fat groupId + cfAdmin.syncOffsets(topicService.buildConsumerGroupId("cf-", tenantId, queueName, null), // the fat groupId groupId, partitionId); TbKafkaConsumerTemplate.TbKafkaConsumerTemplateBuilder> consumerBuilder = TbKafkaConsumerTemplate.builder(); diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbRuleEngineQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbRuleEngineQueueFactory.java index 4a4b36b6ae..b71f6a0471 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbRuleEngineQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbRuleEngineQueueFactory.java @@ -15,6 +15,7 @@ */ package org.thingsboard.server.queue.provider; +import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.queue.Queue; import org.thingsboard.server.gen.js.JsInvokeProtos; import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldStateProto; @@ -121,7 +122,7 @@ public interface TbRuleEngineQueueFactory extends TbUsageStatsClientQueueFactory TbQueueRequestTemplate, TbProtoQueueMsg> createRemoteJsRequestTemplate(); - TbQueueConsumer> createToCalculatedFieldMsgConsumer(Queue configuration, Integer partitionId); + TbQueueConsumer> createToCalculatedFieldMsgConsumer(TenantId tenantId, Integer partitionId); TbQueueAdmin getCalculatedFieldQueueAdmin(); From 107c5c2ef4f33a434341e32986e117a4fb2a3232 Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Tue, 1 Apr 2025 10:22:29 +0300 Subject: [PATCH 06/47] added initialization of cache when partiton change event --- ...alculatedFieldManagerMessageProcessor.java | 15 +++++ .../server/actors/tenant/TenantActor.java | 1 + .../cf/DefaultCalculatedFieldInitService.java | 10 +++- ...aultCalculatedFieldEntityProfileCache.java | 56 ++++++++++++++++--- .../server/dao/cf/CalculatedFieldService.java | 4 ++ .../dao/cf/BaseCalculatedFieldService.java | 13 +++++ .../server/dao/cf/CalculatedFieldLinkDao.java | 2 + .../sql/cf/CalculatedFieldLinkRepository.java | 2 + .../dao/sql/cf/JpaCalculatedFieldLinkDao.java | 5 ++ 9 files changed, 97 insertions(+), 11 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java index 27d3ce315a..521503a82e 100644 --- a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java +++ b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java @@ -513,6 +513,21 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware } public void onPartitionChange(CalculatedFieldPartitionChangeMsg msg) { + initCalculatedFields(); ctx.broadcastToChildren(msg, true); } + + public void initCalculatedFields() { + cfDaoService.findCalculatedFieldsByTenantId(tenantId).forEach(cf -> { + try { + onFieldInitMsg(new CalculatedFieldInitMsg(cf.getTenantId(), cf)); + } catch (CalculatedFieldException e) { + throw new RuntimeException(e); + } + }); + cfDaoService.findAllCalculatedFieldLinksByTenantId(tenantId).forEach(cfLink -> { + onLinkInitMsg(new CalculatedFieldLinkInitMsg(cfLink.getTenantId(), cfLink)); + }); + } + } diff --git a/application/src/main/java/org/thingsboard/server/actors/tenant/TenantActor.java b/application/src/main/java/org/thingsboard/server/actors/tenant/TenantActor.java index 728f715af4..b42d91a4de 100644 --- a/application/src/main/java/org/thingsboard/server/actors/tenant/TenantActor.java +++ b/application/src/main/java/org/thingsboard/server/actors/tenant/TenantActor.java @@ -274,6 +274,7 @@ public class TenantActor extends RuleChainManagerActor { () -> DefaultActorService.CF_MANAGER_DISPATCHER_NAME, () -> new CalculatedFieldManagerActorCreator(systemContext, tenantId), () -> true); + cfActor.tellWithHighPriority(msg); } catch (Exception e) { log.info("[{}] Failed to init CF Actor.", tenantId, e); } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldInitService.java b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldInitService.java index cc3022fa29..5f02be0cde 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldInitService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldInitService.java @@ -24,6 +24,7 @@ import org.thingsboard.server.common.data.ProfileEntityIdInfo; import org.thingsboard.server.common.data.page.PageDataIterable; import org.thingsboard.server.dao.asset.AssetService; import org.thingsboard.server.dao.device.DeviceService; +import org.thingsboard.server.queue.discovery.PartitionService; import org.thingsboard.server.queue.util.AfterStartUp; import org.thingsboard.server.queue.util.TbRuleEngineComponent; import org.thingsboard.server.service.cf.cache.CalculatedFieldEntityProfileCache; @@ -37,6 +38,7 @@ public class DefaultCalculatedFieldInitService implements CalculatedFieldInitSer private final CalculatedFieldEntityProfileCache entityProfileCache; private final AssetService assetService; private final DeviceService deviceService; + private final PartitionService partitionService; @Value("${calculated_fields.init_fetch_pack_size:50000}") @Getter @@ -48,7 +50,9 @@ public class DefaultCalculatedFieldInitService implements CalculatedFieldInitSer for (ProfileEntityIdInfo idInfo : deviceIdInfos) { log.trace("Processing device record: {}", idInfo); try { - entityProfileCache.add(idInfo.getTenantId(), idInfo.getProfileId(), idInfo.getEntityId()); + if (partitionService.isManagedByCurrentService(idInfo.getTenantId())) { + entityProfileCache.add(idInfo.getTenantId(), idInfo.getProfileId(), idInfo.getEntityId()); + } } catch (Exception e) { log.error("Failed to process device record: {}", idInfo, e); } @@ -57,7 +61,9 @@ public class DefaultCalculatedFieldInitService implements CalculatedFieldInitSer for (ProfileEntityIdInfo idInfo : assetIdInfos) { log.trace("Processing asset record: {}", idInfo); try { - entityProfileCache.add(idInfo.getTenantId(), idInfo.getProfileId(), idInfo.getEntityId()); + if (partitionService.isManagedByCurrentService(idInfo.getTenantId())) { + entityProfileCache.add(idInfo.getTenantId(), idInfo.getProfileId(), idInfo.getEntityId()); + } } catch (Exception e) { log.error("Failed to process asset record: {}", idInfo, e); } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/cache/DefaultCalculatedFieldEntityProfileCache.java b/application/src/main/java/org/thingsboard/server/service/cf/cache/DefaultCalculatedFieldEntityProfileCache.java index 8059b3a825..cefdd012d8 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/cache/DefaultCalculatedFieldEntityProfileCache.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/cache/DefaultCalculatedFieldEntityProfileCache.java @@ -15,20 +15,26 @@ */ package org.thingsboard.server.service.cf.cache; +import lombok.Getter; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Service; import org.thingsboard.server.common.data.DataConstants; +import org.thingsboard.server.common.data.Device; +import org.thingsboard.server.common.data.asset.Asset; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.page.PageDataIterable; import org.thingsboard.server.common.msg.queue.ServiceType; +import org.thingsboard.server.dao.asset.AssetService; +import org.thingsboard.server.dao.device.DeviceService; import org.thingsboard.server.queue.discovery.PartitionService; import org.thingsboard.server.queue.discovery.TbApplicationEventListener; import org.thingsboard.server.queue.discovery.event.PartitionChangeEvent; import org.thingsboard.server.queue.util.TbRuleEngineComponent; import java.util.Collection; -import java.util.Optional; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; @@ -42,13 +48,16 @@ public class DefaultCalculatedFieldEntityProfileCache extends TbApplicationEvent private static final Integer UNKNOWN = 0; private final ConcurrentMap tenantCache = new ConcurrentHashMap<>(); private final PartitionService partitionService; + private final AssetService assetService; + private final DeviceService deviceService; + + @Value("${calculated_fields.init_fetch_pack_size:50000}") + @Getter + private int initFetchPackSize; @Override protected void onTbApplicationEvent(PartitionChangeEvent event) { - event.getCfPartitions().forEach(tpi -> { - Optional tenantIdOpt = tpi.getTenantId(); - tenantIdOpt.ifPresent(tenantId -> tenantCache.computeIfAbsent(tenantId, id -> new TenantEntityProfileCache())); - }); + event.getCfPartitions().forEach(tpi -> tpi.getTenantId().ifPresent(this::initCacheForNewTenant)); } @Override @@ -58,10 +67,14 @@ public class DefaultCalculatedFieldEntityProfileCache extends TbApplicationEvent @Override public void update(TenantId tenantId, EntityId oldProfileId, EntityId newProfileId, EntityId entityId) { - var cache = tenantCache.computeIfAbsent(tenantId, id -> new TenantEntityProfileCache()); - //TODO: make this method atomic; - cache.remove(oldProfileId, entityId); - cache.add(newProfileId, entityId); + tenantCache.compute(tenantId, (id, cache) -> { + if (cache == null) { + cache = new TenantEntityProfileCache(); + } + cache.remove(oldProfileId, entityId); + cache.add(newProfileId, entityId); + return cache; + }); } @Override @@ -91,4 +104,29 @@ public class DefaultCalculatedFieldEntityProfileCache extends TbApplicationEvent return tpi.getPartition().orElse(UNKNOWN); } + private void initCacheForNewTenant(TenantId tenantId) { + PageDataIterable devices = new PageDataIterable<>(pageLink -> deviceService.findDevicesByTenantId(tenantId, pageLink), initFetchPackSize); + for (Device device : devices) { + log.trace("Processing device record: {}", device); + try { + if (partitionService.isManagedByCurrentService(device.getTenantId())) { + add(device.getTenantId(), device.getDeviceProfileId(), device.getId()); + } + } catch (Exception e) { + log.error("Failed to process device record: {}", device, e); + } + } + PageDataIterable assets = new PageDataIterable<>(pageLink -> assetService.findAssetsByTenantId(tenantId, pageLink), initFetchPackSize); + for (Asset asset : assets) { + log.trace("Processing asset record: {}", asset); + try { + if (partitionService.isManagedByCurrentService(asset.getTenantId())) { + add(asset.getTenantId(), asset.getAssetProfileId(), asset.getId()); + } + } catch (Exception e) { + log.error("Failed to process asset record: {}", asset, e); + } + } + } + } diff --git a/common/dao-api/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldService.java b/common/dao-api/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldService.java index 3d91790790..8507ebbd42 100644 --- a/common/dao-api/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldService.java +++ b/common/dao-api/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldService.java @@ -39,6 +39,8 @@ public interface CalculatedFieldService extends EntityDaoService { PageData findAllCalculatedFields(PageLink pageLink); + List findCalculatedFieldsByTenantId(TenantId tenantId); + PageData findAllCalculatedFieldsByEntityId(TenantId tenantId, EntityId entityId, PageLink pageLink); void deleteCalculatedField(TenantId tenantId, CalculatedFieldId calculatedFieldId); @@ -53,6 +55,8 @@ public interface CalculatedFieldService extends EntityDaoService { List findAllCalculatedFieldLinksByEntityId(TenantId tenantId, EntityId entityId); + List findAllCalculatedFieldLinksByTenantId(TenantId tenantId); + PageData findAllCalculatedFieldLinks(PageLink pageLink); boolean referencedInAnyCalculatedField(TenantId tenantId, EntityId referencedEntityId); diff --git a/dao/src/main/java/org/thingsboard/server/dao/cf/BaseCalculatedFieldService.java b/dao/src/main/java/org/thingsboard/server/dao/cf/BaseCalculatedFieldService.java index dceef73aba..464041869e 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/cf/BaseCalculatedFieldService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/cf/BaseCalculatedFieldService.java @@ -104,6 +104,13 @@ public class BaseCalculatedFieldService extends AbstractEntityService implements return calculatedFieldDao.findAll(pageLink); } + @Override + public List findCalculatedFieldsByTenantId(TenantId tenantId) { + log.trace("Executing findAllByTenantId, tenantId [{}]", tenantId); + validateId(tenantId, id -> INCORRECT_TENANT_ID + id); + return calculatedFieldDao.findAllByTenantId(tenantId); + } + @Override public PageData findAllCalculatedFieldsByEntityId(TenantId tenantId, EntityId entityId, PageLink pageLink) { log.trace("Executing findAllByEntityId, entityId [{}], pageLink [{}]", entityId, pageLink); @@ -174,6 +181,12 @@ public class BaseCalculatedFieldService extends AbstractEntityService implements return calculatedFieldLinkDao.findCalculatedFieldLinksByEntityId(tenantId, entityId); } + @Override + public List findAllCalculatedFieldLinksByTenantId(TenantId tenantId) { + log.trace("Executing findAllCalculatedFieldLinksByTenantId, tenantId [{}]", tenantId); + return calculatedFieldLinkDao.findCalculatedFieldLinksByTenantId(tenantId); + } + @Override public PageData findAllCalculatedFieldLinks(PageLink pageLink) { log.trace("Executing findAllCalculatedFieldLinks, pageLink [{}]", pageLink); diff --git a/dao/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldLinkDao.java b/dao/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldLinkDao.java index 8b4a5e7086..cf168a9b3d 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldLinkDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldLinkDao.java @@ -31,6 +31,8 @@ public interface CalculatedFieldLinkDao extends Dao { List findCalculatedFieldLinksByEntityId(TenantId tenantId, EntityId entityId); + List findCalculatedFieldLinksByTenantId(TenantId tenantId); + List findAll(); PageData findAll(PageLink pageLink); diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/cf/CalculatedFieldLinkRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/cf/CalculatedFieldLinkRepository.java index 584a3b5199..aeb8e1b04c 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/cf/CalculatedFieldLinkRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/cf/CalculatedFieldLinkRepository.java @@ -27,4 +27,6 @@ public interface CalculatedFieldLinkRepository extends JpaRepository findAllByTenantIdAndEntityId(UUID tenantId, UUID entityId); + List findAllByTenantId(UUID tenantId); + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/cf/JpaCalculatedFieldLinkDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/cf/JpaCalculatedFieldLinkDao.java index dbb2fd87da..39b6e6b890 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/cf/JpaCalculatedFieldLinkDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/cf/JpaCalculatedFieldLinkDao.java @@ -54,6 +54,11 @@ public class JpaCalculatedFieldLinkDao extends JpaAbstractDao findCalculatedFieldLinksByTenantId(TenantId tenantId) { + return DaoUtil.convertDataList(calculatedFieldLinkRepository.findAllByTenantId(tenantId.getId())); + } + @Override public List findAll() { return DaoUtil.convertDataList(calculatedFieldLinkRepository.findAll()); From ae894f9ed3b4a1251e95131889e491d5c6a5c82b Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Tue, 1 Apr 2025 10:49:16 +0300 Subject: [PATCH 07/47] create consumer when create tenant --- .../queue/DefaultTbCalculatedFieldConsumerService.java | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCalculatedFieldConsumerService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCalculatedFieldConsumerService.java index 1af2116805..b01554f936 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCalculatedFieldConsumerService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCalculatedFieldConsumerService.java @@ -258,6 +258,10 @@ public class DefaultTbCalculatedFieldConsumerService extends AbstractPartitionBa } return toRemove; }); + } else if (event.getEvent() == ComponentLifecycleEvent.CREATED) { + if (partitionService.isManagedByCurrentService(event.getTenantId())) { + stateService.init(createConsumer(event.getTenantId())); + } } } } From b4c7521f70e6a4663cbc2e0f6438ba392006d538 Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Thu, 3 Apr 2025 17:08:49 +0300 Subject: [PATCH 08/47] fixed tests --- .../cf/CalculatedFieldStateService.java | 3 +- .../KafkaCalculatedFieldStateService.java | 4 +- .../RocksDBCalculatedFieldStateService.java | 5 +- ...faultTbCalculatedFieldConsumerService.java | 56 +++++++++++++------ .../discovery/event/PartitionChangeEvent.java | 1 + .../InMemoryMonolithQueueFactory.java | 4 +- .../provider/KafkaMonolithQueueFactory.java | 37 +++++++----- .../KafkaTbRuleEngineQueueFactory.java | 38 +++++++------ .../provider/TbRuleEngineQueueFactory.java | 4 +- 9 files changed, 94 insertions(+), 58 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldStateService.java b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldStateService.java index a8b0ccf30e..7cb9e718f6 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldStateService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldStateService.java @@ -15,6 +15,7 @@ */ package org.thingsboard.server.service.cf; +import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.msg.queue.TbCallback; import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; import org.thingsboard.server.exception.CalculatedFieldStateException; @@ -29,7 +30,7 @@ import java.util.Set; public interface CalculatedFieldStateService { - void init(PartitionedQueueConsumerManager> eventConsumer); + void init(TenantId tenantId, PartitionedQueueConsumerManager> eventConsumer); void persistState(CalculatedFieldEntityCtxId stateId, CalculatedFieldState state, TbCallback callback) throws CalculatedFieldStateException; diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/KafkaCalculatedFieldStateService.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/KafkaCalculatedFieldStateService.java index 37f79edf6e..1eb8d0acaa 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/KafkaCalculatedFieldStateService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/KafkaCalculatedFieldStateService.java @@ -67,7 +67,7 @@ public class KafkaCalculatedFieldStateService extends AbstractCalculatedFieldSta private final AtomicInteger counter = new AtomicInteger(); @Override - public void init(PartitionedQueueConsumerManager> eventConsumer) { + public void init(TenantId tenantId, PartitionedQueueConsumerManager> eventConsumer) { var queueKey = new QueueKey(ServiceType.TB_RULE_ENGINE, DataConstants.CF_QUEUE_NAME); PartitionedQueueConsumerManager> stateConsumer = PartitionedQueueConsumerManager.>create() .queueKey(queueKey) @@ -91,7 +91,7 @@ public class KafkaCalculatedFieldStateService extends AbstractCalculatedFieldSta } } }) - .consumerCreator((config, partitionId) -> queueFactory.createCalculatedFieldStateConsumer()) + .consumerCreator((config, partitionId) -> queueFactory.createCalculatedFieldStateConsumer(tenantId)) .queueAdmin(queueFactory.getCalculatedFieldQueueAdmin()) .consumerExecutor(eventConsumer.getConsumerExecutor()) .scheduler(eventConsumer.getScheduler()) diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/RocksDBCalculatedFieldStateService.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/RocksDBCalculatedFieldStateService.java index 3d2f522440..fc30822b85 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/RocksDBCalculatedFieldStateService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/RocksDBCalculatedFieldStateService.java @@ -21,6 +21,7 @@ import lombok.extern.slf4j.Slf4j; import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; import org.springframework.stereotype.Service; import org.thingsboard.server.common.data.DataConstants; +import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.common.msg.queue.TbCallback; import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; @@ -45,8 +46,8 @@ public class RocksDBCalculatedFieldStateService extends AbstractCalculatedFieldS private final CfRocksDb cfRocksDb; @Override - public void init(PartitionedQueueConsumerManager> eventConsumer) { - super.stateServices.put(new QueueKey(ServiceType.TB_RULE_ENGINE, DataConstants.CF_QUEUE_NAME), new DefaultQueueStateService<>(eventConsumer)); + public void init(TenantId tenantId, PartitionedQueueConsumerManager> eventConsumer) { + super.stateServices.put(new QueueKey(ServiceType.TB_RULE_ENGINE, DataConstants.CF_QUEUE_NAME, tenantId), new DefaultQueueStateService<>(eventConsumer)); } @Override diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCalculatedFieldConsumerService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCalculatedFieldConsumerService.java index d4a88c2fe6..9e0376b483 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCalculatedFieldConsumerService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCalculatedFieldConsumerService.java @@ -45,6 +45,7 @@ import org.thingsboard.server.gen.transport.TransportProtos.ToCalculatedFieldMsg import org.thingsboard.server.gen.transport.TransportProtos.ToCalculatedFieldNotificationMsg; import org.thingsboard.server.queue.TbQueueConsumer; import org.thingsboard.server.queue.common.TbProtoQueueMsg; +import org.thingsboard.server.queue.common.consumer.MainQueueConsumerManager; import org.thingsboard.server.queue.common.consumer.PartitionedQueueConsumerManager; import org.thingsboard.server.queue.discovery.PartitionService; import org.thingsboard.server.queue.discovery.QueueKey; @@ -62,6 +63,7 @@ import org.thingsboard.server.service.queue.processing.IdMsgPair; import org.thingsboard.server.service.security.auth.jwt.settings.JwtSettingsService; import java.util.List; +import java.util.Optional; import java.util.Set; import java.util.UUID; import java.util.concurrent.ConcurrentHashMap; @@ -88,7 +90,6 @@ public class DefaultTbCalculatedFieldConsumerService extends AbstractPartitionBa private final ConcurrentMap>> consumers = new ConcurrentHashMap<>(); - public DefaultTbCalculatedFieldConsumerService(TbRuleEngineQueueFactory tbQueueFactory, ActorSystemContext actorContext, TbDeviceProfileCache deviceProfileCache, @@ -115,19 +116,19 @@ public class DefaultTbCalculatedFieldConsumerService extends AbstractPartitionBa PageDataIterable iterator = new PageDataIterable<>(tenantService::findTenantsIds, 1024); for (TenantId tenantId : iterator) { if (partitionService.isManagedByCurrentService(tenantId)) { - stateService.init(createConsumer(tenantId)); + stateService.init(tenantId, createConsumer(tenantId)); } } } private PartitionedQueueConsumerManager> createConsumer(TenantId tenantId) { - QueueKey queueKey = new QueueKey(ServiceType.TB_RULE_ENGINE, DataConstants.CF_QUEUE_NAME); + QueueKey queueKey = new QueueKey(ServiceType.TB_RULE_ENGINE, DataConstants.CF_QUEUE_NAME, tenantId); var eventConsumer = PartitionedQueueConsumerManager.>create() .queueKey(queueKey) - .topic(partitionService.getTopic(queueKey)) + .topic(partitionService.getTopic(new QueueKey(ServiceType.TB_RULE_ENGINE, DataConstants.CF_QUEUE_NAME))) .pollInterval(pollInterval) .msgPackProcessor(this::processMsgs) - .consumerCreator((config, partitionId) -> queueFactory.createToCalculatedFieldMsgConsumer(tenantId, partitionId)) + .consumerCreator((config, partitionId) -> queueFactory.createToCalculatedFieldMsgConsumer(tenantId)) .queueAdmin(queueFactory.getCalculatedFieldQueueAdmin()) .consumerExecutor(consumersExecutor) .scheduler(scheduler) @@ -151,11 +152,30 @@ public class DefaultTbCalculatedFieldConsumerService extends AbstractPartitionBa protected void onPartitionChangeEvent(PartitionChangeEvent event) { try { event.getNewPartitions().forEach((queueKey, partitions) -> { - if (queueKey.getQueueName().equals(DataConstants.CF_QUEUE_NAME)) { - stateService.restore(queueKey, partitions); + if (!queueKey.getQueueName().equals(DataConstants.CF_QUEUE_NAME)) { + return; + } + if (partitionService.isManagedByCurrentService(queueKey.getTenantId())) { + var consumer = Optional.ofNullable(consumers.get(queueKey)).orElseGet(() -> { + var newConsumer = createConsumer(queueKey.getTenantId()); + stateService.init(queueKey.getTenantId(), newConsumer); + return newConsumer; + }); + if (consumer != null) { + stateService.restore(queueKey, partitions); + // eventConsumer's partitions will be updated by stateService + } } }); - // eventConsumer's partitions will be updated by stateService + consumers.keySet().stream() + .collect(Collectors.groupingBy(QueueKey::getTenantId)) + .forEach((tenantId, queueKeys) -> { + if (!partitionService.isManagedByCurrentService(tenantId)) { + queueKeys.forEach(queueKey -> { + Optional.ofNullable(consumers.remove(queueKey)).ifPresent(MainQueueConsumerManager::stop); + }); + } + }); // Cleanup old entities after corresponding consumers are stopped. // Any periodic tasks need to check that the entity is still managed by the current server before processing. @@ -250,23 +270,23 @@ public class DefaultTbCalculatedFieldConsumerService extends AbstractPartitionBa if (event.getEntityId().getEntityType() == EntityType.TENANT) { if (event.getEvent() == ComponentLifecycleEvent.DELETED) { entityProfileCache.removeTenant(event.getTenantId()); - consumers.keySet().removeIf(queueKey -> { - boolean toRemove = queueKey.getTenantId().equals(event.getTenantId()); - if (toRemove) { + + List toRemove = consumers.keySet().stream() + .filter(queueKey -> queueKey.getTenantId().equals(event.getTenantId())) + .toList(); + + toRemove.forEach(queueKey -> { + Optional.ofNullable(consumers.remove(queueKey)).ifPresent(consumer -> { Set partitions = stateService.getPartitions(queueKey); if (!CollectionUtils.isEmpty(partitions)) { stateService.delete(queueKey, partitions); } - var consumer = consumers.get(queueKey); - if (consumer != null) { - consumer.stop(); - } - } - return toRemove; + consumer.stop(); + }); }); } else if (event.getEvent() == ComponentLifecycleEvent.CREATED) { if (partitionService.isManagedByCurrentService(event.getTenantId())) { - stateService.init(createConsumer(event.getTenantId())); + stateService.init(event.getTenantId(), createConsumer(event.getTenantId())); } } } else if (event.getEntityId().getEntityType() == EntityType.ASSET_PROFILE) { diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/event/PartitionChangeEvent.java b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/event/PartitionChangeEvent.java index 32537edba5..7c2d0b0240 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/event/PartitionChangeEvent.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/event/PartitionChangeEvent.java @@ -18,6 +18,7 @@ package org.thingsboard.server.queue.discovery.event; import lombok.Getter; import lombok.ToString; import org.thingsboard.server.common.data.DataConstants; +import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; import org.thingsboard.server.queue.discovery.QueueKey; diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/InMemoryMonolithQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/InMemoryMonolithQueueFactory.java index 94d53e4864..e825a1424d 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/InMemoryMonolithQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/InMemoryMonolithQueueFactory.java @@ -134,7 +134,7 @@ public class InMemoryMonolithQueueFactory implements TbCoreQueueFactory, TbRuleE } @Override - public TbQueueConsumer> createToCalculatedFieldMsgConsumer(TenantId tenantId, Integer partitionId) { + public TbQueueConsumer> createToCalculatedFieldMsgConsumer(TenantId tenantId) { return new InMemoryTbQueueConsumer<>(storage, topicService.buildTopicName(calculatedFieldSettings.getEventTopic())); } @@ -154,7 +154,7 @@ public class InMemoryMonolithQueueFactory implements TbCoreQueueFactory, TbRuleE } @Override - public TbQueueConsumer> createCalculatedFieldStateConsumer() { + public TbQueueConsumer> createCalculatedFieldStateConsumer(TenantId tenantId) { return new InMemoryTbQueueConsumer<>(storage, topicService.buildTopicName(calculatedFieldSettings.getStateTopic())); } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaMonolithQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaMonolithQueueFactory.java index 4cb8fc5793..ca41ee221d 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaMonolithQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaMonolithQueueFactory.java @@ -514,12 +514,12 @@ public class KafkaMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEngi } @Override - public TbQueueConsumer> createToCalculatedFieldMsgConsumer(TenantId tenantId, Integer partitionId) { + public TbQueueConsumer> createToCalculatedFieldMsgConsumer(TenantId tenantId) { String queueName = DataConstants.CF_QUEUE_NAME; - String groupId = topicService.buildConsumerGroupId("cf-", tenantId, queueName, partitionId); + String groupId = topicService.buildConsumerGroupId("cf-", tenantId, queueName, null); cfAdmin.syncOffsets(topicService.buildConsumerGroupId("cf-", tenantId, queueName, null), // the fat groupId - groupId, partitionId); + groupId, null); TbKafkaConsumerTemplate.TbKafkaConsumerTemplateBuilder> consumerBuilder = TbKafkaConsumerTemplate.builder(); consumerBuilder.settings(kafkaSettings); @@ -572,18 +572,25 @@ public class KafkaMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEngi } @Override - public TbQueueConsumer> createCalculatedFieldStateConsumer() { - return TbKafkaConsumerTemplate.>builder() - .settings(kafkaSettings) - .topic(topicService.buildTopicName(calculatedFieldSettings.getStateTopic())) - .readFromBeginning(true) - .stopWhenRead(true) - .clientId("monolith-calculated-field-state-consumer-" + serviceInfoProvider.getServiceId() + "-" + consumerCount.incrementAndGet()) - .groupId(topicService.buildTopicName("monolith-calculated-field-state-consumer")) - .decoder(msg -> new TbProtoQueueMsg<>(msg.getKey(), msg.getData() != null ? CalculatedFieldStateProto.parseFrom(msg.getData()) : null, msg.getHeaders())) - .admin(cfStateAdmin) - .statsService(consumerStatsService) - .build(); + public TbQueueConsumer> createCalculatedFieldStateConsumer(TenantId tenantId) { + String queueName = DataConstants.CF_STATES_QUEUE_NAME; + String groupId = topicService.buildConsumerGroupId("cf-", tenantId, queueName, null); + + cfAdmin.syncOffsets(topicService.buildConsumerGroupId("cf-", tenantId, queueName, null), // the fat groupId + groupId, null); + + TbKafkaConsumerTemplate.TbKafkaConsumerTemplateBuilder> consumerBuilder = TbKafkaConsumerTemplate.builder(); + consumerBuilder.settings(kafkaSettings); + consumerBuilder.topic(topicService.buildTopicName(calculatedFieldSettings.getStateTopic())); + consumerBuilder.readFromBeginning(true); + consumerBuilder.stopWhenRead(true); + consumerBuilder.clientId("cf-" + queueName + "-consumer-" + serviceInfoProvider.getServiceId() + "-" + consumerCount.incrementAndGet()); + consumerBuilder.groupId(groupId); + consumerBuilder.decoder(msg -> new TbProtoQueueMsg<>(msg.getKey(), msg.getData() != null ? CalculatedFieldStateProto.parseFrom(msg.getData()) : null, msg.getHeaders())); + consumerBuilder.admin(cfStateAdmin); + consumerBuilder.statsService(consumerStatsService); + + return consumerBuilder.build(); } @Override diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbRuleEngineQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbRuleEngineQueueFactory.java index ae565a0827..84ad126ff2 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbRuleEngineQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbRuleEngineQueueFactory.java @@ -315,17 +315,17 @@ public class KafkaTbRuleEngineQueueFactory implements TbRuleEngineQueueFactory { } @Override - public TbQueueConsumer> createToCalculatedFieldMsgConsumer(TenantId tenantId, Integer partitionId) { + public TbQueueConsumer> createToCalculatedFieldMsgConsumer(TenantId tenantId) { String queueName = DataConstants.CF_QUEUE_NAME; - String groupId = topicService.buildConsumerGroupId("cf-", tenantId, queueName, partitionId); + String groupId = topicService.buildConsumerGroupId("cf-", tenantId, queueName, null); cfAdmin.syncOffsets(topicService.buildConsumerGroupId("cf-", tenantId, queueName, null), // the fat groupId - groupId, partitionId); + groupId, null); TbKafkaConsumerTemplate.TbKafkaConsumerTemplateBuilder> consumerBuilder = TbKafkaConsumerTemplate.builder(); consumerBuilder.settings(kafkaSettings); consumerBuilder.topic(topicService.buildTopicName(calculatedFieldSettings.getEventTopic())); - consumerBuilder.clientId("cf-" + queueName + "-consumer-" + serviceInfoProvider.getServiceId() + "-" + consumerCount.incrementAndGet()); + consumerBuilder.clientId("cf-" + queueName + "-consumer-" + tenantId + serviceInfoProvider.getServiceId() + "-" + consumerCount.incrementAndGet()); consumerBuilder.groupId(groupId); consumerBuilder.decoder(msg -> new TbProtoQueueMsg<>(msg.getKey(), ToCalculatedFieldMsg.parseFrom(msg.getData()), msg.getHeaders())); consumerBuilder.admin(cfAdmin); @@ -372,18 +372,24 @@ public class KafkaTbRuleEngineQueueFactory implements TbRuleEngineQueueFactory { } @Override - public TbQueueConsumer> createCalculatedFieldStateConsumer() { - return TbKafkaConsumerTemplate.>builder() - .settings(kafkaSettings) - .topic(topicService.buildTopicName(calculatedFieldSettings.getStateTopic())) - .readFromBeginning(true) - .stopWhenRead(true) - .clientId("tb-rule-engine-calculated-field-state-consumer-" + serviceInfoProvider.getServiceId() + "-" + consumerCount.incrementAndGet()) - .groupId(topicService.buildTopicName("tb-rule-engine-calculated-field-state-consumer")) - .decoder(msg -> new TbProtoQueueMsg<>(msg.getKey(), msg.getData() != null ? CalculatedFieldStateProto.parseFrom(msg.getData()) : null, msg.getHeaders())) - .admin(cfStateAdmin) - .statsService(consumerStatsService) - .build(); + public TbQueueConsumer> createCalculatedFieldStateConsumer(TenantId tenantId) { + String queueName = DataConstants.CF_STATES_QUEUE_NAME; + String groupId = topicService.buildConsumerGroupId("cf-", tenantId, queueName, null); + + cfAdmin.syncOffsets(topicService.buildConsumerGroupId("cf-", tenantId, queueName, null), // the fat groupId + groupId, null); + + TbKafkaConsumerTemplate.TbKafkaConsumerTemplateBuilder> consumerBuilder = TbKafkaConsumerTemplate.builder(); + consumerBuilder.settings(kafkaSettings); + consumerBuilder.topic(topicService.buildTopicName(calculatedFieldSettings.getStateTopic())); + consumerBuilder.readFromBeginning(true); + consumerBuilder.stopWhenRead(true); + consumerBuilder.clientId("cf-" + queueName + "-consumer-" + tenantId + serviceInfoProvider.getServiceId() + "-" + consumerCount.incrementAndGet()); + consumerBuilder.groupId(groupId); + consumerBuilder.decoder(msg -> new TbProtoQueueMsg<>(msg.getKey(), msg.getData() != null ? CalculatedFieldStateProto.parseFrom(msg.getData()) : null, msg.getHeaders())); + consumerBuilder.admin(cfStateAdmin); + consumerBuilder.statsService(consumerStatsService); + return consumerBuilder.build(); } @Override diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbRuleEngineQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbRuleEngineQueueFactory.java index b71f6a0471..7076ceb0b0 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbRuleEngineQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbRuleEngineQueueFactory.java @@ -122,7 +122,7 @@ public interface TbRuleEngineQueueFactory extends TbUsageStatsClientQueueFactory TbQueueRequestTemplate, TbProtoQueueMsg> createRemoteJsRequestTemplate(); - TbQueueConsumer> createToCalculatedFieldMsgConsumer(TenantId tenantId, Integer partitionId); + TbQueueConsumer> createToCalculatedFieldMsgConsumer(TenantId tenantId); TbQueueAdmin getCalculatedFieldQueueAdmin(); @@ -132,7 +132,7 @@ public interface TbRuleEngineQueueFactory extends TbUsageStatsClientQueueFactory TbQueueProducer> createToCalculatedFieldNotificationMsgProducer(); - TbQueueConsumer> createCalculatedFieldStateConsumer(); + TbQueueConsumer> createCalculatedFieldStateConsumer(TenantId tenantId); TbQueueProducer> createCalculatedFieldStateProducer(); From 5b8c8d1aedeb367a65c227db42e6d71d96ca2aa2 Mon Sep 17 00:00:00 2001 From: Artem Dzhereleiko Date: Fri, 4 Apr 2025 16:11:46 +0300 Subject: [PATCH 09/47] UI: Show error for bad xml scada symbols --- .../scada-symbol/scada-symbol.component.ts | 51 ++++++++++--------- 1 file changed, 28 insertions(+), 23 deletions(-) diff --git a/ui-ngx/src/app/modules/home/pages/scada-symbol/scada-symbol.component.ts b/ui-ngx/src/app/modules/home/pages/scada-symbol/scada-symbol.component.ts index 955ab16f97..caaf0edc01 100644 --- a/ui-ngx/src/app/modules/home/pages/scada-symbol/scada-symbol.component.ts +++ b/ui-ngx/src/app/modules/home/pages/scada-symbol/scada-symbol.component.ts @@ -79,6 +79,7 @@ import { SaveWidgetTypeAsDialogResult } from '@home/pages/widget/save-widget-type-as-dialog.component'; import { WidgetService } from '@core/http/widget.service'; +import { ActionNotificationShow } from '@core/notification/notification.actions'; @Component({ selector: 'tb-scada-symbol', @@ -214,31 +215,35 @@ export class ScadaSymbolComponent extends PageComponent } const metadata: ScadaSymbolMetadata = this.scadaSymbolFormGroup.get('metadata').value; const scadaSymbolContent = this.prepareScadaSymbolContent(metadata); - const file = createFileFromContent(scadaSymbolContent, this.symbolData.imageResource.fileName, - this.symbolData.imageResource.descriptor.mediaType); - const type = imageResourceType(this.symbolData.imageResource); - let imageInfoObservable = - this.imageService.updateImage(type, this.symbolData.imageResource.resourceKey, file); - if (metadata.title !== this.symbolData.imageResource.title) { - imageInfoObservable = imageInfoObservable.pipe( - switchMap(imageInfo => { - imageInfo.title = metadata.title; - return this.imageService.updateImageInfo(imageInfo); - }) - ); - } - imageInfoObservable.pipe( - switchMap(imageInfo => this.imageService.getImageString( + if (scadaSymbolContent.includes('parsererror')) { + this.store.dispatch(new ActionNotificationShow({ message: scadaSymbolContent, type: 'error' })); + } else { + const file = createFileFromContent(scadaSymbolContent, this.symbolData.imageResource.fileName, + this.symbolData.imageResource.descriptor.mediaType); + const type = imageResourceType(this.symbolData.imageResource); + let imageInfoObservable = + this.imageService.updateImage(type, this.symbolData.imageResource.resourceKey, file); + if (metadata.title !== this.symbolData.imageResource.title) { + imageInfoObservable = imageInfoObservable.pipe( + switchMap(imageInfo => { + imageInfo.title = metadata.title; + return this.imageService.updateImageInfo(imageInfo); + }) + ); + } + imageInfoObservable.pipe( + switchMap(imageInfo => this.imageService.getImageString( `${IMAGES_URL_PREFIX}/${type}/${encodeURIComponent(imageInfo.resourceKey)}`).pipe( - map(content => ({ - imageResource: imageInfo, - scadaSymbolContent: content - })) + map(content => ({ + imageResource: imageInfo, + scadaSymbolContent: content + })) )) - ).subscribe(data => { - this.init(data); - this.updateBreadcrumbs.emit(); - }); + ).subscribe(data => { + this.init(data); + this.updateBreadcrumbs.emit(); + }); + } } } From 88faa2fb75e2f41109eda086013d5e02039c7a46 Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Mon, 7 Apr 2025 17:45:45 +0300 Subject: [PATCH 10/47] wip consumer groups --- .../AbstractCalculatedFieldStateService.java | 18 ++--- .../cf/CalculatedFieldStateService.java | 9 +-- .../KafkaCalculatedFieldStateService.java | 14 ++-- .../RocksDBCalculatedFieldStateService.java | 9 +-- ...faultTbCalculatedFieldConsumerService.java | 77 +++++-------------- .../InMemoryMonolithQueueFactory.java | 5 +- .../provider/KafkaMonolithQueueFactory.java | 42 ++++------ .../KafkaTbRuleEngineQueueFactory.java | 43 ++++------- .../provider/TbRuleEngineQueueFactory.java | 11 ++- 9 files changed, 82 insertions(+), 146 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/cf/AbstractCalculatedFieldStateService.java b/application/src/main/java/org/thingsboard/server/service/cf/AbstractCalculatedFieldStateService.java index 1347705a23..91c08ab6e0 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/AbstractCalculatedFieldStateService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/AbstractCalculatedFieldStateService.java @@ -30,9 +30,7 @@ import org.thingsboard.server.service.cf.ctx.CalculatedFieldEntityCtxId; import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldState; import java.util.Collection; -import java.util.Map; import java.util.Set; -import java.util.concurrent.ConcurrentHashMap; import java.util.stream.Collectors; import static org.thingsboard.server.utils.CalculatedFieldUtils.fromProto; @@ -43,7 +41,7 @@ public abstract class AbstractCalculatedFieldStateService implements CalculatedF @Autowired private ActorSystemContext actorSystemContext; - protected Map, TbProtoQueueMsg>> stateServices = new ConcurrentHashMap<>(); + protected QueueStateService, TbProtoQueueMsg> stateService; @Override public final void persistState(CalculatedFieldEntityCtxId stateId, CalculatedFieldState state, TbCallback callback) { @@ -74,22 +72,22 @@ public abstract class AbstractCalculatedFieldStateService implements CalculatedF @Override public void restore(QueueKey queueKey, Set partitions) { - stateServices.get(queueKey).update(queueKey, partitions); + stateService.update(queueKey, partitions); } @Override - public void delete(QueueKey queueKey, Set partitions) { - stateServices.get(queueKey).delete(partitions); + public void delete(Set partitions) { + stateService.delete(partitions); } @Override - public Set getPartitions(QueueKey queueKey) { - return stateServices.get(queueKey).getPartitions().values().stream().flatMap(Collection::stream).collect(Collectors.toSet()); + public Set getPartitions() { + return stateService.getPartitions().values().stream().flatMap(Collection::stream).collect(Collectors.toSet()); } @Override - public void stop(QueueKey queueKey) { - stateServices.get(queueKey).stop(); + public void stop() { + stateService.stop(); } } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldStateService.java b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldStateService.java index 7cb9e718f6..d0b34f18e8 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldStateService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldStateService.java @@ -15,7 +15,6 @@ */ package org.thingsboard.server.service.cf; -import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.msg.queue.TbCallback; import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; import org.thingsboard.server.exception.CalculatedFieldStateException; @@ -30,7 +29,7 @@ import java.util.Set; public interface CalculatedFieldStateService { - void init(TenantId tenantId, PartitionedQueueConsumerManager> eventConsumer); + void init(PartitionedQueueConsumerManager> eventConsumer); void persistState(CalculatedFieldEntityCtxId stateId, CalculatedFieldState state, TbCallback callback) throws CalculatedFieldStateException; @@ -38,10 +37,10 @@ public interface CalculatedFieldStateService { void restore(QueueKey queueKey, Set partitions); - void delete(QueueKey queueKey, Set partitions); + void delete(Set partitions); - Set getPartitions(QueueKey queueKey); + Set getPartitions(); - void stop(QueueKey queueKey); + void stop(); } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/KafkaCalculatedFieldStateService.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/KafkaCalculatedFieldStateService.java index 1eb8d0acaa..825b68dec2 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/KafkaCalculatedFieldStateService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/KafkaCalculatedFieldStateService.java @@ -67,8 +67,8 @@ public class KafkaCalculatedFieldStateService extends AbstractCalculatedFieldSta private final AtomicInteger counter = new AtomicInteger(); @Override - public void init(TenantId tenantId, PartitionedQueueConsumerManager> eventConsumer) { - var queueKey = new QueueKey(ServiceType.TB_RULE_ENGINE, DataConstants.CF_QUEUE_NAME); + public void init(PartitionedQueueConsumerManager> eventConsumer) { + var queueKey = new QueueKey(ServiceType.TB_RULE_ENGINE, DataConstants.CF_STATES_QUEUE_NAME); PartitionedQueueConsumerManager> stateConsumer = PartitionedQueueConsumerManager.>create() .queueKey(queueKey) .topic(partitionService.getTopic(queueKey)) @@ -91,16 +91,16 @@ public class KafkaCalculatedFieldStateService extends AbstractCalculatedFieldSta } } }) - .consumerCreator((config, partitionId) -> queueFactory.createCalculatedFieldStateConsumer(tenantId)) + .consumerCreator((queueConfig, partitionId) -> queueFactory.createCalculatedFieldStateConsumer()) .queueAdmin(queueFactory.getCalculatedFieldQueueAdmin()) .consumerExecutor(eventConsumer.getConsumerExecutor()) .scheduler(eventConsumer.getScheduler()) .taskExecutor(eventConsumer.getTaskExecutor()) .build(); - super.stateServices.put(queueKey, KafkaQueueStateService., TbProtoQueueMsg>builder() + super.stateService = KafkaQueueStateService., TbProtoQueueMsg>builder() .eventConsumer(eventConsumer) .stateConsumer(stateConsumer) - .build()); + .build(); this.stateProducer = (TbKafkaProducerTemplate>) queueFactory.createCalculatedFieldStateProducer(); } @@ -148,8 +148,8 @@ public class KafkaCalculatedFieldStateService extends AbstractCalculatedFieldSta } @Override - public void stop(QueueKey queueKey) { - super.stop(queueKey); + public void stop() { + super.stop(); stateProducer.stop(); } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/RocksDBCalculatedFieldStateService.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/RocksDBCalculatedFieldStateService.java index fc30822b85..9dc6139ca5 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/RocksDBCalculatedFieldStateService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/RocksDBCalculatedFieldStateService.java @@ -20,9 +20,6 @@ import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; import org.springframework.stereotype.Service; -import org.thingsboard.server.common.data.DataConstants; -import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.common.msg.queue.TbCallback; import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldStateProto; @@ -46,8 +43,8 @@ public class RocksDBCalculatedFieldStateService extends AbstractCalculatedFieldS private final CfRocksDb cfRocksDb; @Override - public void init(TenantId tenantId, PartitionedQueueConsumerManager> eventConsumer) { - super.stateServices.put(new QueueKey(ServiceType.TB_RULE_ENGINE, DataConstants.CF_QUEUE_NAME, tenantId), new DefaultQueueStateService<>(eventConsumer)); + public void init(PartitionedQueueConsumerManager> eventConsumer) { + super.stateService = new DefaultQueueStateService<>(eventConsumer); } @Override @@ -64,7 +61,7 @@ public class RocksDBCalculatedFieldStateService extends AbstractCalculatedFieldS @Override public void restore(QueueKey queueKey, Set partitions) { - if (stateServices.get(queueKey).getPartitions().isEmpty()) { + if (stateService.getPartitions().isEmpty()) { cfRocksDb.forEach((key, value) -> { try { processRestoredState(CalculatedFieldStateProto.parseFrom(value)); diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCalculatedFieldConsumerService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCalculatedFieldConsumerService.java index 9e0376b483..ef4c4485e6 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCalculatedFieldConsumerService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCalculatedFieldConsumerService.java @@ -29,23 +29,22 @@ import org.thingsboard.server.common.data.DataConstants; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.id.EntityIdFactory; import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.common.data.page.PageDataIterable; import org.thingsboard.server.common.data.plugin.ComponentLifecycleEvent; +import org.thingsboard.server.common.data.queue.Queue; import org.thingsboard.server.common.data.queue.QueueConfig; import org.thingsboard.server.common.msg.cf.CalculatedFieldPartitionChangeMsg; import org.thingsboard.server.common.msg.plugin.ComponentLifecycleMsg; import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.common.msg.queue.TbCallback; import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; +import org.thingsboard.server.dao.queue.QueueService; import org.thingsboard.server.dao.tenant.TbTenantProfileCache; -import org.thingsboard.server.dao.tenant.TenantService; import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldLinkedTelemetryMsgProto; import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldTelemetryMsgProto; import org.thingsboard.server.gen.transport.TransportProtos.ToCalculatedFieldMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToCalculatedFieldNotificationMsg; import org.thingsboard.server.queue.TbQueueConsumer; import org.thingsboard.server.queue.common.TbProtoQueueMsg; -import org.thingsboard.server.queue.common.consumer.MainQueueConsumerManager; import org.thingsboard.server.queue.common.consumer.PartitionedQueueConsumerManager; import org.thingsboard.server.queue.discovery.PartitionService; import org.thingsboard.server.queue.discovery.QueueKey; @@ -63,7 +62,6 @@ import org.thingsboard.server.service.queue.processing.IdMsgPair; import org.thingsboard.server.service.security.auth.jwt.settings.JwtSettingsService; import java.util.List; -import java.util.Optional; import java.util.Set; import java.util.UUID; import java.util.concurrent.ConcurrentHashMap; @@ -84,11 +82,9 @@ public class DefaultTbCalculatedFieldConsumerService extends AbstractPartitionBa private long packProcessingTimeout; private final TbRuleEngineQueueFactory queueFactory; - private final TenantService tenantService; private final CalculatedFieldStateService stateService; private final CalculatedFieldEntityProfileCache entityProfileCache; - - private final ConcurrentMap>> consumers = new ConcurrentHashMap<>(); + private final QueueService queueService; public DefaultTbCalculatedFieldConsumerService(TbRuleEngineQueueFactory tbQueueFactory, ActorSystemContext actorContext, @@ -102,39 +98,34 @@ public class DefaultTbCalculatedFieldConsumerService extends AbstractPartitionBa CalculatedFieldCache calculatedFieldCache, CalculatedFieldStateService stateService, CalculatedFieldEntityProfileCache entityProfileCache, - TenantService tenantService) { + QueueService queueService) { super(actorContext, tenantProfileCache, deviceProfileCache, assetProfileCache, calculatedFieldCache, apiUsageStateService, partitionService, eventPublisher, jwtSettingsService); this.queueFactory = tbQueueFactory; this.stateService = stateService; this.entityProfileCache = entityProfileCache; - this.tenantService = tenantService; + this.queueService = queueService; } @Override protected void onStartUp() { - PageDataIterable iterator = new PageDataIterable<>(tenantService::findTenantsIds, 1024); - for (TenantId tenantId : iterator) { - if (partitionService.isManagedByCurrentService(tenantId)) { - stateService.init(tenantId, createConsumer(tenantId)); - } - } + var queueKey = new QueueKey(ServiceType.TB_RULE_ENGINE, DataConstants.CF_QUEUE_NAME); + Queue queue = queueService.findQueueByTenantIdAndName(queueKey.getTenantId(), queueKey.getQueueName()); + createConsumer(queue, queueKey); } - private PartitionedQueueConsumerManager> createConsumer(TenantId tenantId) { - QueueKey queueKey = new QueueKey(ServiceType.TB_RULE_ENGINE, DataConstants.CF_QUEUE_NAME, tenantId); + private PartitionedQueueConsumerManager> createConsumer(Queue queue, QueueKey queueKey) { var eventConsumer = PartitionedQueueConsumerManager.>create() .queueKey(queueKey) - .topic(partitionService.getTopic(new QueueKey(ServiceType.TB_RULE_ENGINE, DataConstants.CF_QUEUE_NAME))) .pollInterval(pollInterval) .msgPackProcessor(this::processMsgs) - .consumerCreator((config, partitionId) -> queueFactory.createToCalculatedFieldMsgConsumer(tenantId)) + .consumerCreator((queueConfig, partitionId) -> queueFactory.createToCalculatedFieldMsgConsumer(queue, partitionId)) .queueAdmin(queueFactory.getCalculatedFieldQueueAdmin()) .consumerExecutor(consumersExecutor) .scheduler(scheduler) .taskExecutor(mgmtExecutor) .build(); - consumers.put(queueKey, eventConsumer); + stateService.init(eventConsumer); return eventConsumer; } @@ -152,30 +143,12 @@ public class DefaultTbCalculatedFieldConsumerService extends AbstractPartitionBa protected void onPartitionChangeEvent(PartitionChangeEvent event) { try { event.getNewPartitions().forEach((queueKey, partitions) -> { - if (!queueKey.getQueueName().equals(DataConstants.CF_QUEUE_NAME)) { - return; - } - if (partitionService.isManagedByCurrentService(queueKey.getTenantId())) { - var consumer = Optional.ofNullable(consumers.get(queueKey)).orElseGet(() -> { - var newConsumer = createConsumer(queueKey.getTenantId()); - stateService.init(queueKey.getTenantId(), newConsumer); - return newConsumer; - }); - if (consumer != null) { + if (DataConstants.CF_QUEUE_NAME.equals(queueKey.getQueueName()) || DataConstants.CF_STATES_QUEUE_NAME.equals(queueKey.getQueueName())) { + if (partitionService.isManagedByCurrentService(queueKey.getTenantId())) { stateService.restore(queueKey, partitions); - // eventConsumer's partitions will be updated by stateService } } }); - consumers.keySet().stream() - .collect(Collectors.groupingBy(QueueKey::getTenantId)) - .forEach((tenantId, queueKeys) -> { - if (!partitionService.isManagedByCurrentService(tenantId)) { - queueKeys.forEach(queueKey -> { - Optional.ofNullable(consumers.remove(queueKey)).ifPresent(MainQueueConsumerManager::stop); - }); - } - }); // Cleanup old entities after corresponding consumers are stopped. // Any periodic tasks need to check that the entity is still managed by the current server before processing. @@ -271,23 +244,13 @@ public class DefaultTbCalculatedFieldConsumerService extends AbstractPartitionBa if (event.getEvent() == ComponentLifecycleEvent.DELETED) { entityProfileCache.removeTenant(event.getTenantId()); - List toRemove = consumers.keySet().stream() - .filter(queueKey -> queueKey.getTenantId().equals(event.getTenantId())) - .toList(); - - toRemove.forEach(queueKey -> { - Optional.ofNullable(consumers.remove(queueKey)).ifPresent(consumer -> { - Set partitions = stateService.getPartitions(queueKey); - if (!CollectionUtils.isEmpty(partitions)) { - stateService.delete(queueKey, partitions); - } - consumer.stop(); - }); - }); - } else if (event.getEvent() == ComponentLifecycleEvent.CREATED) { - if (partitionService.isManagedByCurrentService(event.getTenantId())) { - stateService.init(event.getTenantId(), createConsumer(event.getTenantId())); + Set partitions = stateService.getPartitions(); + if (CollectionUtils.isEmpty(partitions)) { + return; } + stateService.delete(partitions.stream() + .filter(tpi -> tpi.getTenantId().isPresent() && tpi.getTenantId().get().equals(event.getTenantId())) + .collect(Collectors.toSet())); } } else if (event.getEntityId().getEntityType() == EntityType.ASSET_PROFILE) { if (event.getEvent() == ComponentLifecycleEvent.DELETED) { @@ -320,7 +283,7 @@ public class DefaultTbCalculatedFieldConsumerService extends AbstractPartitionBa @Override protected void stopConsumers() { super.stopConsumers(); - consumers.keySet().forEach(stateService::stop); // eventConsumer will be stopped by stateService + stateService.stop(); // eventConsumer will be stopped by stateService } } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/InMemoryMonolithQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/InMemoryMonolithQueueFactory.java index e825a1424d..2a8403040d 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/InMemoryMonolithQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/InMemoryMonolithQueueFactory.java @@ -20,7 +20,6 @@ import lombok.extern.slf4j.Slf4j; import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; import org.springframework.scheduling.annotation.Scheduled; import org.springframework.stereotype.Component; -import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.queue.Queue; import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.gen.js.JsInvokeProtos; @@ -134,7 +133,7 @@ public class InMemoryMonolithQueueFactory implements TbCoreQueueFactory, TbRuleE } @Override - public TbQueueConsumer> createToCalculatedFieldMsgConsumer(TenantId tenantId) { + public TbQueueConsumer> createToCalculatedFieldMsgConsumer(Queue queue, Integer partitionId) { return new InMemoryTbQueueConsumer<>(storage, topicService.buildTopicName(calculatedFieldSettings.getEventTopic())); } @@ -154,7 +153,7 @@ public class InMemoryMonolithQueueFactory implements TbCoreQueueFactory, TbRuleE } @Override - public TbQueueConsumer> createCalculatedFieldStateConsumer(TenantId tenantId) { + public TbQueueConsumer> createCalculatedFieldStateConsumer() { return new InMemoryTbQueueConsumer<>(storage, topicService.buildTopicName(calculatedFieldSettings.getStateTopic())); } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaMonolithQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaMonolithQueueFactory.java index ca41ee221d..9a52c775cb 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaMonolithQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaMonolithQueueFactory.java @@ -20,7 +20,6 @@ import jakarta.annotation.PreDestroy; import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; import org.springframework.context.annotation.Bean; import org.springframework.stereotype.Component; -import org.thingsboard.server.common.data.DataConstants; import org.thingsboard.server.common.data.id.EdgeId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.queue.Queue; @@ -514,12 +513,10 @@ public class KafkaMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEngi } @Override - public TbQueueConsumer> createToCalculatedFieldMsgConsumer(TenantId tenantId) { - String queueName = DataConstants.CF_QUEUE_NAME; - String groupId = topicService.buildConsumerGroupId("cf-", tenantId, queueName, null); - - cfAdmin.syncOffsets(topicService.buildConsumerGroupId("cf-", tenantId, queueName, null), // the fat groupId - groupId, null); + public TbQueueConsumer> createToCalculatedFieldMsgConsumer(Queue queue, Integer partitionId) { + String queueName = queue.getName(); + TenantId tenantId = queue.getTenantId(); + String groupId = topicService.buildConsumerGroupId("cf-", tenantId, queueName, partitionId); TbKafkaConsumerTemplate.TbKafkaConsumerTemplateBuilder> consumerBuilder = TbKafkaConsumerTemplate.builder(); consumerBuilder.settings(kafkaSettings); @@ -572,25 +569,18 @@ public class KafkaMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEngi } @Override - public TbQueueConsumer> createCalculatedFieldStateConsumer(TenantId tenantId) { - String queueName = DataConstants.CF_STATES_QUEUE_NAME; - String groupId = topicService.buildConsumerGroupId("cf-", tenantId, queueName, null); - - cfAdmin.syncOffsets(topicService.buildConsumerGroupId("cf-", tenantId, queueName, null), // the fat groupId - groupId, null); - - TbKafkaConsumerTemplate.TbKafkaConsumerTemplateBuilder> consumerBuilder = TbKafkaConsumerTemplate.builder(); - consumerBuilder.settings(kafkaSettings); - consumerBuilder.topic(topicService.buildTopicName(calculatedFieldSettings.getStateTopic())); - consumerBuilder.readFromBeginning(true); - consumerBuilder.stopWhenRead(true); - consumerBuilder.clientId("cf-" + queueName + "-consumer-" + serviceInfoProvider.getServiceId() + "-" + consumerCount.incrementAndGet()); - consumerBuilder.groupId(groupId); - consumerBuilder.decoder(msg -> new TbProtoQueueMsg<>(msg.getKey(), msg.getData() != null ? CalculatedFieldStateProto.parseFrom(msg.getData()) : null, msg.getHeaders())); - consumerBuilder.admin(cfStateAdmin); - consumerBuilder.statsService(consumerStatsService); - - return consumerBuilder.build(); + public TbQueueConsumer> createCalculatedFieldStateConsumer() { + return TbKafkaConsumerTemplate.>builder() + .settings(kafkaSettings) + .topic(topicService.buildTopicName(calculatedFieldSettings.getStateTopic())) + .readFromBeginning(true) + .stopWhenRead(true) + .clientId("monolith-calculated-field-state-consumer-" + serviceInfoProvider.getServiceId() + "-" + consumerCount.incrementAndGet()) + .groupId(topicService.buildTopicName("monolith-calculated-field-state-consumer")) + .decoder(msg -> new TbProtoQueueMsg<>(msg.getKey(), msg.getData() != null ? CalculatedFieldStateProto.parseFrom(msg.getData()) : null, msg.getHeaders())) + .admin(cfStateAdmin) + .statsService(consumerStatsService) + .build(); } @Override diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbRuleEngineQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbRuleEngineQueueFactory.java index 84ad126ff2..d1e3528189 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbRuleEngineQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbRuleEngineQueueFactory.java @@ -20,7 +20,6 @@ import jakarta.annotation.PreDestroy; import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; import org.springframework.context.annotation.Bean; import org.springframework.stereotype.Component; -import org.thingsboard.server.common.data.DataConstants; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.queue.Queue; import org.thingsboard.server.common.msg.queue.ServiceType; @@ -315,17 +314,15 @@ public class KafkaTbRuleEngineQueueFactory implements TbRuleEngineQueueFactory { } @Override - public TbQueueConsumer> createToCalculatedFieldMsgConsumer(TenantId tenantId) { - String queueName = DataConstants.CF_QUEUE_NAME; - String groupId = topicService.buildConsumerGroupId("cf-", tenantId, queueName, null); - - cfAdmin.syncOffsets(topicService.buildConsumerGroupId("cf-", tenantId, queueName, null), // the fat groupId - groupId, null); + public TbQueueConsumer> createToCalculatedFieldMsgConsumer(Queue queue, Integer partitionId) { + String queueName = queue.getName(); + TenantId tenantId = queue.getTenantId(); + String groupId = topicService.buildConsumerGroupId("cf-", tenantId, queueName, partitionId); TbKafkaConsumerTemplate.TbKafkaConsumerTemplateBuilder> consumerBuilder = TbKafkaConsumerTemplate.builder(); consumerBuilder.settings(kafkaSettings); consumerBuilder.topic(topicService.buildTopicName(calculatedFieldSettings.getEventTopic())); - consumerBuilder.clientId("cf-" + queueName + "-consumer-" + tenantId + serviceInfoProvider.getServiceId() + "-" + consumerCount.incrementAndGet()); + consumerBuilder.clientId("cf-" + queueName + "-consumer-" + serviceInfoProvider.getServiceId() + "-" + consumerCount.incrementAndGet()); consumerBuilder.groupId(groupId); consumerBuilder.decoder(msg -> new TbProtoQueueMsg<>(msg.getKey(), ToCalculatedFieldMsg.parseFrom(msg.getData()), msg.getHeaders())); consumerBuilder.admin(cfAdmin); @@ -372,24 +369,18 @@ public class KafkaTbRuleEngineQueueFactory implements TbRuleEngineQueueFactory { } @Override - public TbQueueConsumer> createCalculatedFieldStateConsumer(TenantId tenantId) { - String queueName = DataConstants.CF_STATES_QUEUE_NAME; - String groupId = topicService.buildConsumerGroupId("cf-", tenantId, queueName, null); - - cfAdmin.syncOffsets(topicService.buildConsumerGroupId("cf-", tenantId, queueName, null), // the fat groupId - groupId, null); - - TbKafkaConsumerTemplate.TbKafkaConsumerTemplateBuilder> consumerBuilder = TbKafkaConsumerTemplate.builder(); - consumerBuilder.settings(kafkaSettings); - consumerBuilder.topic(topicService.buildTopicName(calculatedFieldSettings.getStateTopic())); - consumerBuilder.readFromBeginning(true); - consumerBuilder.stopWhenRead(true); - consumerBuilder.clientId("cf-" + queueName + "-consumer-" + tenantId + serviceInfoProvider.getServiceId() + "-" + consumerCount.incrementAndGet()); - consumerBuilder.groupId(groupId); - consumerBuilder.decoder(msg -> new TbProtoQueueMsg<>(msg.getKey(), msg.getData() != null ? CalculatedFieldStateProto.parseFrom(msg.getData()) : null, msg.getHeaders())); - consumerBuilder.admin(cfStateAdmin); - consumerBuilder.statsService(consumerStatsService); - return consumerBuilder.build(); + public TbQueueConsumer> createCalculatedFieldStateConsumer() { + return TbKafkaConsumerTemplate.>builder() + .settings(kafkaSettings) + .topic(topicService.buildTopicName(calculatedFieldSettings.getStateTopic())) + .readFromBeginning(true) + .stopWhenRead(true) + .clientId("tb-rule-engine-calculated-field-state-consumer-" + serviceInfoProvider.getServiceId() + "-" + consumerCount.incrementAndGet()) + .groupId(topicService.buildTopicName("tb-rule-engine-calculated-field-state-consumer")) + .decoder(msg -> new TbProtoQueueMsg<>(msg.getKey(), msg.getData() != null ? CalculatedFieldStateProto.parseFrom(msg.getData()) : null, msg.getHeaders())) + .admin(cfStateAdmin) + .statsService(consumerStatsService) + .build(); } @Override diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbRuleEngineQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbRuleEngineQueueFactory.java index 7076ceb0b0..5cabfe9cbb 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbRuleEngineQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbRuleEngineQueueFactory.java @@ -15,7 +15,6 @@ */ package org.thingsboard.server.queue.provider; -import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.queue.Queue; import org.thingsboard.server.gen.js.JsInvokeProtos; import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldStateProto; @@ -96,8 +95,8 @@ public interface TbRuleEngineQueueFactory extends TbUsageStatsClientQueueFactory /** * Used to consume messages by TB Rule Engine Service * - * @return * @param configuration + * @return */ TbQueueConsumer> createToRuleEngineMsgConsumer(Queue configuration); @@ -105,9 +104,9 @@ public interface TbRuleEngineQueueFactory extends TbUsageStatsClientQueueFactory * Used to consume messages by TB Rule Engine Service * Intended usage for consumer per partition strategy * - * @return TbQueueConsumer * @param configuration - * @param partitionId as a suffix for consumer name + * @param partitionId as a suffix for consumer name + * @return TbQueueConsumer */ default TbQueueConsumer> createToRuleEngineMsgConsumer(Queue configuration, Integer partitionId) { return createToRuleEngineMsgConsumer(configuration); @@ -122,7 +121,7 @@ public interface TbRuleEngineQueueFactory extends TbUsageStatsClientQueueFactory TbQueueRequestTemplate, TbProtoQueueMsg> createRemoteJsRequestTemplate(); - TbQueueConsumer> createToCalculatedFieldMsgConsumer(TenantId tenantId); + TbQueueConsumer> createToCalculatedFieldMsgConsumer(Queue queue, Integer partitionId); TbQueueAdmin getCalculatedFieldQueueAdmin(); @@ -132,7 +131,7 @@ public interface TbRuleEngineQueueFactory extends TbUsageStatsClientQueueFactory TbQueueProducer> createToCalculatedFieldNotificationMsgProducer(); - TbQueueConsumer> createCalculatedFieldStateConsumer(TenantId tenantId); + TbQueueConsumer> createCalculatedFieldStateConsumer(); TbQueueProducer> createCalculatedFieldStateProducer(); From 434a534ba35005bc7c695ef26049cc2c86b6cdd9 Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Tue, 8 Apr 2025 15:05:19 +0300 Subject: [PATCH 11/47] created cf consumer using topic partition info --- ...faultTbCalculatedFieldConsumerService.java | 63 +++++++++++++------ .../InMemoryMonolithQueueFactory.java | 3 +- .../provider/KafkaMonolithQueueFactory.java | 8 ++- .../KafkaTbRuleEngineQueueFactory.java | 8 ++- .../provider/TbRuleEngineQueueFactory.java | 3 +- 5 files changed, 59 insertions(+), 26 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCalculatedFieldConsumerService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCalculatedFieldConsumerService.java index ef4c4485e6..6adae15fe6 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCalculatedFieldConsumerService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCalculatedFieldConsumerService.java @@ -17,7 +17,6 @@ package org.thingsboard.server.service.queue; import jakarta.annotation.PreDestroy; import lombok.extern.slf4j.Slf4j; -import org.apache.commons.collections4.CollectionUtils; import org.springframework.beans.factory.annotation.Value; import org.springframework.context.ApplicationEventPublisher; import org.springframework.context.event.EventListener; @@ -30,7 +29,6 @@ import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.id.EntityIdFactory; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.plugin.ComponentLifecycleEvent; -import org.thingsboard.server.common.data.queue.Queue; import org.thingsboard.server.common.data.queue.QueueConfig; import org.thingsboard.server.common.msg.cf.CalculatedFieldPartitionChangeMsg; import org.thingsboard.server.common.msg.plugin.ComponentLifecycleMsg; @@ -45,6 +43,7 @@ import org.thingsboard.server.gen.transport.TransportProtos.ToCalculatedFieldMsg import org.thingsboard.server.gen.transport.TransportProtos.ToCalculatedFieldNotificationMsg; import org.thingsboard.server.queue.TbQueueConsumer; import org.thingsboard.server.queue.common.TbProtoQueueMsg; +import org.thingsboard.server.queue.common.consumer.MainQueueConsumerManager; import org.thingsboard.server.queue.common.consumer.PartitionedQueueConsumerManager; import org.thingsboard.server.queue.discovery.PartitionService; import org.thingsboard.server.queue.discovery.QueueKey; @@ -62,6 +61,7 @@ import org.thingsboard.server.service.queue.processing.IdMsgPair; import org.thingsboard.server.service.security.auth.jwt.settings.JwtSettingsService; import java.util.List; +import java.util.Optional; import java.util.Set; import java.util.UUID; import java.util.concurrent.ConcurrentHashMap; @@ -84,7 +84,8 @@ public class DefaultTbCalculatedFieldConsumerService extends AbstractPartitionBa private final TbRuleEngineQueueFactory queueFactory; private final CalculatedFieldStateService stateService; private final CalculatedFieldEntityProfileCache entityProfileCache; - private final QueueService queueService; + + private final ConcurrentMap>> consumers = new ConcurrentHashMap<>(); public DefaultTbCalculatedFieldConsumerService(TbRuleEngineQueueFactory tbQueueFactory, ActorSystemContext actorContext, @@ -97,35 +98,42 @@ public class DefaultTbCalculatedFieldConsumerService extends AbstractPartitionBa JwtSettingsService jwtSettingsService, CalculatedFieldCache calculatedFieldCache, CalculatedFieldStateService stateService, - CalculatedFieldEntityProfileCache entityProfileCache, - QueueService queueService) { + CalculatedFieldEntityProfileCache entityProfileCache) { super(actorContext, tenantProfileCache, deviceProfileCache, assetProfileCache, calculatedFieldCache, apiUsageStateService, partitionService, eventPublisher, jwtSettingsService); this.queueFactory = tbQueueFactory; this.stateService = stateService; this.entityProfileCache = entityProfileCache; - this.queueService = queueService; } @Override protected void onStartUp() { var queueKey = new QueueKey(ServiceType.TB_RULE_ENGINE, DataConstants.CF_QUEUE_NAME); - Queue queue = queueService.findQueueByTenantIdAndName(queueKey.getTenantId(), queueKey.getQueueName()); - createConsumer(queue, queueKey); + createConsumer(queueKey); } - private PartitionedQueueConsumerManager> createConsumer(Queue queue, QueueKey queueKey) { + private PartitionedQueueConsumerManager> createConsumer(QueueKey queueKey) { + String topic = partitionService.getTopic(queueKey); var eventConsumer = PartitionedQueueConsumerManager.>create() .queueKey(queueKey) + .topic(topic) .pollInterval(pollInterval) .msgPackProcessor(this::processMsgs) - .consumerCreator((queueConfig, partitionId) -> queueFactory.createToCalculatedFieldMsgConsumer(queue, partitionId)) + .consumerCreator((queueConfig, partitionId) -> { + TopicPartitionInfo tpi = TopicPartitionInfo.builder() + .tenantId(queueKey.getTenantId()) + .topic(partitionService.getTopic(queueKey)) + .partition(partitionId) + .build(); + return queueFactory.createToCalculatedFieldMsgConsumer(tpi, partitionId); + }) .queueAdmin(queueFactory.getCalculatedFieldQueueAdmin()) .consumerExecutor(consumersExecutor) .scheduler(scheduler) .taskExecutor(mgmtExecutor) .build(); stateService.init(eventConsumer); + consumers.put(queueKey, eventConsumer); return eventConsumer; } @@ -145,11 +153,24 @@ public class DefaultTbCalculatedFieldConsumerService extends AbstractPartitionBa event.getNewPartitions().forEach((queueKey, partitions) -> { if (DataConstants.CF_QUEUE_NAME.equals(queueKey.getQueueName()) || DataConstants.CF_STATES_QUEUE_NAME.equals(queueKey.getQueueName())) { if (partitionService.isManagedByCurrentService(queueKey.getTenantId())) { - stateService.restore(queueKey, partitions); + var consumer = Optional.ofNullable(consumers.get(queueKey)).orElseGet(() -> createConsumer(queueKey)); + if (consumer != null) { + stateService.restore(queueKey, partitions); + } } } }); + consumers.keySet().stream() + .collect(Collectors.groupingBy(QueueKey::getTenantId)) + .forEach((tenantId, queueKeys) -> { + if (!partitionService.isManagedByCurrentService(tenantId)) { + queueKeys.forEach(queueKey -> { + Optional.ofNullable(consumers.remove(queueKey)).ifPresent(MainQueueConsumerManager::stop); + }); + } + }); + // Cleanup old entities after corresponding consumers are stopped. // Any periodic tasks need to check that the entity is still managed by the current server before processing. actorContext.tell(new CalculatedFieldPartitionChangeMsg()); @@ -244,13 +265,19 @@ public class DefaultTbCalculatedFieldConsumerService extends AbstractPartitionBa if (event.getEvent() == ComponentLifecycleEvent.DELETED) { entityProfileCache.removeTenant(event.getTenantId()); - Set partitions = stateService.getPartitions(); - if (CollectionUtils.isEmpty(partitions)) { - return; - } - stateService.delete(partitions.stream() - .filter(tpi -> tpi.getTenantId().isPresent() && tpi.getTenantId().get().equals(event.getTenantId())) - .collect(Collectors.toSet())); + List toRemove = consumers.keySet().stream() + .filter(queueKey -> queueKey.getTenantId().equals(event.getTenantId())) + .toList(); + toRemove.forEach(queueKey -> { + Optional.ofNullable(consumers.remove(queueKey)).ifPresent(consumer -> { + Set partitions = stateService.getPartitions().stream() + .filter(tpi -> tpi.getTenantId().isPresent() && tpi.getTenantId().get().equals(event.getTenantId())) + .collect(Collectors.toSet()); + if (!partitions.isEmpty()) { + consumer.delete(partitions); + } + }); + }); } } else if (event.getEntityId().getEntityType() == EntityType.ASSET_PROFILE) { if (event.getEvent() == ComponentLifecycleEvent.DELETED) { diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/InMemoryMonolithQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/InMemoryMonolithQueueFactory.java index 2a8403040d..9fbd75d347 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/InMemoryMonolithQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/InMemoryMonolithQueueFactory.java @@ -22,6 +22,7 @@ import org.springframework.scheduling.annotation.Scheduled; import org.springframework.stereotype.Component; import org.thingsboard.server.common.data.queue.Queue; import org.thingsboard.server.common.msg.queue.ServiceType; +import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; import org.thingsboard.server.gen.js.JsInvokeProtos; import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldStateProto; @@ -133,7 +134,7 @@ public class InMemoryMonolithQueueFactory implements TbCoreQueueFactory, TbRuleE } @Override - public TbQueueConsumer> createToCalculatedFieldMsgConsumer(Queue queue, Integer partitionId) { + public TbQueueConsumer> createToCalculatedFieldMsgConsumer(TopicPartitionInfo tpi, Integer partitionId) { return new InMemoryTbQueueConsumer<>(storage, topicService.buildTopicName(calculatedFieldSettings.getEventTopic())); } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaMonolithQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaMonolithQueueFactory.java index 9a52c775cb..198e2f7a83 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaMonolithQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaMonolithQueueFactory.java @@ -20,10 +20,12 @@ import jakarta.annotation.PreDestroy; import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; import org.springframework.context.annotation.Bean; import org.springframework.stereotype.Component; +import org.thingsboard.server.common.data.DataConstants; import org.thingsboard.server.common.data.id.EdgeId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.queue.Queue; import org.thingsboard.server.common.msg.queue.ServiceType; +import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; import org.thingsboard.server.gen.js.JsInvokeProtos; import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldStateProto; import org.thingsboard.server.gen.transport.TransportProtos.FromEdqsMsg; @@ -513,9 +515,9 @@ public class KafkaMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEngi } @Override - public TbQueueConsumer> createToCalculatedFieldMsgConsumer(Queue queue, Integer partitionId) { - String queueName = queue.getName(); - TenantId tenantId = queue.getTenantId(); + public TbQueueConsumer> createToCalculatedFieldMsgConsumer(TopicPartitionInfo tpi, Integer partitionId) { + String queueName = DataConstants.CF_QUEUE_NAME; + TenantId tenantId = tpi.getTenantId().orElse(TenantId.SYS_TENANT_ID); String groupId = topicService.buildConsumerGroupId("cf-", tenantId, queueName, partitionId); TbKafkaConsumerTemplate.TbKafkaConsumerTemplateBuilder> consumerBuilder = TbKafkaConsumerTemplate.builder(); diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbRuleEngineQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbRuleEngineQueueFactory.java index d1e3528189..f387d1af11 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbRuleEngineQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbRuleEngineQueueFactory.java @@ -20,9 +20,11 @@ import jakarta.annotation.PreDestroy; import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; import org.springframework.context.annotation.Bean; import org.springframework.stereotype.Component; +import org.thingsboard.server.common.data.DataConstants; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.queue.Queue; import org.thingsboard.server.common.msg.queue.ServiceType; +import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; import org.thingsboard.server.gen.js.JsInvokeProtos; import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldStateProto; import org.thingsboard.server.gen.transport.TransportProtos.FromEdqsMsg; @@ -314,9 +316,9 @@ public class KafkaTbRuleEngineQueueFactory implements TbRuleEngineQueueFactory { } @Override - public TbQueueConsumer> createToCalculatedFieldMsgConsumer(Queue queue, Integer partitionId) { - String queueName = queue.getName(); - TenantId tenantId = queue.getTenantId(); + public TbQueueConsumer> createToCalculatedFieldMsgConsumer(TopicPartitionInfo tpi, Integer partitionId) { + String queueName = DataConstants.CF_QUEUE_NAME; + TenantId tenantId = tpi.getTenantId().orElse(TenantId.SYS_TENANT_ID); String groupId = topicService.buildConsumerGroupId("cf-", tenantId, queueName, partitionId); TbKafkaConsumerTemplate.TbKafkaConsumerTemplateBuilder> consumerBuilder = TbKafkaConsumerTemplate.builder(); diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbRuleEngineQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbRuleEngineQueueFactory.java index 5cabfe9cbb..3980aaa632 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbRuleEngineQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbRuleEngineQueueFactory.java @@ -16,6 +16,7 @@ package org.thingsboard.server.queue.provider; import org.thingsboard.server.common.data.queue.Queue; +import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; import org.thingsboard.server.gen.js.JsInvokeProtos; import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldStateProto; import org.thingsboard.server.gen.transport.TransportProtos.ToCalculatedFieldMsg; @@ -121,7 +122,7 @@ public interface TbRuleEngineQueueFactory extends TbUsageStatsClientQueueFactory TbQueueRequestTemplate, TbProtoQueueMsg> createRemoteJsRequestTemplate(); - TbQueueConsumer> createToCalculatedFieldMsgConsumer(Queue queue, Integer partitionId); + TbQueueConsumer> createToCalculatedFieldMsgConsumer(TopicPartitionInfo tpi, Integer partitionId); TbQueueAdmin getCalculatedFieldQueueAdmin(); From 580c0074d3f586e4e08f0cfca888a619c6db1edb Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Wed, 9 Apr 2025 16:59:17 +0300 Subject: [PATCH 12/47] used tpi in main consumer manager --- .../KafkaCalculatedFieldStateService.java | 2 +- ...faultTbCalculatedFieldConsumerService.java | 63 ++++--------------- .../queue/DefaultTbCoreConsumerService.java | 2 +- .../queue/DefaultTbEdgeConsumerService.java | 5 +- .../TbRuleEngineQueueConsumerManager.java | 7 ++- .../server/edqs/processor/EdqsProcessor.java | 2 +- .../edqs/state/KafkaEdqsStateService.java | 5 +- .../consumer/MainQueueConsumerManager.java | 6 +- .../PartitionedQueueConsumerManager.java | 2 +- .../InMemoryMonolithQueueFactory.java | 2 +- .../provider/KafkaMonolithQueueFactory.java | 5 +- .../KafkaTbRuleEngineQueueFactory.java | 5 +- .../provider/TbRuleEngineQueueFactory.java | 2 +- 13 files changed, 38 insertions(+), 70 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/KafkaCalculatedFieldStateService.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/KafkaCalculatedFieldStateService.java index 825b68dec2..19d2b74bb5 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/KafkaCalculatedFieldStateService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/KafkaCalculatedFieldStateService.java @@ -91,7 +91,7 @@ public class KafkaCalculatedFieldStateService extends AbstractCalculatedFieldSta } } }) - .consumerCreator((queueConfig, partitionId) -> queueFactory.createCalculatedFieldStateConsumer()) + .consumerCreator((queueConfig, tpi) -> queueFactory.createCalculatedFieldStateConsumer()) .queueAdmin(queueFactory.getCalculatedFieldQueueAdmin()) .consumerExecutor(eventConsumer.getConsumerExecutor()) .scheduler(eventConsumer.getScheduler()) diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCalculatedFieldConsumerService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCalculatedFieldConsumerService.java index 6adae15fe6..f108949efa 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCalculatedFieldConsumerService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCalculatedFieldConsumerService.java @@ -17,6 +17,7 @@ package org.thingsboard.server.service.queue; import jakarta.annotation.PreDestroy; import lombok.extern.slf4j.Slf4j; +import org.apache.commons.collections4.CollectionUtils; import org.springframework.beans.factory.annotation.Value; import org.springframework.context.ApplicationEventPublisher; import org.springframework.context.event.EventListener; @@ -35,7 +36,6 @@ import org.thingsboard.server.common.msg.plugin.ComponentLifecycleMsg; import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.common.msg.queue.TbCallback; import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; -import org.thingsboard.server.dao.queue.QueueService; import org.thingsboard.server.dao.tenant.TbTenantProfileCache; import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldLinkedTelemetryMsgProto; import org.thingsboard.server.gen.transport.TransportProtos.CalculatedFieldTelemetryMsgProto; @@ -43,7 +43,6 @@ import org.thingsboard.server.gen.transport.TransportProtos.ToCalculatedFieldMsg import org.thingsboard.server.gen.transport.TransportProtos.ToCalculatedFieldNotificationMsg; import org.thingsboard.server.queue.TbQueueConsumer; import org.thingsboard.server.queue.common.TbProtoQueueMsg; -import org.thingsboard.server.queue.common.consumer.MainQueueConsumerManager; import org.thingsboard.server.queue.common.consumer.PartitionedQueueConsumerManager; import org.thingsboard.server.queue.discovery.PartitionService; import org.thingsboard.server.queue.discovery.QueueKey; @@ -61,7 +60,6 @@ import org.thingsboard.server.service.queue.processing.IdMsgPair; import org.thingsboard.server.service.security.auth.jwt.settings.JwtSettingsService; import java.util.List; -import java.util.Optional; import java.util.Set; import java.util.UUID; import java.util.concurrent.ConcurrentHashMap; @@ -85,8 +83,6 @@ public class DefaultTbCalculatedFieldConsumerService extends AbstractPartitionBa private final CalculatedFieldStateService stateService; private final CalculatedFieldEntityProfileCache entityProfileCache; - private final ConcurrentMap>> consumers = new ConcurrentHashMap<>(); - public DefaultTbCalculatedFieldConsumerService(TbRuleEngineQueueFactory tbQueueFactory, ActorSystemContext actorContext, TbDeviceProfileCache deviceProfileCache, @@ -109,32 +105,18 @@ public class DefaultTbCalculatedFieldConsumerService extends AbstractPartitionBa @Override protected void onStartUp() { var queueKey = new QueueKey(ServiceType.TB_RULE_ENGINE, DataConstants.CF_QUEUE_NAME); - createConsumer(queueKey); - } - - private PartitionedQueueConsumerManager> createConsumer(QueueKey queueKey) { - String topic = partitionService.getTopic(queueKey); var eventConsumer = PartitionedQueueConsumerManager.>create() .queueKey(queueKey) - .topic(topic) + .topic(partitionService.getTopic(queueKey)) .pollInterval(pollInterval) .msgPackProcessor(this::processMsgs) - .consumerCreator((queueConfig, partitionId) -> { - TopicPartitionInfo tpi = TopicPartitionInfo.builder() - .tenantId(queueKey.getTenantId()) - .topic(partitionService.getTopic(queueKey)) - .partition(partitionId) - .build(); - return queueFactory.createToCalculatedFieldMsgConsumer(tpi, partitionId); - }) + .consumerCreator((queueConfig, tpi) -> queueFactory.createToCalculatedFieldMsgConsumer(tpi)) .queueAdmin(queueFactory.getCalculatedFieldQueueAdmin()) .consumerExecutor(consumersExecutor) .scheduler(scheduler) .taskExecutor(mgmtExecutor) .build(); stateService.init(eventConsumer); - consumers.put(queueKey, eventConsumer); - return eventConsumer; } @PreDestroy @@ -151,26 +133,11 @@ public class DefaultTbCalculatedFieldConsumerService extends AbstractPartitionBa protected void onPartitionChangeEvent(PartitionChangeEvent event) { try { event.getNewPartitions().forEach((queueKey, partitions) -> { - if (DataConstants.CF_QUEUE_NAME.equals(queueKey.getQueueName()) || DataConstants.CF_STATES_QUEUE_NAME.equals(queueKey.getQueueName())) { - if (partitionService.isManagedByCurrentService(queueKey.getTenantId())) { - var consumer = Optional.ofNullable(consumers.get(queueKey)).orElseGet(() -> createConsumer(queueKey)); - if (consumer != null) { - stateService.restore(queueKey, partitions); - } - } + if (DataConstants.CF_QUEUE_NAME.equals(queueKey.getQueueName())) { + stateService.restore(queueKey, partitions); } }); - consumers.keySet().stream() - .collect(Collectors.groupingBy(QueueKey::getTenantId)) - .forEach((tenantId, queueKeys) -> { - if (!partitionService.isManagedByCurrentService(tenantId)) { - queueKeys.forEach(queueKey -> { - Optional.ofNullable(consumers.remove(queueKey)).ifPresent(MainQueueConsumerManager::stop); - }); - } - }); - // Cleanup old entities after corresponding consumers are stopped. // Any periodic tasks need to check that the entity is still managed by the current server before processing. actorContext.tell(new CalculatedFieldPartitionChangeMsg()); @@ -265,19 +232,13 @@ public class DefaultTbCalculatedFieldConsumerService extends AbstractPartitionBa if (event.getEvent() == ComponentLifecycleEvent.DELETED) { entityProfileCache.removeTenant(event.getTenantId()); - List toRemove = consumers.keySet().stream() - .filter(queueKey -> queueKey.getTenantId().equals(event.getTenantId())) - .toList(); - toRemove.forEach(queueKey -> { - Optional.ofNullable(consumers.remove(queueKey)).ifPresent(consumer -> { - Set partitions = stateService.getPartitions().stream() - .filter(tpi -> tpi.getTenantId().isPresent() && tpi.getTenantId().get().equals(event.getTenantId())) - .collect(Collectors.toSet()); - if (!partitions.isEmpty()) { - consumer.delete(partitions); - } - }); - }); + Set partitions = stateService.getPartitions(); + if (CollectionUtils.isEmpty(partitions)) { + return; + } + stateService.delete(partitions.stream() + .filter(tpi -> tpi.getTenantId().isPresent() && tpi.getTenantId().get().equals(event.getTenantId())) + .collect(Collectors.toSet())); } } else if (event.getEntityId().getEntityType() == EntityType.ASSET_PROFILE) { if (event.getEvent() == ComponentLifecycleEvent.DELETED) { diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java index a3003ba6ff..17decae3a0 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java @@ -206,7 +206,7 @@ public class DefaultTbCoreConsumerService extends AbstractConsumerService queueFactory.createToCoreMsgConsumer()) + .consumerCreator((config, tpi) -> queueFactory.createToCoreMsgConsumer()) .consumerExecutor(consumersExecutor) .scheduler(scheduler) .taskExecutor(mgmtExecutor) diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbEdgeConsumerService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbEdgeConsumerService.java index fdaa2103e2..7415b0bdad 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbEdgeConsumerService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbEdgeConsumerService.java @@ -19,7 +19,6 @@ import com.google.common.util.concurrent.FutureCallback; import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; import com.google.common.util.concurrent.MoreExecutors; -import lombok.Data; import lombok.extern.slf4j.Slf4j; import org.checkerframework.checker.nullness.qual.Nullable; import org.jetbrains.annotations.NotNull; @@ -45,12 +44,12 @@ import org.thingsboard.server.gen.transport.TransportProtos.ToEdgeMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToEdgeNotificationMsg; import org.thingsboard.server.queue.TbQueueConsumer; import org.thingsboard.server.queue.common.TbProtoQueueMsg; +import org.thingsboard.server.queue.common.consumer.MainQueueConsumerManager; import org.thingsboard.server.queue.discovery.QueueKey; import org.thingsboard.server.queue.discovery.event.PartitionChangeEvent; import org.thingsboard.server.queue.provider.TbCoreQueueFactory; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.edge.EdgeContextComponent; -import org.thingsboard.server.queue.common.consumer.MainQueueConsumerManager; import org.thingsboard.server.service.queue.processing.AbstractConsumerService; import org.thingsboard.server.service.queue.processing.IdMsgPair; @@ -104,7 +103,7 @@ public class DefaultTbEdgeConsumerService extends AbstractConsumerService queueFactory.createEdgeMsgConsumer()) + .consumerCreator((config, tpi) -> queueFactory.createEdgeMsgConsumer()) .consumerExecutor(consumersExecutor) .scheduler(scheduler) .taskExecutor(mgmtExecutor) diff --git a/application/src/main/java/org/thingsboard/server/service/queue/ruleengine/TbRuleEngineQueueConsumerManager.java b/application/src/main/java/org/thingsboard/server/service/queue/ruleengine/TbRuleEngineQueueConsumerManager.java index c7f7f600a7..c147836dab 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/ruleengine/TbRuleEngineQueueConsumerManager.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/ruleengine/TbRuleEngineQueueConsumerManager.java @@ -72,7 +72,12 @@ public class TbRuleEngineQueueConsumerManager extends MainQueueConsumerManager { + Integer partitionId = topicPartitionInfo.getPartition().orElse(-1); + return ctx.getQueueFactory().createToRuleEngineMsgConsumer(queueConfig, partitionId); + }, + consumerExecutor, scheduler, taskExecutor, null); this.ctx = ctx; this.stats = new TbRuleEngineConsumerStats(queueKey, ctx.getStatsFactory()); } diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/processor/EdqsProcessor.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/processor/EdqsProcessor.java index 07575220eb..78d17ef368 100644 --- a/common/edqs/src/main/java/org/thingsboard/server/edqs/processor/EdqsProcessor.java +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/processor/EdqsProcessor.java @@ -139,7 +139,7 @@ public class EdqsProcessor implements TbQueueHandler, } consumer.commit(); }) - .consumerCreator((config, partitionId) -> queueFactory.createEdqsEventsConsumer()) + .consumerCreator((config, tpi) -> queueFactory.createEdqsEventsConsumer()) .queueAdmin(queueFactory.getEdqsQueueAdmin()) .consumerExecutor(consumersExecutor) .taskExecutor(taskExecutor) diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/state/KafkaEdqsStateService.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/state/KafkaEdqsStateService.java index efdb1ead1c..0efe6e7d3b 100644 --- a/common/edqs/src/main/java/org/thingsboard/server/edqs/state/KafkaEdqsStateService.java +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/state/KafkaEdqsStateService.java @@ -59,7 +59,8 @@ public class KafkaEdqsStateService implements EdqsStateService { private final EdqsConfig config; private final EdqsPartitionService partitionService; private final KafkaEdqsQueueFactory queueFactory; - @Autowired @Lazy + @Autowired + @Lazy private EdqsProcessor edqsProcessor; private PartitionedQueueConsumerManager> stateConsumer; @@ -93,7 +94,7 @@ public class KafkaEdqsStateService implements EdqsStateService { } consumer.commit(); }) - .consumerCreator((config, partitionId) -> queueFactory.createEdqsStateConsumer()) + .consumerCreator((config, tpi) -> queueFactory.createEdqsStateConsumer()) .queueAdmin(queueAdmin) .consumerExecutor(eventConsumer.getConsumerExecutor()) .taskExecutor(eventConsumer.getTaskExecutor()) diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/common/consumer/MainQueueConsumerManager.java b/common/queue/src/main/java/org/thingsboard/server/queue/common/consumer/MainQueueConsumerManager.java index ef9728344c..2233855a37 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/common/consumer/MainQueueConsumerManager.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/common/consumer/MainQueueConsumerManager.java @@ -54,7 +54,7 @@ public class MainQueueConsumerManager msgPackProcessor; - protected final BiFunction> consumerCreator; + protected final BiFunction> consumerCreator; @Getter protected final ExecutorService consumerExecutor; @Getter @@ -74,7 +74,7 @@ public class MainQueueConsumerManager msgPackProcessor, - BiFunction> consumerCreator, + BiFunction> consumerCreator, ExecutorService consumerExecutor, ScheduledExecutorService scheduler, ExecutorService taskExecutor, @@ -313,7 +313,7 @@ public class MainQueueConsumerManager onStop.accept(tpi) : null; TbQueueConsumerTask consumer = new TbQueueConsumerTask<>(key, () -> { - TbQueueConsumer queueConsumer = consumerCreator.apply(config, partitionId); + TbQueueConsumer queueConsumer = consumerCreator.apply(config, tpi); if (startOffsetProvider != null && queueConsumer instanceof TbKafkaConsumerTemplate kafkaConsumer) { kafkaConsumer.setStartOffsetProvider(startOffsetProvider); } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/common/consumer/PartitionedQueueConsumerManager.java b/common/queue/src/main/java/org/thingsboard/server/queue/common/consumer/PartitionedQueueConsumerManager.java index 0de1e53753..1b19fcab0e 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/common/consumer/PartitionedQueueConsumerManager.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/common/consumer/PartitionedQueueConsumerManager.java @@ -45,7 +45,7 @@ public class PartitionedQueueConsumerManager extends MainQ @Builder(builderMethodName = "create") // not to conflict with super.builder() public PartitionedQueueConsumerManager(QueueKey queueKey, String topic, long pollInterval, MsgPackProcessor msgPackProcessor, - BiFunction> consumerCreator, TbQueueAdmin queueAdmin, + BiFunction> consumerCreator, TbQueueAdmin queueAdmin, ExecutorService consumerExecutor, ScheduledExecutorService scheduler, ExecutorService taskExecutor, Consumer uncaughtErrorHandler) { super(queueKey, QueueConfig.of(true, pollInterval), msgPackProcessor, consumerCreator, consumerExecutor, scheduler, taskExecutor, uncaughtErrorHandler); diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/InMemoryMonolithQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/InMemoryMonolithQueueFactory.java index 9fbd75d347..085d04f28c 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/InMemoryMonolithQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/InMemoryMonolithQueueFactory.java @@ -134,7 +134,7 @@ public class InMemoryMonolithQueueFactory implements TbCoreQueueFactory, TbRuleE } @Override - public TbQueueConsumer> createToCalculatedFieldMsgConsumer(TopicPartitionInfo tpi, Integer partitionId) { + public TbQueueConsumer> createToCalculatedFieldMsgConsumer(TopicPartitionInfo tpi) { return new InMemoryTbQueueConsumer<>(storage, topicService.buildTopicName(calculatedFieldSettings.getEventTopic())); } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaMonolithQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaMonolithQueueFactory.java index 198e2f7a83..1c46a624c3 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaMonolithQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaMonolithQueueFactory.java @@ -105,7 +105,7 @@ public class KafkaMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEngi private final TbQueueAdmin housekeeperReprocessingAdmin; private final TbQueueAdmin edgeAdmin; private final TbQueueAdmin edgeEventAdmin; - private final TbKafkaAdmin cfAdmin; + private final TbQueueAdmin cfAdmin; private final TbQueueAdmin cfStateAdmin; private final TbQueueAdmin edqsEventsAdmin; private final TbKafkaAdmin edqsRequestsAdmin; @@ -515,9 +515,10 @@ public class KafkaMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEngi } @Override - public TbQueueConsumer> createToCalculatedFieldMsgConsumer(TopicPartitionInfo tpi, Integer partitionId) { + public TbQueueConsumer> createToCalculatedFieldMsgConsumer(TopicPartitionInfo tpi) { String queueName = DataConstants.CF_QUEUE_NAME; TenantId tenantId = tpi.getTenantId().orElse(TenantId.SYS_TENANT_ID); + Integer partitionId = tpi.getPartition().orElseThrow(() -> new IllegalArgumentException("PartitionId is required.")); String groupId = topicService.buildConsumerGroupId("cf-", tenantId, queueName, partitionId); TbKafkaConsumerTemplate.TbKafkaConsumerTemplateBuilder> consumerBuilder = TbKafkaConsumerTemplate.builder(); diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbRuleEngineQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbRuleEngineQueueFactory.java index f387d1af11..112af9f646 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbRuleEngineQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbRuleEngineQueueFactory.java @@ -93,7 +93,7 @@ public class KafkaTbRuleEngineQueueFactory implements TbRuleEngineQueueFactory { private final TbQueueAdmin housekeeperAdmin; private final TbQueueAdmin edgeAdmin; private final TbQueueAdmin edgeEventAdmin; - private final TbKafkaAdmin cfAdmin; + private final TbQueueAdmin cfAdmin; private final TbQueueAdmin cfStateAdmin; private final TbQueueAdmin edqsEventsAdmin; private final AtomicLong consumerCount = new AtomicLong(); @@ -316,9 +316,10 @@ public class KafkaTbRuleEngineQueueFactory implements TbRuleEngineQueueFactory { } @Override - public TbQueueConsumer> createToCalculatedFieldMsgConsumer(TopicPartitionInfo tpi, Integer partitionId) { + public TbQueueConsumer> createToCalculatedFieldMsgConsumer(TopicPartitionInfo tpi) { String queueName = DataConstants.CF_QUEUE_NAME; TenantId tenantId = tpi.getTenantId().orElse(TenantId.SYS_TENANT_ID); + Integer partitionId = tpi.getPartition().orElseThrow(() -> new IllegalArgumentException("PartitionId is required.")); String groupId = topicService.buildConsumerGroupId("cf-", tenantId, queueName, partitionId); TbKafkaConsumerTemplate.TbKafkaConsumerTemplateBuilder> consumerBuilder = TbKafkaConsumerTemplate.builder(); diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbRuleEngineQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbRuleEngineQueueFactory.java index 3980aaa632..18bb6db14a 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbRuleEngineQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbRuleEngineQueueFactory.java @@ -122,7 +122,7 @@ public interface TbRuleEngineQueueFactory extends TbUsageStatsClientQueueFactory TbQueueRequestTemplate, TbProtoQueueMsg> createRemoteJsRequestTemplate(); - TbQueueConsumer> createToCalculatedFieldMsgConsumer(TopicPartitionInfo tpi, Integer partitionId); + TbQueueConsumer> createToCalculatedFieldMsgConsumer(TopicPartitionInfo tpi); TbQueueAdmin getCalculatedFieldQueueAdmin(); From 7b57316b5f53b58ed938e03ff274b41cb321f403 Mon Sep 17 00:00:00 2001 From: deaflynx Date: Tue, 1 Apr 2025 13:22:12 +0300 Subject: [PATCH 13/47] UI: Fix erase of externalId on save widget. --- .../src/app/modules/home/models/widget-component.models.ts | 5 ++++- ui-ngx/src/app/shared/models/widget.models.ts | 2 +- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/ui-ngx/src/app/modules/home/models/widget-component.models.ts b/ui-ngx/src/app/modules/home/models/widget-component.models.ts index 00afa2b68d..aee890a5b2 100644 --- a/ui-ngx/src/app/modules/home/models/widget-component.models.ts +++ b/ui-ngx/src/app/modules/home/models/widget-component.models.ts @@ -115,6 +115,7 @@ import { DataKeySettingsFunction } from '@home/components/widget/lib/settings/co import { UtilsService } from '@core/services/utils.service'; import { CompiledTbFunction } from '@shared/models/js-function.models'; import { FormProperty } from '@shared/models/dynamic-form.models'; +import { ExportableEntity } from '@shared/models/base-data'; export interface IWidgetAction { name: string; @@ -573,7 +574,7 @@ export interface IDynamicWidgetComponent { [key: string]: any; } -export interface WidgetInfo extends WidgetTypeDescriptor, WidgetControllerDescriptor { +export interface WidgetInfo extends WidgetTypeDescriptor, WidgetControllerDescriptor, ExportableEntity { widgetName: string; fullFqn: string; deprecated: boolean; @@ -681,6 +682,7 @@ export const toWidgetInfo = (widgetTypeEntity: WidgetType): WidgetInfo => ({ fullFqn: fullWidgetTypeFqn(widgetTypeEntity), deprecated: widgetTypeEntity.deprecated, scada: widgetTypeEntity.scada, + externalId: widgetTypeEntity.externalId, type: widgetTypeEntity.descriptor.type, sizeX: widgetTypeEntity.descriptor.sizeX, sizeY: widgetTypeEntity.descriptor.sizeY, @@ -736,6 +738,7 @@ export const toWidgetType = (widgetInfo: WidgetInfo, id: WidgetTypeId, tenantId: name: widgetInfo.widgetName, deprecated: widgetInfo.deprecated, scada: widgetInfo.scada, + externalId: widgetInfo.externalId, descriptor }; }; diff --git a/ui-ngx/src/app/shared/models/widget.models.ts b/ui-ngx/src/app/shared/models/widget.models.ts index ec55d4e8a3..c55dcdee37 100644 --- a/ui-ngx/src/app/shared/models/widget.models.ts +++ b/ui-ngx/src/app/shared/models/widget.models.ts @@ -206,7 +206,7 @@ export interface WidgetControllerDescriptor { actionSources?: {[actionSourceId: string]: WidgetActionSource}; } -export interface BaseWidgetType extends BaseData, HasTenantId, HasVersion { +export interface BaseWidgetType extends BaseData, HasTenantId, HasVersion, ExportableEntity { tenantId: TenantId; fqn: string; name: string; From ac3b389203abfeafbbb2959dcd8e8de71a13fc0c Mon Sep 17 00:00:00 2001 From: Vladyslav_Prykhodko Date: Thu, 10 Apr 2025 10:51:11 +0300 Subject: [PATCH 14/47] UI: Add Echart in modules map --- ui-ngx/src/app/modules/common/modules-map.ts | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/ui-ngx/src/app/modules/common/modules-map.ts b/ui-ngx/src/app/modules/common/modules-map.ts index 70432e8cdf..530a41f006 100644 --- a/ui-ngx/src/app/modules/common/modules-map.ts +++ b/ui-ngx/src/app/modules/common/modules-map.ts @@ -76,6 +76,8 @@ import * as TranslateCore from '@ngx-translate/core'; import * as MatDateTimePicker from '@mat-datetimepicker/core'; import _moment from 'moment'; import * as tslib from 'tslib'; +import * as Echarts from 'echarts'; +import * as EchartsCore from 'echarts/core'; import * as TbCore from '@core/public-api'; import * as TbShared from '@shared/public-api'; @@ -405,6 +407,8 @@ class ModulesMap implements IModulesMap { '@mat-datetimepicker/core': MatDateTimePicker, moment: _moment, tslib, + 'echarts': Echarts, + 'echarts/core': EchartsCore, '@core/public-api': TbCore, '@shared/public-api': TbShared, From ced71331efa6e3e840b557a0b555e1d786ac724c Mon Sep 17 00:00:00 2001 From: deaflynx Date: Tue, 1 Apr 2025 12:04:26 +0300 Subject: [PATCH 15/47] UI: JavaScript library (Module type) minor style enhancement: remove redundant space before required symbol, decrease help popover width (prod-5636). --- .../home/pages/admin/resource/js-resource.component.html | 3 +-- ui-ngx/src/app/shared/components/js-func.component.scss | 4 ++++ .../src/assets/help/en_US/resource/js-resource-module_fn.md | 3 ++- 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/ui-ngx/src/app/modules/home/pages/admin/resource/js-resource.component.html b/ui-ngx/src/app/modules/home/pages/admin/resource/js-resource.component.html index 6287d27f2c..15d899df6b 100644 --- a/ui-ngx/src/app/modules/home/pages/admin/resource/js-resource.component.html +++ b/ui-ngx/src/app/modules/home/pages/admin/resource/js-resource.component.html @@ -91,8 +91,7 @@ + {{ 'javascript.module-script' | translate }} +that are exported for use in other parts of the application. ##### Examples You can declare variables: From 713050937c50151eb5c87bb65426f2071326835f Mon Sep 17 00:00:00 2001 From: deaflynx Date: Thu, 10 Apr 2025 10:53:09 +0300 Subject: [PATCH 16/47] UI: JavaScript resource help popup too wide width issue: use [helpPopupStyle]. --- .../home/pages/admin/resource/js-resource.component.html | 1 + ui-ngx/src/assets/help/en_US/resource/js-resource-module_fn.md | 3 +-- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/ui-ngx/src/app/modules/home/pages/admin/resource/js-resource.component.html b/ui-ngx/src/app/modules/home/pages/admin/resource/js-resource.component.html index 15d899df6b..62036a8ea1 100644 --- a/ui-ngx/src/app/modules/home/pages/admin/resource/js-resource.component.html +++ b/ui-ngx/src/app/modules/home/pages/admin/resource/js-resource.component.html @@ -80,6 +80,7 @@ (fileNameChanged)="entityForm?.get('fileName').patchValue($event)"> -that are exported for use in other parts of the application. +These modules can contain any JavaScript code, facilitating the reuse of specific logic. This includes variables or functions that are exported for use in other parts of the application. ##### Examples You can declare variables: From 59fa19d1c861d10aca3a44073a1786c10aab0d9c Mon Sep 17 00:00:00 2001 From: ViacheslavKlimov Date: Thu, 10 Apr 2025 11:40:19 +0300 Subject: [PATCH 17/47] Change EDQS consumers poll interval from 125 ms to 25 ms --- application/src/main/resources/thingsboard.yml | 2 +- .../main/java/org/thingsboard/server/queue/edqs/EdqsConfig.java | 2 +- edqs/src/main/resources/edqs.yml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/application/src/main/resources/thingsboard.yml b/application/src/main/resources/thingsboard.yml index b4d178a737..42f5331e21 100644 --- a/application/src/main/resources/thingsboard.yml +++ b/application/src/main/resources/thingsboard.yml @@ -1773,7 +1773,7 @@ queue: # EDQS responses topic responses_topic: "${TB_EDQS_RESPONSES_TOPIC:edqs.responses}" # Poll interval for EDQS topics - poll_interval: "${TB_EDQS_POLL_INTERVAL_MS:125}" + poll_interval: "${TB_EDQS_POLL_INTERVAL_MS:25}" # Maximum amount of pending requests to EDQS max_pending_requests: "${TB_EDQS_MAX_PENDING_REQUESTS:10000}" # Maximum timeout for requests to EDQS diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/edqs/EdqsConfig.java b/common/queue/src/main/java/org/thingsboard/server/queue/edqs/EdqsConfig.java index 401b451f59..3c927f135b 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/edqs/EdqsConfig.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/edqs/EdqsConfig.java @@ -38,7 +38,7 @@ public class EdqsConfig { private String requestsTopic; @Value("${queue.edqs.responses_topic:edqs.responses}") private String responsesTopic; - @Value("${queue.edqs.poll_interval:125}") + @Value("${queue.edqs.poll_interval:25}") private long pollInterval; @Value("${queue.edqs.max_pending_requests:10000}") private int maxPendingRequests; diff --git a/edqs/src/main/resources/edqs.yml b/edqs/src/main/resources/edqs.yml index 2c0e0b8c5d..8ea40c03c8 100644 --- a/edqs/src/main/resources/edqs.yml +++ b/edqs/src/main/resources/edqs.yml @@ -66,7 +66,7 @@ queue: # EDQS responses topic responses_topic: "${TB_EDQS_RESPONSES_TOPIC:edqs.responses}" # Poll interval for EDQS topics - poll_interval: "${TB_EDQS_POLL_INTERVAL_MS:125}" + poll_interval: "${TB_EDQS_POLL_INTERVAL_MS:25}" # Maximum amount of pending requests to EDQS max_pending_requests: "${TB_EDQS_MAX_PENDING_REQUESTS:10000}" # Maximum timeout for requests to EDQS From d1f0ec5033774f02b9cf094d3eb42cb5f8bd1e79 Mon Sep 17 00:00:00 2001 From: ViacheslavKlimov Date: Thu, 10 Apr 2025 11:41:19 +0300 Subject: [PATCH 18/47] Additionally check for string compression threshold on EDQS side --- .../server/edqs/util/EdqsConverter.java | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/util/EdqsConverter.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/util/EdqsConverter.java index 167037b889..18e2f521e3 100644 --- a/common/edqs/src/main/java/org/thingsboard/server/edqs/util/EdqsConverter.java +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/util/EdqsConverter.java @@ -171,11 +171,21 @@ public class EdqsConverter { } else if (proto.hasDoubleV()) { return new DoubleDataPoint(ts, proto.getDoubleV()); } else if (proto.hasStringV()) { - return new StringDataPoint(ts, proto.getStringV()); + String stringV = proto.getStringV(); + if (stringV.length() < stringCompressionLengthThreshold) { + return new StringDataPoint(ts, stringV); + } else { + return new CompressedStringDataPoint(ts, compress(stringV)); + } } else if (proto.hasCompressedStringV()) { return new CompressedStringDataPoint(ts, proto.getCompressedStringV().toByteArray()); } else if (proto.hasJsonV()) { - return new JsonDataPoint(ts, proto.getJsonV()); + String jsonV = proto.getJsonV(); + if (jsonV.length() < stringCompressionLengthThreshold) { + return new JsonDataPoint(ts, jsonV); + } else { + return new CompressedJsonDataPoint(ts, compress(jsonV)); + } } else if (proto.hasCompressedJsonV()) { return new CompressedJsonDataPoint(ts, proto.getCompressedJsonV().toByteArray()); } else { From 1ee635ddd8ca11286fb20018e1248464af2c0a2d Mon Sep 17 00:00:00 2001 From: Dmytro Skarzhynets Date: Thu, 10 Apr 2025 12:19:37 +0300 Subject: [PATCH 19/47] Improve hint for processing settings block in save nodes --- ui-ngx/src/assets/locale/locale.constant-en_US.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ui-ngx/src/assets/locale/locale.constant-en_US.json b/ui-ngx/src/assets/locale/locale.constant-en_US.json index 2f93a48aa4..27e2e319d7 100644 --- a/ui-ngx/src/assets/locale/locale.constant-en_US.json +++ b/ui-ngx/src/assets/locale/locale.constant-en_US.json @@ -5258,7 +5258,7 @@ "advanced-mode": "Advanced", "save-time-series": { "processing-settings": "Processing settings", - "processing-settings-hint": "Define how incoming messages are processed. In Basic mode, select a preconfigured processing strategy or enable only WebSocket updates. Advanced mode allows you to select individual processing strategies for each action.", + "processing-settings-hint": "Define how incoming messages are processed. Basic processing settings allow you to select preconfigured strategies, while Advanced settings allow you to select individual processing strategies for each action.", "advanced-settings-hint": "Be cautious when configuring processing strategies. Certain combinations can lead to unexpected behavior.", "strategy": "Strategy", "deduplication-interval": "Deduplication interval", @@ -5277,7 +5277,7 @@ }, "save-attribute": { "processing-settings": "Processing settings", - "processing-settings-hint": "Define how incoming messages are processed. In Basic mode, select a preconfigured processing strategy or enable only WebSocket updates. Advanced mode allows you to select individual processing strategies for each action.", + "processing-settings-hint": "Define how incoming messages are processed. Basic processing settings allow you to select preconfigured strategies, while Advanced settings allow you to select individual processing strategies for each action.", "advanced-settings-hint": "Be cautious when configuring processing strategies. Certain combinations can lead to unexpected behavior.", "strategy": "Strategy", "deduplication-interval": "Deduplication interval", From b806a41e62d473df72a1e2c04b1536d29690a717 Mon Sep 17 00:00:00 2001 From: ViacheslavKlimov Date: Thu, 10 Apr 2025 14:02:56 +0300 Subject: [PATCH 20/47] More stats for EDQS --- .../server/edqs/data/BaseEntityData.java | 2 +- .../edqs/data/dp/CompressedJsonDataPoint.java | 6 +- .../data/dp/CompressedStringDataPoint.java | 10 +- .../edqs/repo/DefaultEdqsRepository.java | 9 +- .../server/edqs/repo/TenantRepo.java | 2 - .../edqs/stats/DefaultEdqsStatsService.java | 93 ++++++++++++++----- .../server/edqs/util/EdqsConverter.java | 29 ++++-- .../common/stats/DefaultStatsFactory.java | 11 +++ .../common/stats/DummyEdqsStatsService.java | 16 +++- .../server/common/stats/EdqsStatsService.java | 12 ++- .../server/common/stats/StatsFactory.java | 6 ++ .../thingsboard/common/util/TbBytePool.java | 6 +- .../thingsboard/common/util/TbStringPool.java | 6 +- .../server/dao/entity/BaseEntityService.java | 9 +- edqs/src/main/resources/edqs.yml | 2 + 15 files changed, 158 insertions(+), 61 deletions(-) diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/data/BaseEntityData.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/data/BaseEntityData.java index 33f32b9781..21b9ff0c4f 100644 --- a/common/edqs/src/main/java/org/thingsboard/server/edqs/data/BaseEntityData.java +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/data/BaseEntityData.java @@ -125,7 +125,7 @@ public abstract class BaseEntityData implements EntityDa return switch (key) { case "createdTime" -> new LongDataPoint(System.currentTimeMillis(), fields.getCreatedTime()); case "edgeTemplate" -> new BoolDataPoint(System.currentTimeMillis(), fields.isEdgeTemplate()); - case "parentId" -> new StringDataPoint(System.currentTimeMillis(), getRelatedParentId(ctx)); + case "parentId" -> new StringDataPoint(System.currentTimeMillis(), getRelatedParentId(ctx), false); default -> new StringDataPoint(System.currentTimeMillis(), getField(key), false); }; } diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/data/dp/CompressedJsonDataPoint.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/data/dp/CompressedJsonDataPoint.java index bce9d86875..c05a724e79 100644 --- a/common/edqs/src/main/java/org/thingsboard/server/edqs/data/dp/CompressedJsonDataPoint.java +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/data/dp/CompressedJsonDataPoint.java @@ -17,10 +17,12 @@ package org.thingsboard.server.edqs.data.dp; import org.thingsboard.server.common.data.kv.DataType; +import java.util.function.Function; + public class CompressedJsonDataPoint extends CompressedStringDataPoint { - public CompressedJsonDataPoint(long ts, byte[] compressedValue) { - super(ts, compressedValue); + public CompressedJsonDataPoint(long ts, byte[] compressedValue, Function uncompressor) { + super(ts, compressedValue, uncompressor); } @Override diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/data/dp/CompressedStringDataPoint.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/data/dp/CompressedStringDataPoint.java index cf4267e443..45db6bf72a 100644 --- a/common/edqs/src/main/java/org/thingsboard/server/edqs/data/dp/CompressedStringDataPoint.java +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/data/dp/CompressedStringDataPoint.java @@ -19,17 +19,21 @@ import lombok.Getter; import lombok.SneakyThrows; import org.thingsboard.common.util.TbBytePool; import org.thingsboard.server.common.data.kv.DataType; -import org.xerial.snappy.Snappy; + +import java.util.function.Function; public class CompressedStringDataPoint extends AbstractDataPoint { @Getter private final byte[] compressedValue; + protected final Function uncompressor; + @SneakyThrows - public CompressedStringDataPoint(long ts, byte[] compressedValue) { + public CompressedStringDataPoint(long ts, byte[] compressedValue, Function uncompressor) { super(ts); this.compressedValue = TbBytePool.intern(compressedValue); + this.uncompressor = uncompressor; } @Override @@ -40,7 +44,7 @@ public class CompressedStringDataPoint extends AbstractDataPoint { @SneakyThrows @Override public String getStr() { - return Snappy.uncompressString(compressedValue); + return uncompressor.apply(compressedValue); } @Override diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/repo/DefaultEdqsRepository.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/repo/DefaultEdqsRepository.java index 215c64194f..88e4cc63d2 100644 --- a/common/edqs/src/main/java/org/thingsboard/server/edqs/repo/DefaultEdqsRepository.java +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/repo/DefaultEdqsRepository.java @@ -16,6 +16,7 @@ package org.thingsboard.server.edqs.repo; import lombok.AllArgsConstructor; +import lombok.Getter; import lombok.extern.slf4j.Slf4j; import org.springframework.stereotype.Service; import org.thingsboard.server.common.data.ObjectType; @@ -40,6 +41,7 @@ import java.util.function.Predicate; @Slf4j public class DefaultEdqsRepository implements EdqsRepository { + @Getter private final static ConcurrentMap repos = new ConcurrentHashMap<>(); private final EdqsStatsService statsService; @@ -52,6 +54,7 @@ public class DefaultEdqsRepository implements EdqsRepository { if (event.getEventType() == EdqsEventType.DELETED && event.getObjectType() == ObjectType.TENANT) { log.info("Tenant {} deleted", event.getTenantId()); repos.remove(event.getTenantId()); + statsService.reportRemoved(ObjectType.TENANT); } else { get(event.getTenantId()).processEvent(event); } @@ -61,8 +64,7 @@ public class DefaultEdqsRepository implements EdqsRepository { public long countEntitiesByQuery(TenantId tenantId, CustomerId customerId, EntityCountQuery query, boolean ignorePermissionCheck) { long startNs = System.nanoTime(); long result = get(tenantId).countEntitiesByQuery(customerId, query, ignorePermissionCheck); - double timingMs = (double) (System.nanoTime() - startNs) / 1000_000; - log.info("countEntitiesByQuery done in {} ms", timingMs); + statsService.reportEdqsCountQuery(tenantId, query, System.nanoTime() - startNs); return result; } @@ -71,8 +73,7 @@ public class DefaultEdqsRepository implements EdqsRepository { EntityDataQuery query, boolean ignorePermissionCheck) { long startNs = System.nanoTime(); var result = get(tenantId).findEntityDataByQuery(customerId, query, ignorePermissionCheck); - double timingMs = (double) (System.nanoTime() - startNs) / 1000_000; - log.info("findEntityDataByQuery done in {} ms", timingMs); + statsService.reportEdqsDataQuery(tenantId, query, System.nanoTime() - startNs); return result; } diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/repo/TenantRepo.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/repo/TenantRepo.java index a47559d6d8..ffba7583c9 100644 --- a/common/edqs/src/main/java/org/thingsboard/server/edqs/repo/TenantRepo.java +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/repo/TenantRepo.java @@ -332,7 +332,6 @@ public class TenantRepo { public PageData findEntityDataByQuery(CustomerId customerId, EntityDataQuery oldQuery, boolean ignorePermissionCheck) { EdqsDataQuery query = RepositoryUtils.toNewQuery(oldQuery); - log.info("[{}][{}] findEntityDataByQuery: {}", tenantId, customerId, query); QueryContext ctx = buildContext(customerId, query.getEntityFilter(), ignorePermissionCheck); EntityQueryProcessor queryProcessor = EntityQueryProcessorFactory.create(this, ctx, query); return sortAndConvert(query, queryProcessor.processQuery(), ctx); @@ -340,7 +339,6 @@ public class TenantRepo { public long countEntitiesByQuery(CustomerId customerId, EntityCountQuery oldQuery, boolean ignorePermissionCheck) { EdqsQuery query = RepositoryUtils.toNewQuery(oldQuery); - log.info("[{}][{}] countEntitiesByQuery: {}", tenantId, customerId, query); QueryContext ctx = buildContext(customerId, query.getEntityFilter(), ignorePermissionCheck); EntityQueryProcessor queryProcessor = EntityQueryProcessorFactory.create(this, ctx, query); return queryProcessor.count(); diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/stats/DefaultEdqsStatsService.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/stats/DefaultEdqsStatsService.java index 3767d0f60b..46fb153771 100644 --- a/common/edqs/src/main/java/org/thingsboard/server/edqs/stats/DefaultEdqsStatsService.java +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/stats/DefaultEdqsStatsService.java @@ -15,78 +15,123 @@ */ package org.thingsboard.server.edqs.stats; +import jakarta.annotation.PostConstruct; +import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Value; import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; import org.springframework.stereotype.Service; +import org.thingsboard.common.util.TbBytePool; +import org.thingsboard.common.util.TbStringPool; import org.thingsboard.server.common.data.ObjectType; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.query.EntityCountQuery; import org.thingsboard.server.common.data.query.EntityDataQuery; import org.thingsboard.server.common.stats.EdqsStatsService; +import org.thingsboard.server.common.stats.StatsCounter; import org.thingsboard.server.common.stats.StatsFactory; import org.thingsboard.server.common.stats.StatsTimer; import org.thingsboard.server.common.stats.StatsType; +import org.thingsboard.server.edqs.repo.DefaultEdqsRepository; +import java.util.Map; import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; @Service @Slf4j +@RequiredArgsConstructor @ConditionalOnExpression("'${queue.edqs.api.supported:true}' == 'true' && '${queue.edqs.stats.enabled:true}' == 'true'") public class DefaultEdqsStatsService implements EdqsStatsService { private final StatsFactory statsFactory; - @Value("${queue.edqs.stats.slow_query_threshold:3000}") + @Value("${queue.edqs.stats.slow_query_threshold}") private int slowQueryThreshold; - private final ConcurrentHashMap objectCounters = new ConcurrentHashMap<>(); - private final StatsTimer dataQueryTimer; - private final StatsTimer countQueryTimer; + private final ConcurrentMap objectCounters = new ConcurrentHashMap<>(); + private final ConcurrentMap timers = new ConcurrentHashMap<>(); + private final ConcurrentMap counters = new ConcurrentHashMap<>(); + private final ConcurrentMap gauges = new ConcurrentHashMap<>(); - private DefaultEdqsStatsService(StatsFactory statsFactory) { - this.statsFactory = statsFactory; - dataQueryTimer = statsFactory.createTimer(StatsType.EDQS, "entityDataQueryTimer"); - countQueryTimer = statsFactory.createTimer(StatsType.EDQS, "entityCountQueryTimer"); + @PostConstruct + private void init() { + statsFactory.createGauge(StatsType.EDQS, "stringPoolSize", TbStringPool.getPool(), Map::size); + statsFactory.createGauge(StatsType.EDQS, "bytePoolSize", TbBytePool.getPool(), Map::size); + statsFactory.createGauge(StatsType.EDQS, "tenantReposSize", DefaultEdqsRepository.getRepos(), Map::size); } @Override public void reportAdded(ObjectType objectType) { - getObjectCounter(objectType).incrementAndGet(); + getObjectGauge(objectType).incrementAndGet(); } @Override public void reportRemoved(ObjectType objectType) { - getObjectCounter(objectType).decrementAndGet(); + getObjectGauge(objectType).decrementAndGet(); } @Override - public void reportDataQuery(TenantId tenantId, EntityDataQuery query, long timingNanos) { - double timingMs = timingNanos / 1000_000.0; - if (timingMs < slowQueryThreshold) { - log.debug("[{}] Executed data query in {} ms: {}", tenantId, timingMs, query); - } else { - log.warn("[{}] Executed slow data query in {} ms: {}", tenantId, timingMs, query); - } - dataQueryTimer.record(timingNanos, TimeUnit.NANOSECONDS); + public void reportEntityDataQuery(TenantId tenantId, EntityDataQuery query, long timingNanos) { + checkTiming(tenantId, query, timingNanos); + getTimer("entityDataQueryTimer").record(timingNanos, TimeUnit.NANOSECONDS); + } + + @Override + public void reportEntityCountQuery(TenantId tenantId, EntityCountQuery query, long timingNanos) { + checkTiming(tenantId, query, timingNanos); + getTimer("entityCountQueryTimer").record(timingNanos, TimeUnit.NANOSECONDS); + } + + @Override + public void reportEdqsDataQuery(TenantId tenantId, EntityDataQuery query, long timingNanos) { + checkTiming(tenantId, query, timingNanos); + getTimer("edqsDataQueryTimer").record(timingNanos, TimeUnit.NANOSECONDS); + } + + @Override + public void reportEdqsCountQuery(TenantId tenantId, EntityCountQuery query, long timingNanos) { + checkTiming(tenantId, query, timingNanos); + getTimer("edqsCountQueryTimer").record(timingNanos, TimeUnit.NANOSECONDS); + } + + @Override + public void reportStringCompressed() { + getCounter("stringsCompressed").increment(); } @Override - public void reportCountQuery(TenantId tenantId, EntityCountQuery query, long timingNanos) { + public void reportStringUncompressed() { + getCounter("stringsUncompressed").increment(); + } + + private void checkTiming(TenantId tenantId, EntityCountQuery query, long timingNanos) { double timingMs = timingNanos / 1000_000.0; + String queryType = query instanceof EntityDataQuery ? "data" : "count"; if (timingMs < slowQueryThreshold) { - log.debug("[{}] Executed count query in {} ms: {}", tenantId, timingMs, query); + log.debug("[{}] Executed " + queryType + " query in {} ms: {}", tenantId, timingMs, query); } else { - log.warn("[{}] Executed slow count query in {} ms: {}", tenantId, timingMs, query); + log.warn("[{}] Executed slow " + queryType + " query in {} ms: {}", tenantId, timingMs, query); } - countQueryTimer.record(timingNanos, TimeUnit.NANOSECONDS); } - private AtomicInteger getObjectCounter(ObjectType objectType) { + private StatsTimer getTimer(String name) { + return timers.computeIfAbsent(name, __ -> statsFactory.createTimer(StatsType.EDQS, name)); + } + + private StatsCounter getCounter(String name) { + return counters.computeIfAbsent(name, __ -> statsFactory.createStatsCounter(StatsType.EDQS.getName(), name)); + } + + private AtomicInteger getGauge(String name) { + return gauges.computeIfAbsent(name, __ -> statsFactory.createGauge(StatsType.EDQS, name, new AtomicInteger())); + } + + private AtomicInteger getObjectGauge(ObjectType objectType) { return objectCounters.computeIfAbsent(objectType, type -> - statsFactory.createGauge("edqsObjectsCount", new AtomicInteger(), "objectType", type.name())); + statsFactory.createGauge(StatsType.EDQS, "objectsCount", new AtomicInteger(), "objectType", type.name())); } } diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/util/EdqsConverter.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/util/EdqsConverter.java index 18e2f521e3..9a84436f36 100644 --- a/common/edqs/src/main/java/org/thingsboard/server/edqs/util/EdqsConverter.java +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/util/EdqsConverter.java @@ -43,6 +43,7 @@ import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.EntityIdFactory; import org.thingsboard.server.common.data.kv.KvEntry; import org.thingsboard.server.common.data.relation.EntityRelation; +import org.thingsboard.server.common.stats.EdqsStatsService; import org.thingsboard.server.common.util.ProtoUtils; import org.thingsboard.server.edqs.data.dp.BoolDataPoint; import org.thingsboard.server.edqs.data.dp.CompressedJsonDataPoint; @@ -56,14 +57,18 @@ import org.thingsboard.server.gen.transport.TransportProtos.DataPointProto; import org.xerial.snappy.Snappy; import java.io.IOException; +import java.nio.charset.StandardCharsets; import java.util.HashMap; import java.util.Map; import java.util.UUID; @Service +@RequiredArgsConstructor @Slf4j public class EdqsConverter { + private final EdqsStatsService edqsStatsService; + @Value("${queue.edqs.string_compression_length_threshold:512}") private int stringCompressionLengthThreshold; @@ -175,32 +180,40 @@ public class EdqsConverter { if (stringV.length() < stringCompressionLengthThreshold) { return new StringDataPoint(ts, stringV); } else { - return new CompressedStringDataPoint(ts, compress(stringV)); + return new CompressedStringDataPoint(ts, compress(stringV), this::uncompress); } } else if (proto.hasCompressedStringV()) { - return new CompressedStringDataPoint(ts, proto.getCompressedStringV().toByteArray()); + return new CompressedStringDataPoint(ts, proto.getCompressedStringV().toByteArray(), this::uncompress); } else if (proto.hasJsonV()) { String jsonV = proto.getJsonV(); if (jsonV.length() < stringCompressionLengthThreshold) { return new JsonDataPoint(ts, jsonV); } else { - return new CompressedJsonDataPoint(ts, compress(jsonV)); + return new CompressedJsonDataPoint(ts, compress(jsonV), this::uncompress); } } else if (proto.hasCompressedJsonV()) { - return new CompressedJsonDataPoint(ts, proto.getCompressedJsonV().toByteArray()); + return new CompressedJsonDataPoint(ts, proto.getCompressedJsonV().toByteArray(), this::uncompress); } else { throw new IllegalArgumentException("Unsupported data point proto: " + proto); } } @SneakyThrows - private static byte[] compress(String value) { - byte[] compressed = Snappy.compress(value); - // TODO: limit the size - log.debug("Compressed {} bytes to {} bytes", value.length(), compressed.length); + private byte[] compress(String value) { + byte[] compressed = Snappy.compress(value, StandardCharsets.UTF_8); + log.debug("Compressed {} chars to {} bytes", value.length(), compressed.length); + edqsStatsService.reportStringCompressed(); return compressed; } + @SneakyThrows + private String uncompress(byte[] compressed) { + String value = Snappy.uncompressString(compressed, StandardCharsets.UTF_8); + log.debug("Uncompressed {} bytes to {} chars", compressed.length, value.length()); + edqsStatsService.reportStringUncompressed(); + return value; + } + public static Entity toEntity(EntityType entityType, Object entity) { Entity edqsEntity = new Entity(); edqsEntity.setType(entityType); diff --git a/common/stats/src/main/java/org/thingsboard/server/common/stats/DefaultStatsFactory.java b/common/stats/src/main/java/org/thingsboard/server/common/stats/DefaultStatsFactory.java index 8e2291270d..15c155ecfd 100644 --- a/common/stats/src/main/java/org/thingsboard/server/common/stats/DefaultStatsFactory.java +++ b/common/stats/src/main/java/org/thingsboard/server/common/stats/DefaultStatsFactory.java @@ -27,6 +27,7 @@ import org.springframework.stereotype.Service; import org.thingsboard.server.common.data.StringUtils; import java.util.concurrent.atomic.AtomicInteger; +import java.util.function.ToDoubleFunction; @Service public class DefaultStatsFactory implements StatsFactory { @@ -86,6 +87,16 @@ public class DefaultStatsFactory implements StatsFactory { return meterRegistry.gauge(key, Tags.of(tags), number); } + @Override + public T createGauge(StatsType statsType, String name, T number, String... tags) { + return createGauge(statsType.getName(), number, getTags(name, tags)); + } + + @Override + public void createGauge(StatsType statsType, String name, S stateObject, ToDoubleFunction numberProvider, String... tags) { + meterRegistry.gauge(statsType.getName(), Tags.of(getTags(name, tags)), stateObject, numberProvider); + } + @Override public MessagesStats createMessagesStats(String key) { StatsCounter totalCounter = createStatsCounter(key, TOTAL_MSGS); diff --git a/common/stats/src/main/java/org/thingsboard/server/common/stats/DummyEdqsStatsService.java b/common/stats/src/main/java/org/thingsboard/server/common/stats/DummyEdqsStatsService.java index df78e5fc89..3db1c6c790 100644 --- a/common/stats/src/main/java/org/thingsboard/server/common/stats/DummyEdqsStatsService.java +++ b/common/stats/src/main/java/org/thingsboard/server/common/stats/DummyEdqsStatsService.java @@ -33,9 +33,21 @@ public class DummyEdqsStatsService implements EdqsStatsService { public void reportRemoved(ObjectType objectType) {} @Override - public void reportDataQuery(TenantId tenantId, EntityDataQuery query, long timingNanos) {} + public void reportEntityDataQuery(TenantId tenantId, EntityDataQuery query, long timingNanos) {} @Override - public void reportCountQuery(TenantId tenantId, EntityCountQuery query, long timingNanos) {} + public void reportEntityCountQuery(TenantId tenantId, EntityCountQuery query, long timingNanos) {} + + @Override + public void reportEdqsDataQuery(TenantId tenantId, EntityDataQuery query, long timingNanos) {} + + @Override + public void reportEdqsCountQuery(TenantId tenantId, EntityCountQuery query, long timingNanos) {} + + @Override + public void reportStringCompressed() {} + + @Override + public void reportStringUncompressed() {} } diff --git a/common/stats/src/main/java/org/thingsboard/server/common/stats/EdqsStatsService.java b/common/stats/src/main/java/org/thingsboard/server/common/stats/EdqsStatsService.java index 106e43e913..3a6eeb26b8 100644 --- a/common/stats/src/main/java/org/thingsboard/server/common/stats/EdqsStatsService.java +++ b/common/stats/src/main/java/org/thingsboard/server/common/stats/EdqsStatsService.java @@ -26,8 +26,16 @@ public interface EdqsStatsService { void reportRemoved(ObjectType objectType); - void reportDataQuery(TenantId tenantId, EntityDataQuery query, long timingNanos); + void reportEntityDataQuery(TenantId tenantId, EntityDataQuery query, long timingNanos); - void reportCountQuery(TenantId tenantId, EntityCountQuery query, long timingNanos); + void reportEntityCountQuery(TenantId tenantId, EntityCountQuery query, long timingNanos); + + void reportEdqsDataQuery(TenantId tenantId, EntityDataQuery query, long timingNanos); + + void reportEdqsCountQuery(TenantId tenantId, EntityCountQuery query, long timingNanos); + + void reportStringCompressed(); + + void reportStringUncompressed(); } diff --git a/common/stats/src/main/java/org/thingsboard/server/common/stats/StatsFactory.java b/common/stats/src/main/java/org/thingsboard/server/common/stats/StatsFactory.java index bd46c09285..e438af3ad7 100644 --- a/common/stats/src/main/java/org/thingsboard/server/common/stats/StatsFactory.java +++ b/common/stats/src/main/java/org/thingsboard/server/common/stats/StatsFactory.java @@ -17,6 +17,8 @@ package org.thingsboard.server.common.stats; import io.micrometer.core.instrument.Timer; +import java.util.function.ToDoubleFunction; + public interface StatsFactory { StatsCounter createStatsCounter(String key, String statsName, String... otherTags); @@ -25,6 +27,10 @@ public interface StatsFactory { T createGauge(String key, T number, String... tags); + T createGauge(StatsType statsType, String name, T number, String... tags); + + void createGauge(StatsType statsType, String name, S stateObject, ToDoubleFunction numberProvider, String... tags); + MessagesStats createMessagesStats(String key); Timer createTimer(String key, String... tags); diff --git a/common/util/src/main/java/org/thingsboard/common/util/TbBytePool.java b/common/util/src/main/java/org/thingsboard/common/util/TbBytePool.java index fe16a14e7c..1aadb49129 100644 --- a/common/util/src/main/java/org/thingsboard/common/util/TbBytePool.java +++ b/common/util/src/main/java/org/thingsboard/common/util/TbBytePool.java @@ -16,12 +16,14 @@ package org.thingsboard.common.util; import com.google.common.hash.Hashing; +import lombok.Getter; import org.springframework.util.ConcurrentReferenceHashMap; import java.util.concurrent.ConcurrentMap; public class TbBytePool { + @Getter private static final ConcurrentMap pool = new ConcurrentReferenceHashMap<>(); public static byte[] intern(byte[] data) { @@ -32,8 +34,4 @@ public class TbBytePool { return pool.computeIfAbsent(checksum, c -> data); } - public static int size(){ - return pool.size(); - } - } diff --git a/common/util/src/main/java/org/thingsboard/common/util/TbStringPool.java b/common/util/src/main/java/org/thingsboard/common/util/TbStringPool.java index 38c010fbd3..167bf1bbed 100644 --- a/common/util/src/main/java/org/thingsboard/common/util/TbStringPool.java +++ b/common/util/src/main/java/org/thingsboard/common/util/TbStringPool.java @@ -15,12 +15,14 @@ */ package org.thingsboard.common.util; +import lombok.Getter; import org.springframework.util.ConcurrentReferenceHashMap; import java.util.concurrent.ConcurrentMap; public class TbStringPool { + @Getter private static final ConcurrentMap pool = new ConcurrentReferenceHashMap<>(); public static String intern(String data) { @@ -30,8 +32,4 @@ public class TbStringPool { return pool.computeIfAbsent(data, str -> str); } - public static int size(){ - return pool.size(); - } - } diff --git a/dao/src/main/java/org/thingsboard/server/dao/entity/BaseEntityService.java b/dao/src/main/java/org/thingsboard/server/dao/entity/BaseEntityService.java index 4df33779ee..ee9320cce3 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/entity/BaseEntityService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/entity/BaseEntityService.java @@ -20,7 +20,6 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.context.annotation.Lazy; import org.springframework.stereotype.Service; import org.springframework.util.CollectionUtils; -import org.thingsboard.common.util.TbStopWatch; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.HasCustomerId; import org.thingsboard.server.common.data.HasEmail; @@ -101,7 +100,7 @@ public class BaseEntityService extends AbstractEntityService implements EntitySe validateId(customerId, id -> INCORRECT_CUSTOMER_ID + id); validateEntityCountQuery(query); - TbStopWatch stopWatch = TbStopWatch.create(); + long startNs = System.nanoTime(); Long result; if (edqsApiService.isEnabled() && validForEdqs(query) && !tenantId.isSysTenantId()) { EdqsRequest request = EdqsRequest.builder() @@ -112,7 +111,7 @@ public class BaseEntityService extends AbstractEntityService implements EntitySe } else { result = entityQueryDao.countEntitiesByQuery(tenantId, customerId, query); } - edqsStatsService.reportCountQuery(tenantId, query, stopWatch.stopAndGetTotalTimeNanos()); + edqsStatsService.reportEntityCountQuery(tenantId, query, System.nanoTime() - startNs); return result; } @@ -123,7 +122,7 @@ public class BaseEntityService extends AbstractEntityService implements EntitySe validateId(customerId, id -> INCORRECT_CUSTOMER_ID + id); validateEntityDataQuery(query); - TbStopWatch stopWatch = TbStopWatch.create(); + long startNs = System.nanoTime(); PageData result; if (edqsApiService.isEnabled() && validForEdqs(query)) { EdqsRequest request = EdqsRequest.builder() @@ -146,7 +145,7 @@ public class BaseEntityService extends AbstractEntityService implements EntitySe } } } - edqsStatsService.reportDataQuery(tenantId, query, stopWatch.stopAndGetTotalTimeNanos()); + edqsStatsService.reportEntityDataQuery(tenantId, query, System.nanoTime() - startNs); return result; } diff --git a/edqs/src/main/resources/edqs.yml b/edqs/src/main/resources/edqs.yml index 8ea40c03c8..1cc32a4230 100644 --- a/edqs/src/main/resources/edqs.yml +++ b/edqs/src/main/resources/edqs.yml @@ -76,6 +76,8 @@ queue: stats: # Enable/disable statistics for EDQS enabled: "${TB_EDQS_STATS_ENABLED:true}" + # Threshold for slow queries to log, in milliseconds + slow_query_threshold: "${TB_EDQS_SLOW_QUERY_THRESHOLD_MS:200}" kafka: # Kafka Bootstrap nodes in "host:port" format From 70906101683eddb1afa178cbde0c17c543b657a9 Mon Sep 17 00:00:00 2001 From: ViacheslavKlimov Date: Thu, 10 Apr 2025 14:54:42 +0300 Subject: [PATCH 21/47] Fix EDQS stats collector names --- .../stats/HousekeeperStatsService.java | 2 +- .../edqs/stats/DefaultEdqsStatsService.java | 15 +++++++-------- .../server/common/stats/DefaultStatsFactory.java | 12 ++++++------ .../server/common/stats/StatsFactory.java | 6 +++--- 4 files changed, 17 insertions(+), 18 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/housekeeper/stats/HousekeeperStatsService.java b/application/src/main/java/org/thingsboard/server/service/housekeeper/stats/HousekeeperStatsService.java index 5da5fa2be0..5946b84d09 100644 --- a/application/src/main/java/org/thingsboard/server/service/housekeeper/stats/HousekeeperStatsService.java +++ b/application/src/main/java/org/thingsboard/server/service/housekeeper/stats/HousekeeperStatsService.java @@ -106,7 +106,7 @@ public class HousekeeperStatsService { this.failedProcessingCounter = register("failedProcessing", statsFactory); this.reprocessedCounter = register("reprocessed", statsFactory); this.failedReprocessingCounter = register("failedReprocessing", statsFactory); - this.processingTimer = statsFactory.createTimer(StatsType.HOUSEKEEPER, "processingTime", "taskType", taskType.name()); + this.processingTimer = statsFactory.createStatsTimer(StatsType.HOUSEKEEPER.getName(), "processingTime", "taskType", taskType.name()); } private StatsCounter register(String statsName, StatsFactory statsFactory) { diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/stats/DefaultEdqsStatsService.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/stats/DefaultEdqsStatsService.java index 46fb153771..4853920c44 100644 --- a/common/edqs/src/main/java/org/thingsboard/server/edqs/stats/DefaultEdqsStatsService.java +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/stats/DefaultEdqsStatsService.java @@ -31,7 +31,6 @@ import org.thingsboard.server.common.stats.EdqsStatsService; import org.thingsboard.server.common.stats.StatsCounter; import org.thingsboard.server.common.stats.StatsFactory; import org.thingsboard.server.common.stats.StatsTimer; -import org.thingsboard.server.common.stats.StatsType; import org.thingsboard.server.edqs.repo.DefaultEdqsRepository; import java.util.Map; @@ -58,9 +57,9 @@ public class DefaultEdqsStatsService implements EdqsStatsService { @PostConstruct private void init() { - statsFactory.createGauge(StatsType.EDQS, "stringPoolSize", TbStringPool.getPool(), Map::size); - statsFactory.createGauge(StatsType.EDQS, "bytePoolSize", TbBytePool.getPool(), Map::size); - statsFactory.createGauge(StatsType.EDQS, "tenantReposSize", DefaultEdqsRepository.getRepos(), Map::size); + statsFactory.createGauge("edqsGauges", "stringPoolSize", TbStringPool.getPool(), Map::size); + statsFactory.createGauge("edqsGauges", "bytePoolSize", TbBytePool.getPool(), Map::size); + statsFactory.createGauge("edqsGauges", "tenantReposSize", DefaultEdqsRepository.getRepos(), Map::size); } @Override @@ -118,20 +117,20 @@ public class DefaultEdqsStatsService implements EdqsStatsService { } private StatsTimer getTimer(String name) { - return timers.computeIfAbsent(name, __ -> statsFactory.createTimer(StatsType.EDQS, name)); + return timers.computeIfAbsent(name, __ -> statsFactory.createStatsTimer("edqsTimers", name)); } private StatsCounter getCounter(String name) { - return counters.computeIfAbsent(name, __ -> statsFactory.createStatsCounter(StatsType.EDQS.getName(), name)); + return counters.computeIfAbsent(name, __ -> statsFactory.createStatsCounter("edqsCounters", name)); } private AtomicInteger getGauge(String name) { - return gauges.computeIfAbsent(name, __ -> statsFactory.createGauge(StatsType.EDQS, name, new AtomicInteger())); + return gauges.computeIfAbsent(name, __ -> statsFactory.createGauge("edqsGauges", name, new AtomicInteger())); } private AtomicInteger getObjectGauge(ObjectType objectType) { return objectCounters.computeIfAbsent(objectType, type -> - statsFactory.createGauge(StatsType.EDQS, "objectsCount", new AtomicInteger(), "objectType", type.name())); + statsFactory.createGauge("edqsGauges", "objectsCount", new AtomicInteger(), "objectType", type.name())); } } diff --git a/common/stats/src/main/java/org/thingsboard/server/common/stats/DefaultStatsFactory.java b/common/stats/src/main/java/org/thingsboard/server/common/stats/DefaultStatsFactory.java index 15c155ecfd..4d858be4bb 100644 --- a/common/stats/src/main/java/org/thingsboard/server/common/stats/DefaultStatsFactory.java +++ b/common/stats/src/main/java/org/thingsboard/server/common/stats/DefaultStatsFactory.java @@ -88,13 +88,13 @@ public class DefaultStatsFactory implements StatsFactory { } @Override - public T createGauge(StatsType statsType, String name, T number, String... tags) { - return createGauge(statsType.getName(), number, getTags(name, tags)); + public T createGauge(String type, String name, T number, String... tags) { + return createGauge(type, number, getTags(name, tags)); } @Override - public void createGauge(StatsType statsType, String name, S stateObject, ToDoubleFunction numberProvider, String... tags) { - meterRegistry.gauge(statsType.getName(), Tags.of(getTags(name, tags)), stateObject, numberProvider); + public void createGauge(String type, String name, S stateObject, ToDoubleFunction numberProvider, String... tags) { + meterRegistry.gauge(type, Tags.of(getTags(name, tags)), stateObject, numberProvider); } @Override @@ -117,8 +117,8 @@ public class DefaultStatsFactory implements StatsFactory { } @Override - public StatsTimer createTimer(StatsType type, String name, String... tags) { - return new StatsTimer(name, Timer.builder(type.getName()) + public StatsTimer createStatsTimer(String type, String name, String... tags) { + return new StatsTimer(name, Timer.builder(type) .tags(getTags(name, tags)) .register(meterRegistry)); } diff --git a/common/stats/src/main/java/org/thingsboard/server/common/stats/StatsFactory.java b/common/stats/src/main/java/org/thingsboard/server/common/stats/StatsFactory.java index e438af3ad7..291b95e25a 100644 --- a/common/stats/src/main/java/org/thingsboard/server/common/stats/StatsFactory.java +++ b/common/stats/src/main/java/org/thingsboard/server/common/stats/StatsFactory.java @@ -27,14 +27,14 @@ public interface StatsFactory { T createGauge(String key, T number, String... tags); - T createGauge(StatsType statsType, String name, T number, String... tags); + T createGauge(String type, String name, T number, String... tags); - void createGauge(StatsType statsType, String name, S stateObject, ToDoubleFunction numberProvider, String... tags); + void createGauge(String type, String name, S stateObject, ToDoubleFunction numberProvider, String... tags); MessagesStats createMessagesStats(String key); Timer createTimer(String key, String... tags); - StatsTimer createTimer(StatsType type, String name, String... tags); + StatsTimer createStatsTimer(String type, String name, String... tags); } From b854dcf0c7242560bbb87d50f3ef6ea6e12e5df1 Mon Sep 17 00:00:00 2001 From: Artem Dzhereleiko Date: Thu, 10 Apr 2025 14:59:11 +0300 Subject: [PATCH 22/47] UI: Refactoring error msg --- .../components/widget/lib/scada/scada-symbol.models.ts | 2 +- .../home/pages/scada-symbol/scada-symbol.component.ts | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/ui-ngx/src/app/modules/home/components/widget/lib/scada/scada-symbol.models.ts b/ui-ngx/src/app/modules/home/components/widget/lib/scada/scada-symbol.models.ts index e52c6f9f54..abe570fa3c 100644 --- a/ui-ngx/src/app/modules/home/components/widget/lib/scada/scada-symbol.models.ts +++ b/ui-ngx/src/app/modules/home/components/widget/lib/scada/scada-symbol.models.ts @@ -271,7 +271,7 @@ export const updateScadaSymbolMetadataInContent = (svgContent: string, metadata: const svgDoc = new DOMParser().parseFromString(svgContent, 'image/svg+xml'); const parsererror = svgDoc.getElementsByTagName('parsererror'); if (parsererror?.length) { - return parsererror[0].outerHTML; + throw Error(parsererror[0].textContent) } updateScadaSymbolMetadataInDom(svgDoc, metadata); return svgDoc.documentElement.outerHTML; diff --git a/ui-ngx/src/app/modules/home/pages/scada-symbol/scada-symbol.component.ts b/ui-ngx/src/app/modules/home/pages/scada-symbol/scada-symbol.component.ts index caaf0edc01..c618380565 100644 --- a/ui-ngx/src/app/modules/home/pages/scada-symbol/scada-symbol.component.ts +++ b/ui-ngx/src/app/modules/home/pages/scada-symbol/scada-symbol.component.ts @@ -214,10 +214,8 @@ export class ScadaSymbolComponent extends PageComponent this.editObjectCallbacks.tagsUpdated(tags); } const metadata: ScadaSymbolMetadata = this.scadaSymbolFormGroup.get('metadata').value; - const scadaSymbolContent = this.prepareScadaSymbolContent(metadata); - if (scadaSymbolContent.includes('parsererror')) { - this.store.dispatch(new ActionNotificationShow({ message: scadaSymbolContent, type: 'error' })); - } else { + try { + const scadaSymbolContent = this.prepareScadaSymbolContent(metadata); const file = createFileFromContent(scadaSymbolContent, this.symbolData.imageResource.fileName, this.symbolData.imageResource.descriptor.mediaType); const type = imageResourceType(this.symbolData.imageResource); @@ -243,6 +241,8 @@ export class ScadaSymbolComponent extends PageComponent this.init(data); this.updateBreadcrumbs.emit(); }); + } catch (e) { + this.store.dispatch(new ActionNotificationShow({ message: e.message, type: 'error' })); } } } From 6af342090e577016145a3023c04f0dd9782a6e1f Mon Sep 17 00:00:00 2001 From: Ekaterina Chantsova Date: Thu, 10 Apr 2025 15:30:20 +0300 Subject: [PATCH 23/47] Timewindow: added timezone selection in widget settings --- .../widget/config/timewindow-config-panel.component.html | 1 + 1 file changed, 1 insertion(+) diff --git a/ui-ngx/src/app/modules/home/components/widget/config/timewindow-config-panel.component.html b/ui-ngx/src/app/modules/home/components/widget/config/timewindow-config-panel.component.html index ff0acd5fca..138dde324e 100644 --- a/ui-ngx/src/app/modules/home/components/widget/config/timewindow-config-panel.component.html +++ b/ui-ngx/src/app/modules/home/components/widget/config/timewindow-config-panel.component.html @@ -33,6 +33,7 @@ noMargin isEdit="true" alwaysDisplayTypePrefix + timezone="true" [historyOnly]="onlyHistoryTimewindow" quickIntervalOnly="{{ widgetType === widgetTypes.latest }}" aggregation="{{ widgetType === widgetTypes.timeseries }}" From 2abebbd352f6354d3920e207d84c21a40563ad2f Mon Sep 17 00:00:00 2001 From: Vladyslav_Prykhodko Date: Thu, 10 Apr 2025 15:35:25 +0300 Subject: [PATCH 24/47] UI: Improved CSS styles for copy-code button in markdown --- .../device-check-connectivity-dialog.component.scss | 1 - .../src/app/shared/components/markdown.component.scss | 10 +++++----- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/ui-ngx/src/app/modules/home/pages/device/device-check-connectivity-dialog.component.scss b/ui-ngx/src/app/modules/home/pages/device/device-check-connectivity-dialog.component.scss index 123bcfb708..eca728893c 100644 --- a/ui-ngx/src/app/modules/home/pages/device/device-check-connectivity-dialog.component.scss +++ b/ui-ngx/src/app/modules/home/pages/device/device-check-connectivity-dialog.component.scss @@ -124,7 +124,6 @@ background: #F3F6FA; border-color: #305680; padding-right: 38px; - overflow: scroll; padding-bottom: 4px; min-height: 42px; scrollbar-width: thin; diff --git a/ui-ngx/src/app/shared/components/markdown.component.scss b/ui-ngx/src/app/shared/components/markdown.component.scss index 4989a77021..65ad64b189 100644 --- a/ui-ngx/src/app/shared/components/markdown.component.scss +++ b/ui-ngx/src/app/shared/components/markdown.component.scss @@ -270,7 +270,7 @@ outline: none; position: absolute; width: 206px; - height: 42px; + height: 32px; top: 0; right: 32px; background: 0 0; @@ -281,11 +281,11 @@ user-select: none; &.multiline { - right: 44px; + right: 38px; } p { - padding: 8px; + padding: 8px 8px 0; top: 1px; transition: .2s; color: #2a7dec; @@ -301,10 +301,10 @@ background-color: #fff; position: absolute; width: 38px; - height: 38px; + height: 28px; top: 3px; right: 3px; - padding: 10px; + padding: 10px 10px 0; img { position: initial; From 4882bf26ebc300a0dd4f02fb54ca20739a95b0ab Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Thu, 10 Apr 2025 16:45:24 +0300 Subject: [PATCH 25/47] removed entity profile cache --- .../server/actors/ActorSystemContext.java | 8 +- .../CalculatedFieldManagerActor.java | 8 ++ ...alculatedFieldManagerMessageProcessor.java | 99 +++++++++++-- .../server/actors/tenant/TenantActor.java | 5 +- .../cf/DefaultCalculatedFieldInitService.java | 15 +- .../CalculatedFieldEntityProfileCache.java | 40 ------ ...aultCalculatedFieldEntityProfileCache.java | 132 ------------------ .../cf/cache/TenantEntityProfileCache.java | 5 + ...faultTbCalculatedFieldConsumerService.java | 16 +-- .../server/dao/asset/AssetService.java | 2 + .../server/dao/cf/CalculatedFieldService.java | 4 +- .../server/dao/device/DeviceService.java | 2 + .../server/common/msg/MsgType.java | 2 + .../msg/cf/CalculatedFieldActorInitMsg.java | 33 +++++ .../common/msg/cf/ProfileEntityMsg.java | 36 +++++ .../provider/KafkaMonolithQueueFactory.java | 2 +- .../KafkaTbRuleEngineQueueFactory.java | 2 +- .../server/queue/util/AfterStartUp.java | 5 +- .../server/dao/asset/AssetDao.java | 2 + .../server/dao/asset/BaseAssetService.java | 8 ++ .../dao/cf/BaseCalculatedFieldService.java | 15 +- .../server/dao/cf/CalculatedFieldDao.java | 2 + .../server/dao/cf/CalculatedFieldLinkDao.java | 2 + .../server/dao/device/DeviceDao.java | 4 +- .../server/dao/device/DeviceServiceImpl.java | 8 ++ .../server/dao/sql/asset/JpaAssetDao.java | 10 +- .../sql/cf/CalculatedFieldLinkRepository.java | 4 + .../dao/sql/cf/CalculatedFieldRepository.java | 2 + .../dao/sql/cf/JpaCalculatedFieldDao.java | 6 + .../dao/sql/cf/JpaCalculatedFieldLinkDao.java | 6 + .../device/DefaultNativeAssetRepository.java | 17 ++- .../device/DefaultNativeDeviceRepository.java | 11 ++ .../server/dao/sql/device/JpaDeviceDao.java | 16 ++- .../device/NativeProfileEntityRepository.java | 4 + 34 files changed, 292 insertions(+), 241 deletions(-) delete mode 100644 application/src/main/java/org/thingsboard/server/service/cf/cache/CalculatedFieldEntityProfileCache.java delete mode 100644 application/src/main/java/org/thingsboard/server/service/cf/cache/DefaultCalculatedFieldEntityProfileCache.java create mode 100644 common/message/src/main/java/org/thingsboard/server/common/msg/cf/CalculatedFieldActorInitMsg.java create mode 100644 common/message/src/main/java/org/thingsboard/server/common/msg/cf/ProfileEntityMsg.java diff --git a/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java b/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java index 1ed919e922..b5fa1e6fbd 100644 --- a/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java +++ b/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java @@ -30,9 +30,9 @@ import org.springframework.context.annotation.Lazy; import org.springframework.data.redis.core.RedisTemplate; import org.springframework.stereotype.Component; import org.thingsboard.common.util.JacksonUtil; +import org.thingsboard.rule.engine.api.DeviceStateManager; import org.thingsboard.rule.engine.api.MailService; import org.thingsboard.rule.engine.api.NotificationCenter; -import org.thingsboard.rule.engine.api.DeviceStateManager; import org.thingsboard.rule.engine.api.SmsService; import org.thingsboard.rule.engine.api.notification.SlackService; import org.thingsboard.rule.engine.api.sms.SmsSenderFactory; @@ -110,7 +110,6 @@ import org.thingsboard.server.service.apiusage.TbApiUsageStateService; import org.thingsboard.server.service.cf.CalculatedFieldProcessingService; import org.thingsboard.server.service.cf.CalculatedFieldQueueService; import org.thingsboard.server.service.cf.CalculatedFieldStateService; -import org.thingsboard.server.service.cf.cache.CalculatedFieldEntityProfileCache; import org.thingsboard.server.service.cf.ctx.state.ArgumentEntry; import org.thingsboard.server.service.component.ComponentDiscoveryService; import org.thingsboard.server.service.edge.rpc.EdgeRpcService; @@ -546,11 +545,6 @@ public class ActorSystemContext { @Getter private CalculatedFieldQueueService calculatedFieldQueueService; - @Lazy - @Autowired(required = false) - @Getter - private CalculatedFieldEntityProfileCache calculatedFieldEntityProfileCache; - @Value("${actors.session.max_concurrent_sessions_per_device:1}") @Getter private int maxConcurrentSessionsPerDevice; diff --git a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerActor.java b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerActor.java index 70ed2849e8..511449c2a9 100644 --- a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerActor.java +++ b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerActor.java @@ -22,10 +22,12 @@ import org.thingsboard.server.actors.TbActorException; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.msg.TbActorStopReason; import org.thingsboard.server.common.msg.ToCalculatedFieldSystemMsg; +import org.thingsboard.server.common.msg.cf.CalculatedFieldActorInitMsg; import org.thingsboard.server.common.msg.cf.CalculatedFieldEntityLifecycleMsg; import org.thingsboard.server.common.msg.cf.CalculatedFieldInitMsg; import org.thingsboard.server.common.msg.cf.CalculatedFieldLinkInitMsg; import org.thingsboard.server.common.msg.cf.CalculatedFieldPartitionChangeMsg; +import org.thingsboard.server.common.msg.cf.ProfileEntityMsg; /** * Created by ashvayka on 15.03.18. @@ -65,6 +67,12 @@ public class CalculatedFieldManagerActor extends AbstractCalculatedFieldActor { case CF_PARTITIONS_CHANGE_MSG: processor.onPartitionChange((CalculatedFieldPartitionChangeMsg) msg); break; + case CF_ACTOR_INIT_MSG: + processor.onActorInitMsg((CalculatedFieldActorInitMsg) msg); + break; + case CF_PROFILE_ENTITY_MSG: + processor.onProfileEntityMsg((ProfileEntityMsg) msg); + break; case CF_INIT_MSG: processor.onFieldInitMsg((CalculatedFieldInitMsg) msg); break; diff --git a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java index 060893ec8d..9dad4e820e 100644 --- a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java +++ b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java @@ -15,7 +15,9 @@ */ package org.thingsboard.server.actors.calculatedField; +import lombok.Getter; import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Value; import org.thingsboard.server.actors.ActorSystemContext; import org.thingsboard.server.actors.TbActorCtx; import org.thingsboard.server.actors.TbActorRef; @@ -24,6 +26,7 @@ import org.thingsboard.server.actors.service.DefaultActorService; import org.thingsboard.server.actors.shared.AbstractContextAwareMsgProcessor; import org.thingsboard.server.common.data.DataConstants; import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.ProfileEntityIdInfo; import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.cf.CalculatedFieldLink; import org.thingsboard.server.common.data.id.AssetId; @@ -31,17 +34,22 @@ import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.page.PageDataIterable; +import org.thingsboard.server.common.msg.cf.CalculatedFieldActorInitMsg; import org.thingsboard.server.common.msg.cf.CalculatedFieldEntityLifecycleMsg; import org.thingsboard.server.common.msg.cf.CalculatedFieldInitMsg; import org.thingsboard.server.common.msg.cf.CalculatedFieldLinkInitMsg; import org.thingsboard.server.common.msg.cf.CalculatedFieldPartitionChangeMsg; +import org.thingsboard.server.common.msg.cf.ProfileEntityMsg; import org.thingsboard.server.common.msg.plugin.ComponentLifecycleMsg; import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.common.msg.queue.TbCallback; +import org.thingsboard.server.dao.asset.AssetService; import org.thingsboard.server.dao.cf.CalculatedFieldService; +import org.thingsboard.server.dao.device.DeviceService; import org.thingsboard.server.service.cf.CalculatedFieldProcessingService; import org.thingsboard.server.service.cf.CalculatedFieldStateService; -import org.thingsboard.server.service.cf.cache.CalculatedFieldEntityProfileCache; +import org.thingsboard.server.service.cf.cache.TenantEntityProfileCache; import org.thingsboard.server.service.cf.ctx.CalculatedFieldEntityCtxId; import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldCtx; import org.thingsboard.server.service.profile.TbAssetProfileCache; @@ -68,22 +76,30 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware private final CalculatedFieldProcessingService cfExecService; private final CalculatedFieldStateService cfStateService; - private final CalculatedFieldEntityProfileCache cfEntityCache; private final CalculatedFieldService cfDaoService; + private final DeviceService deviceService; + private final AssetService assetService; private final TbAssetProfileCache assetProfileCache; private final TbDeviceProfileCache deviceProfileCache; + private final TenantEntityProfileCache entityProfileCache; protected final TenantId tenantId; protected TbActorCtx ctx; + @Value("${calculated_fields.init_fetch_pack_size:50000}") + @Getter + private int initFetchPackSize; + CalculatedFieldManagerMessageProcessor(ActorSystemContext systemContext, TenantId tenantId) { super(systemContext); - this.cfEntityCache = systemContext.getCalculatedFieldEntityProfileCache(); this.cfExecService = systemContext.getCalculatedFieldProcessingService(); this.cfStateService = systemContext.getCalculatedFieldStateService(); this.cfDaoService = systemContext.getCalculatedFieldService(); + this.deviceService = systemContext.getDeviceService(); + this.assetService = systemContext.getAssetService(); this.assetProfileCache = systemContext.getAssetProfileCache(); this.deviceProfileCache = systemContext.getDeviceProfileCache(); + this.entityProfileCache = new TenantEntityProfileCache(); this.tenantId = tenantId; } @@ -100,6 +116,19 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware ctx.stop(ctx.getSelf()); } + public void onActorInitMsg(CalculatedFieldActorInitMsg msg) { + log.debug("[{}] Processing CF actor init message.", msg.getTenantId().getId()); + initEntityProfileCache(); + initCalculatedFields(); + msg.getCallback().onSuccess(); + } + + public void onProfileEntityMsg(ProfileEntityMsg msg) { + log.debug("[{}] Processing profile entity message.", msg.getTenantId().getId()); + entityProfileCache.add(msg.getProfileEntityId(), msg.getEntityId()); + msg.getCallback().onSuccess(); + } + public void onFieldInitMsg(CalculatedFieldInitMsg msg) throws CalculatedFieldException { log.debug("[{}] Processing CF init message.", msg.getCf().getId()); var cf = msg.getCf(); @@ -180,16 +209,35 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware } break; } + case DEVICE_PROFILE: + case ASSET_PROFILE: { + switch (event) { + case DELETED: + onProfileDeleted(msg.getData(), msg.getCallback()); + break; + default: + msg.getCallback().onSuccess(); + break; + } + break; + } default: { msg.getCallback().onSuccess(); } } } + private void onProfileDeleted(ComponentLifecycleMsg msg, TbCallback callback) { + entityProfileCache.removeProfileId(msg.getEntityId()); + callback.onSuccess(); + } + private void onEntityCreated(ComponentLifecycleMsg msg, TbCallback callback) { EntityId entityId = msg.getEntityId(); EntityId profileId = getProfileId(tenantId, entityId); - cfEntityCache.add(tenantId, profileId, entityId); + if (profileId != null) { + entityProfileCache.add(profileId, entityId); + } if (!isMyPartition(entityId, callback)) { return; } @@ -207,7 +255,7 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware private void onEntityUpdated(ComponentLifecycleMsg msg, TbCallback callback) { if (msg.getOldProfileId() != null && !msg.getOldProfileId().equals(msg.getProfileId())) { - cfEntityCache.update(tenantId, msg.getOldProfileId(), msg.getProfileId(), msg.getEntityId()); + entityProfileCache.update(msg.getOldProfileId(), msg.getProfileId(), msg.getEntityId()); if (!isMyPartition(msg.getEntityId(), callback)) { return; } @@ -226,7 +274,7 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware } private void onEntityDeleted(ComponentLifecycleMsg msg, TbCallback callback) { - cfEntityCache.evict(tenantId, msg.getEntityId()); + entityProfileCache.removeEntityId(msg.getEntityId()); if (isMyPartition(msg.getEntityId(), callback)) { log.debug("Pushing entity lifecycle msg to specific actor [{}]", msg.getEntityId()); getOrCreateActor(msg.getEntityId()).tell(new CalculatedFieldEntityDeleteMsg(tenantId, msg.getEntityId(), callback)); @@ -322,7 +370,7 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware EntityId entityId = cfCtx.getEntityId(); EntityType entityType = cfCtx.getEntityId().getEntityType(); if (isProfileEntity(entityType)) { - var entityIds = cfEntityCache.getEntityIdsByProfileId(tenantId, entityId); + var entityIds = entityProfileCache.getEntityIdsByProfileId(entityId); if (!entityIds.isEmpty()) { //TODO: no need to do this if we cache all created actors and know which one belong to us; var multiCallback = new MultipleTbCallback(entityIds.size(), callback); @@ -383,7 +431,7 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware var cf = calculatedFields.get(link.cfId()); if (EntityType.DEVICE_PROFILE.equals(targetEntityType) || EntityType.ASSET_PROFILE.equals(targetEntityType)) { // iterate over all entities that belong to profile and push the message for corresponding CF - var entityIds = cfEntityCache.getEntityIdsByProfileId(tenantId, targetEntityId); + var entityIds = entityProfileCache.getEntityIdsByProfileId(targetEntityId); if (!entityIds.isEmpty()) { MultipleTbCallback multipleCallback = new MultipleTbCallback(entityIds.size(), callback); var newMsg = new EntityCalculatedFieldLinkedTelemetryMsg(tenantId, sourceEntityId, proto.getMsg(), cf, multipleCallback); @@ -445,7 +493,7 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware EntityId entityId = cfCtx.getEntityId(); EntityType entityType = cfCtx.getEntityId().getEntityType(); if (isProfileEntity(entityType)) { - var entityIds = cfEntityCache.getEntityIdsByProfileId(tenantId, entityId); + var entityIds = entityProfileCache.getEntityIdsByProfileId(entityId); if (!entityIds.isEmpty()) { var multiCallback = new MultipleTbCallback(entityIds.size(), callback); entityIds.forEach(id -> { @@ -513,21 +561,46 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware } public void onPartitionChange(CalculatedFieldPartitionChangeMsg msg) { - initCalculatedFields(); ctx.broadcastToChildren(msg, true); } public void initCalculatedFields() { - cfDaoService.findCalculatedFieldsByTenantId(tenantId).forEach(cf -> { + PageDataIterable cfs = new PageDataIterable<>(pageLink -> cfDaoService.findCalculatedFieldsByTenantId(tenantId, pageLink), initFetchPackSize); + cfs.forEach(cf -> { try { onFieldInitMsg(new CalculatedFieldInitMsg(cf.getTenantId(), cf)); } catch (CalculatedFieldException e) { throw new RuntimeException(e); } }); - cfDaoService.findAllCalculatedFieldLinksByTenantId(tenantId).forEach(cfLink -> { - onLinkInitMsg(new CalculatedFieldLinkInitMsg(cfLink.getTenantId(), cfLink)); + calculatedFields.values().forEach(cf -> { + entityIdCalculatedFields.computeIfAbsent(cf.getEntityId(), id -> new CopyOnWriteArrayList<>()).add(cf); }); + PageDataIterable cfls = new PageDataIterable<>(pageLink -> cfDaoService.findAllCalculatedFieldLinksByTenantId(tenantId, pageLink), initFetchPackSize); + cfls.forEach(link -> { + onLinkInitMsg(new CalculatedFieldLinkInitMsg(link.getTenantId(), link)); + }); + } + + private void initEntityProfileCache() { + PageDataIterable deviceIdInfos = new PageDataIterable<>(pageLink -> deviceService.findProfileEntityIdInfosByTenantId(tenantId, pageLink), initFetchPackSize); + for (ProfileEntityIdInfo idInfo : deviceIdInfos) { + log.trace("Processing device record: {}", idInfo); + try { + entityProfileCache.add(idInfo.getProfileId(), idInfo.getEntityId()); + } catch (Exception e) { + log.error("Failed to process device record: {}", idInfo, e); + } + } + PageDataIterable assetIdInfos = new PageDataIterable<>(pageLink -> assetService.findProfileEntityIdInfosByTenantId(tenantId, pageLink), initFetchPackSize); + for (ProfileEntityIdInfo idInfo : assetIdInfos) { + log.trace("Processing asset record: {}", idInfo); + try { + entityProfileCache.add(idInfo.getProfileId(), idInfo.getEntityId()); + } catch (Exception e) { + log.error("Failed to process asset record: {}", idInfo, e); + } + } } } diff --git a/application/src/main/java/org/thingsboard/server/actors/tenant/TenantActor.java b/application/src/main/java/org/thingsboard/server/actors/tenant/TenantActor.java index b42d91a4de..4e04962dcb 100644 --- a/application/src/main/java/org/thingsboard/server/actors/tenant/TenantActor.java +++ b/application/src/main/java/org/thingsboard/server/actors/tenant/TenantActor.java @@ -50,6 +50,7 @@ import org.thingsboard.server.common.msg.TbMsg; import org.thingsboard.server.common.msg.ToCalculatedFieldSystemMsg; import org.thingsboard.server.common.msg.aware.DeviceAwareMsg; import org.thingsboard.server.common.msg.aware.RuleChainAwareMsg; +import org.thingsboard.server.common.msg.cf.CalculatedFieldActorInitMsg; import org.thingsboard.server.common.msg.cf.CalculatedFieldEntityLifecycleMsg; import org.thingsboard.server.common.msg.plugin.ComponentLifecycleMsg; import org.thingsboard.server.common.msg.queue.PartitionChangeMsg; @@ -176,6 +177,8 @@ public class TenantActor extends RuleChainManagerActor { case RULE_CHAIN_TO_RULE_CHAIN_MSG: onRuleChainMsg((RuleChainAwareMsg) msg); break; + case CF_ACTOR_INIT_MSG: + case CF_PROFILE_ENTITY_MSG: case CF_INIT_MSG: case CF_LINK_INIT_MSG: case CF_STATE_RESTORE_MSG: @@ -274,7 +277,7 @@ public class TenantActor extends RuleChainManagerActor { () -> DefaultActorService.CF_MANAGER_DISPATCHER_NAME, () -> new CalculatedFieldManagerActorCreator(systemContext, tenantId), () -> true); - cfActor.tellWithHighPriority(msg); + cfActor.tellWithHighPriority(new CalculatedFieldActorInitMsg(tenantId)); } catch (Exception e) { log.info("[{}] Failed to init CF Actor.", tenantId, e); } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldInitService.java b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldInitService.java index 5f02be0cde..be27b5318b 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldInitService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldInitService.java @@ -20,14 +20,14 @@ import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Service; +import org.thingsboard.server.actors.ActorSystemContext; import org.thingsboard.server.common.data.ProfileEntityIdInfo; import org.thingsboard.server.common.data.page.PageDataIterable; +import org.thingsboard.server.common.msg.cf.ProfileEntityMsg; import org.thingsboard.server.dao.asset.AssetService; import org.thingsboard.server.dao.device.DeviceService; -import org.thingsboard.server.queue.discovery.PartitionService; import org.thingsboard.server.queue.util.AfterStartUp; import org.thingsboard.server.queue.util.TbRuleEngineComponent; -import org.thingsboard.server.service.cf.cache.CalculatedFieldEntityProfileCache; @Slf4j @Service @@ -35,10 +35,9 @@ import org.thingsboard.server.service.cf.cache.CalculatedFieldEntityProfileCache @RequiredArgsConstructor public class DefaultCalculatedFieldInitService implements CalculatedFieldInitService { - private final CalculatedFieldEntityProfileCache entityProfileCache; private final AssetService assetService; private final DeviceService deviceService; - private final PartitionService partitionService; + private final ActorSystemContext actorSystemContext; @Value("${calculated_fields.init_fetch_pack_size:50000}") @Getter @@ -50,9 +49,7 @@ public class DefaultCalculatedFieldInitService implements CalculatedFieldInitSer for (ProfileEntityIdInfo idInfo : deviceIdInfos) { log.trace("Processing device record: {}", idInfo); try { - if (partitionService.isManagedByCurrentService(idInfo.getTenantId())) { - entityProfileCache.add(idInfo.getTenantId(), idInfo.getProfileId(), idInfo.getEntityId()); - } + actorSystemContext.tell(new ProfileEntityMsg(idInfo.getTenantId(), idInfo.getProfileId(), idInfo.getEntityId())); } catch (Exception e) { log.error("Failed to process device record: {}", idInfo, e); } @@ -61,9 +58,7 @@ public class DefaultCalculatedFieldInitService implements CalculatedFieldInitSer for (ProfileEntityIdInfo idInfo : assetIdInfos) { log.trace("Processing asset record: {}", idInfo); try { - if (partitionService.isManagedByCurrentService(idInfo.getTenantId())) { - entityProfileCache.add(idInfo.getTenantId(), idInfo.getProfileId(), idInfo.getEntityId()); - } + actorSystemContext.tell(new ProfileEntityMsg(idInfo.getTenantId(), idInfo.getProfileId(), idInfo.getEntityId())); } catch (Exception e) { log.error("Failed to process asset record: {}", idInfo, e); } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/cache/CalculatedFieldEntityProfileCache.java b/application/src/main/java/org/thingsboard/server/service/cf/cache/CalculatedFieldEntityProfileCache.java deleted file mode 100644 index df2dc88f6b..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/cf/cache/CalculatedFieldEntityProfileCache.java +++ /dev/null @@ -1,40 +0,0 @@ -/** - * Copyright © 2016-2025 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.cf.cache; - -import org.springframework.context.ApplicationListener; -import org.thingsboard.server.common.data.id.EntityId; -import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.queue.discovery.event.PartitionChangeEvent; - -import java.util.Collection; - -public interface CalculatedFieldEntityProfileCache extends ApplicationListener { - - void add(TenantId tenantId, EntityId profileId, EntityId entityId); - - void update(TenantId tenantId, EntityId oldProfileId, EntityId newProfileId, EntityId entityId); - - void evict(TenantId tenantId, EntityId entityId); - - void evictProfile(TenantId tenantId, EntityId profileId); - - void removeTenant(TenantId tenantId); - - Collection getEntityIdsByProfileId(TenantId tenantId, EntityId profileId); - - int getEntityIdPartition(TenantId tenantId, EntityId entityId); -} diff --git a/application/src/main/java/org/thingsboard/server/service/cf/cache/DefaultCalculatedFieldEntityProfileCache.java b/application/src/main/java/org/thingsboard/server/service/cf/cache/DefaultCalculatedFieldEntityProfileCache.java deleted file mode 100644 index cefdd012d8..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/cf/cache/DefaultCalculatedFieldEntityProfileCache.java +++ /dev/null @@ -1,132 +0,0 @@ -/** - * Copyright © 2016-2025 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.cf.cache; - -import lombok.Getter; -import lombok.RequiredArgsConstructor; -import lombok.extern.slf4j.Slf4j; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.stereotype.Service; -import org.thingsboard.server.common.data.DataConstants; -import org.thingsboard.server.common.data.Device; -import org.thingsboard.server.common.data.asset.Asset; -import org.thingsboard.server.common.data.id.EntityId; -import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.common.data.page.PageDataIterable; -import org.thingsboard.server.common.msg.queue.ServiceType; -import org.thingsboard.server.dao.asset.AssetService; -import org.thingsboard.server.dao.device.DeviceService; -import org.thingsboard.server.queue.discovery.PartitionService; -import org.thingsboard.server.queue.discovery.TbApplicationEventListener; -import org.thingsboard.server.queue.discovery.event.PartitionChangeEvent; -import org.thingsboard.server.queue.util.TbRuleEngineComponent; - -import java.util.Collection; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ConcurrentMap; - -@TbRuleEngineComponent -@Service -@Slf4j -@RequiredArgsConstructor -//TODO ashvayka: remove and use TenantEntityProfileCache in each CalculatedFieldManagerMessageProcessor; -public class DefaultCalculatedFieldEntityProfileCache extends TbApplicationEventListener implements CalculatedFieldEntityProfileCache { - - private static final Integer UNKNOWN = 0; - private final ConcurrentMap tenantCache = new ConcurrentHashMap<>(); - private final PartitionService partitionService; - private final AssetService assetService; - private final DeviceService deviceService; - - @Value("${calculated_fields.init_fetch_pack_size:50000}") - @Getter - private int initFetchPackSize; - - @Override - protected void onTbApplicationEvent(PartitionChangeEvent event) { - event.getCfPartitions().forEach(tpi -> tpi.getTenantId().ifPresent(this::initCacheForNewTenant)); - } - - @Override - public void add(TenantId tenantId, EntityId profileId, EntityId entityId) { - tenantCache.computeIfAbsent(tenantId, id -> new TenantEntityProfileCache()).add(profileId, entityId); - } - - @Override - public void update(TenantId tenantId, EntityId oldProfileId, EntityId newProfileId, EntityId entityId) { - tenantCache.compute(tenantId, (id, cache) -> { - if (cache == null) { - cache = new TenantEntityProfileCache(); - } - cache.remove(oldProfileId, entityId); - cache.add(newProfileId, entityId); - return cache; - }); - } - - @Override - public void evict(TenantId tenantId, EntityId entityId) { - var cache = tenantCache.computeIfAbsent(tenantId, id -> new TenantEntityProfileCache()); - cache.removeEntityId(entityId); - } - - @Override - public void evictProfile(TenantId tenantId, EntityId profileId) { - tenantCache.computeIfAbsent(tenantId, id -> new TenantEntityProfileCache()).removeProfileId(profileId); - } - - @Override - public void removeTenant(TenantId tenantId) { - tenantCache.remove(tenantId); - } - - @Override - public Collection getEntityIdsByProfileId(TenantId tenantId, EntityId profileId) { - return tenantCache.computeIfAbsent(tenantId, id -> new TenantEntityProfileCache()).getEntityIdsByProfileId(profileId); - } - - @Override - public int getEntityIdPartition(TenantId tenantId, EntityId entityId) { - var tpi = partitionService.resolve(ServiceType.TB_RULE_ENGINE, DataConstants.CF_QUEUE_NAME, tenantId, entityId); - return tpi.getPartition().orElse(UNKNOWN); - } - - private void initCacheForNewTenant(TenantId tenantId) { - PageDataIterable devices = new PageDataIterable<>(pageLink -> deviceService.findDevicesByTenantId(tenantId, pageLink), initFetchPackSize); - for (Device device : devices) { - log.trace("Processing device record: {}", device); - try { - if (partitionService.isManagedByCurrentService(device.getTenantId())) { - add(device.getTenantId(), device.getDeviceProfileId(), device.getId()); - } - } catch (Exception e) { - log.error("Failed to process device record: {}", device, e); - } - } - PageDataIterable assets = new PageDataIterable<>(pageLink -> assetService.findAssetsByTenantId(tenantId, pageLink), initFetchPackSize); - for (Asset asset : assets) { - log.trace("Processing asset record: {}", asset); - try { - if (partitionService.isManagedByCurrentService(asset.getTenantId())) { - add(asset.getTenantId(), asset.getAssetProfileId(), asset.getId()); - } - } catch (Exception e) { - log.error("Failed to process asset record: {}", asset, e); - } - } - } - -} diff --git a/application/src/main/java/org/thingsboard/server/service/cf/cache/TenantEntityProfileCache.java b/application/src/main/java/org/thingsboard/server/service/cf/cache/TenantEntityProfileCache.java index 27c2bdd6d5..6435ffb14c 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/cache/TenantEntityProfileCache.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/cache/TenantEntityProfileCache.java @@ -76,6 +76,11 @@ public class TenantEntityProfileCache { } } + public void update(EntityId oldProfileId, EntityId newProfileId, EntityId entityId) { + remove(oldProfileId, entityId); + add(newProfileId, entityId); + } + public Collection getEntityIdsByProfileId(EntityId profileId) { lock.readLock().lock(); try { diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCalculatedFieldConsumerService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCalculatedFieldConsumerService.java index f108949efa..ab03a56225 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCalculatedFieldConsumerService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCalculatedFieldConsumerService.java @@ -52,7 +52,6 @@ import org.thingsboard.server.queue.util.TbRuleEngineComponent; import org.thingsboard.server.service.apiusage.TbApiUsageStateService; import org.thingsboard.server.service.cf.CalculatedFieldCache; import org.thingsboard.server.service.cf.CalculatedFieldStateService; -import org.thingsboard.server.service.cf.cache.CalculatedFieldEntityProfileCache; import org.thingsboard.server.service.profile.TbAssetProfileCache; import org.thingsboard.server.service.profile.TbDeviceProfileCache; import org.thingsboard.server.service.queue.processing.AbstractPartitionBasedConsumerService; @@ -81,7 +80,6 @@ public class DefaultTbCalculatedFieldConsumerService extends AbstractPartitionBa private final TbRuleEngineQueueFactory queueFactory; private final CalculatedFieldStateService stateService; - private final CalculatedFieldEntityProfileCache entityProfileCache; public DefaultTbCalculatedFieldConsumerService(TbRuleEngineQueueFactory tbQueueFactory, ActorSystemContext actorContext, @@ -93,13 +91,11 @@ public class DefaultTbCalculatedFieldConsumerService extends AbstractPartitionBa ApplicationEventPublisher eventPublisher, JwtSettingsService jwtSettingsService, CalculatedFieldCache calculatedFieldCache, - CalculatedFieldStateService stateService, - CalculatedFieldEntityProfileCache entityProfileCache) { + CalculatedFieldStateService stateService) { super(actorContext, tenantProfileCache, deviceProfileCache, assetProfileCache, calculatedFieldCache, apiUsageStateService, partitionService, eventPublisher, jwtSettingsService); this.queueFactory = tbQueueFactory; this.stateService = stateService; - this.entityProfileCache = entityProfileCache; } @Override @@ -230,8 +226,6 @@ public class DefaultTbCalculatedFieldConsumerService extends AbstractPartitionBa public void handleComponentLifecycleEvent(ComponentLifecycleMsg event) { if (event.getEntityId().getEntityType() == EntityType.TENANT) { if (event.getEvent() == ComponentLifecycleEvent.DELETED) { - entityProfileCache.removeTenant(event.getTenantId()); - Set partitions = stateService.getPartitions(); if (CollectionUtils.isEmpty(partitions)) { return; @@ -240,14 +234,6 @@ public class DefaultTbCalculatedFieldConsumerService extends AbstractPartitionBa .filter(tpi -> tpi.getTenantId().isPresent() && tpi.getTenantId().get().equals(event.getTenantId())) .collect(Collectors.toSet())); } - } else if (event.getEntityId().getEntityType() == EntityType.ASSET_PROFILE) { - if (event.getEvent() == ComponentLifecycleEvent.DELETED) { - entityProfileCache.evictProfile(event.getTenantId(), event.getEntityId()); - } - } else if (event.getEntityId().getEntityType() == EntityType.DEVICE_PROFILE) { - if (event.getEvent() == ComponentLifecycleEvent.DELETED) { - entityProfileCache.evictProfile(event.getTenantId(), event.getEntityId()); - } } } diff --git a/common/dao-api/src/main/java/org/thingsboard/server/dao/asset/AssetService.java b/common/dao-api/src/main/java/org/thingsboard/server/dao/asset/AssetService.java index 09bc8f1f93..c22c9c4140 100644 --- a/common/dao-api/src/main/java/org/thingsboard/server/dao/asset/AssetService.java +++ b/common/dao-api/src/main/java/org/thingsboard/server/dao/asset/AssetService.java @@ -66,6 +66,8 @@ public interface AssetService extends EntityDaoService { PageData findProfileEntityIdInfos(PageLink pageLink); + PageData findProfileEntityIdInfosByTenantId(TenantId tenantId, PageLink pageLink); + PageData findAssetIdsByTenantIdAndAssetProfileId(TenantId tenantId, AssetProfileId assetProfileId, PageLink pageLink); ListenableFuture> findAssetsByTenantIdAndIdsAsync(TenantId tenantId, List assetIds); diff --git a/common/dao-api/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldService.java b/common/dao-api/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldService.java index 8507ebbd42..b6f43cda67 100644 --- a/common/dao-api/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldService.java +++ b/common/dao-api/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldService.java @@ -39,7 +39,7 @@ public interface CalculatedFieldService extends EntityDaoService { PageData findAllCalculatedFields(PageLink pageLink); - List findCalculatedFieldsByTenantId(TenantId tenantId); + PageData findCalculatedFieldsByTenantId(TenantId tenantId, PageLink pageLink); PageData findAllCalculatedFieldsByEntityId(TenantId tenantId, EntityId entityId, PageLink pageLink); @@ -57,6 +57,8 @@ public interface CalculatedFieldService extends EntityDaoService { List findAllCalculatedFieldLinksByTenantId(TenantId tenantId); + PageData findAllCalculatedFieldLinksByTenantId(TenantId tenantId, PageLink pageLink); + PageData findAllCalculatedFieldLinks(PageLink pageLink); boolean referencedInAnyCalculatedField(TenantId tenantId, EntityId referencedEntityId); diff --git a/common/dao-api/src/main/java/org/thingsboard/server/dao/device/DeviceService.java b/common/dao-api/src/main/java/org/thingsboard/server/dao/device/DeviceService.java index 4ef653855d..9eb258f182 100644 --- a/common/dao-api/src/main/java/org/thingsboard/server/dao/device/DeviceService.java +++ b/common/dao-api/src/main/java/org/thingsboard/server/dao/device/DeviceService.java @@ -76,6 +76,8 @@ public interface DeviceService extends EntityDaoService { PageData findProfileEntityIdInfos(PageLink pageLink); + PageData findProfileEntityIdInfosByTenantId(TenantId tenantId, PageLink pageLink); + PageData findDevicesByTenantIdAndType(TenantId tenantId, String type, PageLink pageLink); PageData findDeviceIdsByTenantIdAndDeviceProfileId(TenantId tenantId, DeviceProfileId deviceProfileId, PageLink pageLink); diff --git a/common/message/src/main/java/org/thingsboard/server/common/msg/MsgType.java b/common/message/src/main/java/org/thingsboard/server/common/msg/MsgType.java index 178caf7961..bfbd5fb0d6 100644 --- a/common/message/src/main/java/org/thingsboard/server/common/msg/MsgType.java +++ b/common/message/src/main/java/org/thingsboard/server/common/msg/MsgType.java @@ -136,6 +136,8 @@ public enum MsgType { EDGE_SYNC_RESPONSE_FROM_EDGE_SESSION_MSG, + CF_ACTOR_INIT_MSG, // Sent to init caches for CF actor; + CF_PROFILE_ENTITY_MSG, // Sent to init profile entities cache; CF_INIT_MSG, // Sent to init particular calculated field; CF_LINK_INIT_MSG, // Sent to init particular calculated field; CF_STATE_RESTORE_MSG, // Sent to restore particular calculated field entity state; diff --git a/common/message/src/main/java/org/thingsboard/server/common/msg/cf/CalculatedFieldActorInitMsg.java b/common/message/src/main/java/org/thingsboard/server/common/msg/cf/CalculatedFieldActorInitMsg.java new file mode 100644 index 0000000000..851c7709bb --- /dev/null +++ b/common/message/src/main/java/org/thingsboard/server/common/msg/cf/CalculatedFieldActorInitMsg.java @@ -0,0 +1,33 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.msg.cf; + +import lombok.Data; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.msg.MsgType; +import org.thingsboard.server.common.msg.ToCalculatedFieldSystemMsg; + +@Data +public class CalculatedFieldActorInitMsg implements ToCalculatedFieldSystemMsg { + + private final TenantId tenantId; + + @Override + public MsgType getMsgType() { + return MsgType.CF_ACTOR_INIT_MSG; + } + +} diff --git a/common/message/src/main/java/org/thingsboard/server/common/msg/cf/ProfileEntityMsg.java b/common/message/src/main/java/org/thingsboard/server/common/msg/cf/ProfileEntityMsg.java new file mode 100644 index 0000000000..81196c150e --- /dev/null +++ b/common/message/src/main/java/org/thingsboard/server/common/msg/cf/ProfileEntityMsg.java @@ -0,0 +1,36 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.msg.cf; + +import lombok.Data; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.msg.MsgType; +import org.thingsboard.server.common.msg.ToCalculatedFieldSystemMsg; + +@Data +public class ProfileEntityMsg implements ToCalculatedFieldSystemMsg { + + private final TenantId tenantId; + private final EntityId profileEntityId; + private final EntityId entityId; + + @Override + public MsgType getMsgType() { + return MsgType.CF_PROFILE_ENTITY_MSG; + } + +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaMonolithQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaMonolithQueueFactory.java index 1c46a624c3..87a9d6a697 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaMonolithQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaMonolithQueueFactory.java @@ -579,7 +579,7 @@ public class KafkaMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEngi .readFromBeginning(true) .stopWhenRead(true) .clientId("monolith-calculated-field-state-consumer-" + serviceInfoProvider.getServiceId() + "-" + consumerCount.incrementAndGet()) - .groupId(topicService.buildTopicName("monolith-calculated-field-state-consumer")) + .groupId(null) // not using consumer group management .decoder(msg -> new TbProtoQueueMsg<>(msg.getKey(), msg.getData() != null ? CalculatedFieldStateProto.parseFrom(msg.getData()) : null, msg.getHeaders())) .admin(cfStateAdmin) .statsService(consumerStatsService) diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbRuleEngineQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbRuleEngineQueueFactory.java index 112af9f646..fdf2b2f3f7 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbRuleEngineQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbRuleEngineQueueFactory.java @@ -379,7 +379,7 @@ public class KafkaTbRuleEngineQueueFactory implements TbRuleEngineQueueFactory { .readFromBeginning(true) .stopWhenRead(true) .clientId("tb-rule-engine-calculated-field-state-consumer-" + serviceInfoProvider.getServiceId() + "-" + consumerCount.incrementAndGet()) - .groupId(topicService.buildTopicName("tb-rule-engine-calculated-field-state-consumer")) + .groupId(null) // not using consumer group management .decoder(msg -> new TbProtoQueueMsg<>(msg.getKey(), msg.getData() != null ? CalculatedFieldStateProto.parseFrom(msg.getData()) : null, msg.getHeaders())) .admin(cfStateAdmin) .statsService(consumerStatsService) diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/util/AfterStartUp.java b/common/queue/src/main/java/org/thingsboard/server/queue/util/AfterStartUp.java index 46c29b867b..6a3aaa4a7f 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/util/AfterStartUp.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/util/AfterStartUp.java @@ -36,10 +36,11 @@ public @interface AfterStartUp { int STARTUP_SERVICE = 8; int ACTOR_SYSTEM = 9; - int REGULAR_SERVICE = 10; int CF_READ_PROFILE_ENTITIES_SERVICE = 10; - int CF_READ_CF_SERVICE = 11; + int CF_READ_CF_SERVICE = 10; + + int REGULAR_SERVICE = 11; int BEFORE_TRANSPORT_SERVICE = Integer.MAX_VALUE - 1001; int TRANSPORT_SERVICE = Integer.MAX_VALUE - 1000; diff --git a/dao/src/main/java/org/thingsboard/server/dao/asset/AssetDao.java b/dao/src/main/java/org/thingsboard/server/dao/asset/AssetDao.java index 36700ff59f..098dc4e83d 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/asset/AssetDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/asset/AssetDao.java @@ -240,4 +240,6 @@ public interface AssetDao extends Dao, TenantEntityDao, Exportable PageData findProfileEntityIdInfos(PageLink pageLink); + PageData findProfileEntityIdInfosByTenantId(UUID tenantId, PageLink pageLink); + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/asset/BaseAssetService.java b/dao/src/main/java/org/thingsboard/server/dao/asset/BaseAssetService.java index 21d5ea7f4e..fed201d403 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/asset/BaseAssetService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/asset/BaseAssetService.java @@ -293,6 +293,14 @@ public class BaseAssetService extends AbstractCachedEntityService findProfileEntityIdInfosByTenantId(TenantId tenantId, PageLink pageLink) { + log.trace("Executing findProfileEntityIdInfosByTenantId, tenantId[{}], pageLink [{}]", tenantId, pageLink); + validateId(tenantId, id -> INCORRECT_TENANT_ID + id); + validatePageLink(pageLink); + return assetDao.findProfileEntityIdInfosByTenantId(tenantId.getId(), pageLink); + } + @Override public PageData findAssetIdsByTenantIdAndAssetProfileId(TenantId tenantId, AssetProfileId assetProfileId, PageLink pageLink) { log.trace("Executing findAssetIdsByTenantIdAndAssetProfileId, tenantId [{}], assetProfileId [{}]", tenantId, assetProfileId); diff --git a/dao/src/main/java/org/thingsboard/server/dao/cf/BaseCalculatedFieldService.java b/dao/src/main/java/org/thingsboard/server/dao/cf/BaseCalculatedFieldService.java index 464041869e..d3fea73849 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/cf/BaseCalculatedFieldService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/cf/BaseCalculatedFieldService.java @@ -105,10 +105,11 @@ public class BaseCalculatedFieldService extends AbstractEntityService implements } @Override - public List findCalculatedFieldsByTenantId(TenantId tenantId) { - log.trace("Executing findAllByTenantId, tenantId [{}]", tenantId); + public PageData findCalculatedFieldsByTenantId(TenantId tenantId, PageLink pageLink) { + log.trace("Executing findAllByTenantId, tenantId [{}], pageLink [{}]", tenantId, pageLink); validateId(tenantId, id -> INCORRECT_TENANT_ID + id); - return calculatedFieldDao.findAllByTenantId(tenantId); + validatePageLink(pageLink); + return calculatedFieldDao.findAllByTenantId(tenantId, pageLink); } @Override @@ -187,6 +188,14 @@ public class BaseCalculatedFieldService extends AbstractEntityService implements return calculatedFieldLinkDao.findCalculatedFieldLinksByTenantId(tenantId); } + @Override + public PageData findAllCalculatedFieldLinksByTenantId(TenantId tenantId, PageLink pageLink) { + log.trace("Executing findAllCalculatedFieldLinksByTenantId, tenantId[{}] pageLink [{}]", tenantId, pageLink); + validateId(tenantId, id -> INCORRECT_TENANT_ID + id); + validatePageLink(pageLink); + return calculatedFieldLinkDao.findAllByTenantId(tenantId, pageLink); + } + @Override public PageData findAllCalculatedFieldLinks(PageLink pageLink) { log.trace("Executing findAllCalculatedFieldLinks, pageLink [{}]", pageLink); diff --git a/dao/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldDao.java b/dao/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldDao.java index a966977968..aadae93893 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldDao.java @@ -37,6 +37,8 @@ public interface CalculatedFieldDao extends Dao { PageData findAll(PageLink pageLink); + PageData findAllByTenantId(TenantId tenantId, PageLink pageLink); + PageData findAllByEntityId(TenantId tenantId, EntityId entityId, PageLink pageLink); List removeAllByEntityId(TenantId tenantId, EntityId entityId); diff --git a/dao/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldLinkDao.java b/dao/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldLinkDao.java index cf168a9b3d..dd184289ed 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldLinkDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldLinkDao.java @@ -37,4 +37,6 @@ public interface CalculatedFieldLinkDao extends Dao { PageData findAll(PageLink pageLink); + PageData findAllByTenantId(TenantId tenantId, PageLink pageLink); + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/device/DeviceDao.java b/dao/src/main/java/org/thingsboard/server/dao/device/DeviceDao.java index efc57119eb..6bc8903d20 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/device/DeviceDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/device/DeviceDao.java @@ -23,9 +23,7 @@ import org.thingsboard.server.common.data.DeviceInfoFilter; import org.thingsboard.server.common.data.DeviceTransportType; import org.thingsboard.server.common.data.EntitySubtype; import org.thingsboard.server.common.data.ProfileEntityIdInfo; -import org.thingsboard.server.common.data.id.AssetId; import org.thingsboard.server.common.data.id.DeviceId; -import org.thingsboard.server.common.data.id.DeviceProfileId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.ota.OtaPackageType; import org.thingsboard.server.common.data.page.PageData; @@ -233,6 +231,8 @@ public interface DeviceDao extends Dao, TenantEntityDao, Exporta PageData findProfileEntityIdInfos(PageLink pageLink); + PageData findProfileEntityIdInfosByTenantId(UUID tenantId, PageLink pageLink); + PageData findDeviceInfosByFilter(DeviceInfoFilter filter, PageLink pageLink); } diff --git a/dao/src/main/java/org/thingsboard/server/dao/device/DeviceServiceImpl.java b/dao/src/main/java/org/thingsboard/server/dao/device/DeviceServiceImpl.java index 2c890082a0..6d993f3e3d 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/device/DeviceServiceImpl.java +++ b/dao/src/main/java/org/thingsboard/server/dao/device/DeviceServiceImpl.java @@ -394,6 +394,14 @@ public class DeviceServiceImpl extends CachedVersionedEntityService findProfileEntityIdInfosByTenantId(TenantId tenantId, PageLink pageLink) { + log.trace("Executing findProfileEntityIdInfosByTenantId, tenantId[{}], pageLink [{}]", tenantId, pageLink); + validateId(tenantId, id -> INCORRECT_TENANT_ID + id); + validatePageLink(pageLink); + return deviceDao.findProfileEntityIdInfosByTenantId(tenantId.getId(), pageLink); + } + @Override public PageData findDevicesByTenantIdAndType(TenantId tenantId, String type, PageLink pageLink) { log.trace("Executing findDevicesByTenantIdAndType, tenantId [{}], type [{}], pageLink [{}]", tenantId, type, pageLink); diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/asset/JpaAssetDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/asset/JpaAssetDao.java index c61d894de5..4e99fb57e4 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/asset/JpaAssetDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/asset/JpaAssetDao.java @@ -39,14 +39,12 @@ import org.thingsboard.server.dao.model.sql.AssetEntity; import org.thingsboard.server.dao.model.sql.AssetInfoEntity; import org.thingsboard.server.dao.sql.JpaAbstractDao; import org.thingsboard.server.dao.sql.device.NativeAssetRepository; -import org.thingsboard.server.dao.sql.device.NativeDeviceRepository; import org.thingsboard.server.dao.util.SqlDao; import java.util.Arrays; import java.util.List; import java.util.Optional; import java.util.UUID; -import java.util.stream.Collectors; import static org.thingsboard.server.dao.DaoUtil.convertTenantEntityInfosToDto; @@ -262,10 +260,16 @@ public class JpaAssetDao extends JpaAbstractDao implements A @Override public PageData findProfileEntityIdInfos(PageLink pageLink) { - log.debug("Find profile device id infos by pageLink [{}]", pageLink); + log.debug("Find profile asset id infos by pageLink [{}]", pageLink); return nativeAssetRepository.findProfileEntityIdInfos(DaoUtil.toPageable(pageLink)); } + @Override + public PageData findProfileEntityIdInfosByTenantId(UUID tenantId, PageLink pageLink) { + log.debug("Find profile asset id infos by pageLink [{}]", pageLink); + return nativeAssetRepository.findProfileEntityIdInfosByTenantId(tenantId, DaoUtil.toPageable(pageLink)); + } + @Override public Long countByTenantId(TenantId tenantId) { return assetRepository.countByTenantId(tenantId.getId()); diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/cf/CalculatedFieldLinkRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/cf/CalculatedFieldLinkRepository.java index aeb8e1b04c..6f6a0775f3 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/cf/CalculatedFieldLinkRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/cf/CalculatedFieldLinkRepository.java @@ -15,6 +15,8 @@ */ package org.thingsboard.server.dao.sql.cf; +import org.springframework.data.domain.Page; +import org.springframework.data.domain.Pageable; import org.springframework.data.jpa.repository.JpaRepository; import org.thingsboard.server.dao.model.sql.CalculatedFieldLinkEntity; @@ -29,4 +31,6 @@ public interface CalculatedFieldLinkRepository extends JpaRepository findAllByTenantId(UUID tenantId); + Page findAllByTenantId(UUID tenantId, Pageable pageable); + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/cf/CalculatedFieldRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/cf/CalculatedFieldRepository.java index 0f48f3b00d..be122816ba 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/cf/CalculatedFieldRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/cf/CalculatedFieldRepository.java @@ -32,6 +32,8 @@ public interface CalculatedFieldRepository extends JpaRepository findAllByTenantIdAndEntityId(UUID tenantId, UUID entityId); + Page findAllByTenantId(UUID tenantId, Pageable pageable); + Page findAllByTenantIdAndEntityId(UUID tenantId, UUID entityId, Pageable pageable); List findAllByTenantId(UUID tenantId); diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/cf/JpaCalculatedFieldDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/cf/JpaCalculatedFieldDao.java index 8922eaca4e..4bb52c29db 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/cf/JpaCalculatedFieldDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/cf/JpaCalculatedFieldDao.java @@ -71,6 +71,12 @@ public class JpaCalculatedFieldDao extends JpaAbstractDao findAllByTenantId(TenantId tenantId, PageLink pageLink) { + log.debug("Try to find calculated fields by tenantId[{}] and pageLink [{}]", tenantId, pageLink); + return DaoUtil.toPageData(calculatedFieldRepository.findAllByTenantId(tenantId.getId(), DaoUtil.toPageable(pageLink))); + } + @Override public PageData findAllByEntityId(TenantId tenantId, EntityId entityId, PageLink pageLink) { log.debug("Try to find calculated fields by entityId[{}] and pageLink [{}]", entityId, pageLink); diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/cf/JpaCalculatedFieldLinkDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/cf/JpaCalculatedFieldLinkDao.java index 39b6e6b890..38871f2fb8 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/cf/JpaCalculatedFieldLinkDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/cf/JpaCalculatedFieldLinkDao.java @@ -70,6 +70,12 @@ public class JpaCalculatedFieldLinkDao extends JpaAbstractDao findAllByTenantId(TenantId tenantId, PageLink pageLink) { + log.debug("Try to find calculated field links by tenantId [{}], pageLink [{}]", tenantId, pageLink); + return DaoUtil.toPageData(calculatedFieldLinkRepository.findAllByTenantId(tenantId.getId(), DaoUtil.toPageable(pageLink))); + } + @Override protected Class getEntityClass() { return CalculatedFieldLinkEntity.class; diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/device/DefaultNativeAssetRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/device/DefaultNativeAssetRepository.java index 43d66e2ff0..6c2a8dc506 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/device/DefaultNativeAssetRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/device/DefaultNativeAssetRepository.java @@ -20,12 +20,9 @@ import org.springframework.data.domain.Pageable; import org.springframework.jdbc.core.namedparam.NamedParameterJdbcTemplate; import org.springframework.stereotype.Repository; import org.springframework.transaction.support.TransactionTemplate; -import org.thingsboard.server.common.data.DeviceIdInfo; import org.thingsboard.server.common.data.ProfileEntityIdInfo; import org.thingsboard.server.common.data.id.AssetId; import org.thingsboard.server.common.data.id.AssetProfileId; -import org.thingsboard.server.common.data.id.DeviceId; -import org.thingsboard.server.common.data.id.DeviceProfileId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageData; @@ -43,8 +40,8 @@ public class DefaultNativeAssetRepository extends AbstractNativeRepository imple @Override public PageData findProfileEntityIdInfos(Pageable pageable) { - String PROFILE_DEVICE_ID_INFO_QUERY = "SELECT tenant_id as tenantId, asset_profile_id as profileId, id as id FROM asset ORDER BY created_time ASC LIMIT %s OFFSET %s"; - return find(COUNT_QUERY, PROFILE_DEVICE_ID_INFO_QUERY, pageable, row -> { + String PROFILE_ASSET_ID_INFO_QUERY = "SELECT tenant_id as tenantId, asset_profile_id as profileId, id as id FROM asset ORDER BY created_time ASC LIMIT %s OFFSET %s"; + return find(COUNT_QUERY, PROFILE_ASSET_ID_INFO_QUERY, pageable, row -> { AssetId id = new AssetId((UUID) row.get("id")); AssetProfileId profileId = new AssetProfileId((UUID) row.get("profileId")); var tenantIdObj = row.get("tenantId"); @@ -52,4 +49,14 @@ public class DefaultNativeAssetRepository extends AbstractNativeRepository imple }); } + @Override + public PageData findProfileEntityIdInfosByTenantId(UUID tenantId, Pageable pageable) { + String PROFILE_ASSET_ID_INFO_QUERY = String.format("SELECT tenant_id as tenantId, asset_profile_id as profileId, id as id FROM asset WHERE tenant_id = %s ORDER BY created_time ASC LIMIT %%s OFFSET %%s", tenantId); + return find(COUNT_QUERY, PROFILE_ASSET_ID_INFO_QUERY, pageable, row -> { + AssetId id = new AssetId((UUID) row.get("id")); + AssetProfileId profileId = new AssetProfileId((UUID) row.get("profileId")); + var tenantIdObj = row.get("tenantId"); + return ProfileEntityIdInfo.create(tenantIdObj != null ? (UUID) tenantIdObj : TenantId.SYS_TENANT_ID.getId(), profileId, id); + }); + } } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/device/DefaultNativeDeviceRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/device/DefaultNativeDeviceRepository.java index 776dedc2d5..1648bb2255 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/device/DefaultNativeDeviceRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/device/DefaultNativeDeviceRepository.java @@ -61,4 +61,15 @@ public class DefaultNativeDeviceRepository extends AbstractNativeRepository impl }); } + @Override + public PageData findProfileEntityIdInfosByTenantId(UUID tenantId, Pageable pageable) { + String PROFILE_DEVICE_ID_INFO_QUERY = String.format("SELECT tenant_id as tenantId, device_profile_id as profileId, id as id FROM device WHERE tenant_id = %s ORDER BY created_time ASC LIMIT %%s OFFSET %%s", tenantId); + return find(COUNT_QUERY, PROFILE_DEVICE_ID_INFO_QUERY, pageable, row -> { + DeviceId id = new DeviceId((UUID) row.get("id")); + DeviceProfileId profileId = new DeviceProfileId((UUID) row.get("profileId")); + var tenantIdObj = row.get("tenantId"); + return ProfileEntityIdInfo.create(tenantIdObj != null ? (UUID) tenantIdObj : TenantId.SYS_TENANT_ID.getId(), profileId, id); + }); + } + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/device/JpaDeviceDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/device/JpaDeviceDao.java index 34835f52f1..9c79637e39 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/device/JpaDeviceDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/device/JpaDeviceDao.java @@ -179,11 +179,11 @@ public class JpaDeviceDao extends JpaAbstractDao implement @Override public PageData findDeviceIdsByTenantIdAndDeviceProfileId(UUID tenantId, UUID deviceProfileId, PageLink pageLink) { return DaoUtil.pageToPageData( - deviceRepository.findIdsByTenantIdAndDeviceProfileId( - tenantId, - deviceProfileId, - pageLink.getTextSearch(), - DaoUtil.toPageable(pageLink))) + deviceRepository.findIdsByTenantIdAndDeviceProfileId( + tenantId, + deviceProfileId, + pageLink.getTextSearch(), + DaoUtil.toPageable(pageLink))) .mapData(DeviceId::new); } @@ -281,6 +281,12 @@ public class JpaDeviceDao extends JpaAbstractDao implement return nativeDeviceRepository.findProfileEntityIdInfos(DaoUtil.toPageable(pageLink)); } + @Override + public PageData findProfileEntityIdInfosByTenantId(UUID tenantId, PageLink pageLink) { + log.debug("Find profile device id infos by tenantId[{}], pageLink [{}]", tenantId, pageLink); + return nativeDeviceRepository.findProfileEntityIdInfosByTenantId(tenantId, DaoUtil.toPageable(pageLink)); + } + @Override public Device findByTenantIdAndExternalId(UUID tenantId, UUID externalId) { return DaoUtil.getData(deviceRepository.findByTenantIdAndExternalId(tenantId, externalId)); diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/device/NativeProfileEntityRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/device/NativeProfileEntityRepository.java index 750f0c8787..3568f01851 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/device/NativeProfileEntityRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/device/NativeProfileEntityRepository.java @@ -19,8 +19,12 @@ import org.springframework.data.domain.Pageable; import org.thingsboard.server.common.data.ProfileEntityIdInfo; import org.thingsboard.server.common.data.page.PageData; +import java.util.UUID; + public interface NativeProfileEntityRepository { PageData findProfileEntityIdInfos(Pageable pageable); + PageData findProfileEntityIdInfosByTenantId(UUID tenantId, Pageable pageable); + } From ec657df3cb3653ca49ea24b63dde155d02482e22 Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Thu, 10 Apr 2025 17:03:41 +0300 Subject: [PATCH 26/47] returned queue name --- .../service/cf/ctx/state/KafkaCalculatedFieldStateService.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/KafkaCalculatedFieldStateService.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/KafkaCalculatedFieldStateService.java index 19d2b74bb5..533b487d38 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/KafkaCalculatedFieldStateService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/KafkaCalculatedFieldStateService.java @@ -106,7 +106,7 @@ public class KafkaCalculatedFieldStateService extends AbstractCalculatedFieldSta @Override protected void doPersist(CalculatedFieldEntityCtxId stateId, CalculatedFieldStateProto stateMsgProto, TbCallback callback) { - TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_RULE_ENGINE, DataConstants.CF_QUEUE_NAME, stateId.tenantId(), stateId.entityId()); + TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_RULE_ENGINE, DataConstants.CF_STATES_QUEUE_NAME, stateId.tenantId(), stateId.entityId()); TbProtoQueueMsg msg = new TbProtoQueueMsg<>(stateId.entityId().getId(), stateMsgProto); if (stateMsgProto == null) { putStateId(msg.getHeaders(), stateId); From 8567dd0bd24d2535f80211a53a01e8848698888d Mon Sep 17 00:00:00 2001 From: ViacheslavKlimov Date: Thu, 10 Apr 2025 17:41:19 +0300 Subject: [PATCH 27/47] Fix partition change events handling by consumer services --- ...faultTbCalculatedFieldConsumerService.java | 2 -- .../queue/DefaultTbCoreConsumerService.java | 2 -- .../queue/DefaultTbEdgeConsumerService.java | 5 +---- .../DefaultTbRuleEngineConsumerService.java | 2 -- .../processing/AbstractConsumerService.java | 6 ++++-- ...AbstractPartitionBasedConsumerService.java | 19 ++++++++++++++++--- 6 files changed, 21 insertions(+), 15 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCalculatedFieldConsumerService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCalculatedFieldConsumerService.java index 125a06299d..e3bc3be90d 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCalculatedFieldConsumerService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCalculatedFieldConsumerService.java @@ -16,7 +16,6 @@ package org.thingsboard.server.service.queue; import jakarta.annotation.PreDestroy; -import lombok.extern.slf4j.Slf4j; import org.apache.commons.collections4.CollectionUtils; import org.springframework.beans.factory.annotation.Value; import org.springframework.context.ApplicationEventPublisher; @@ -70,7 +69,6 @@ import java.util.stream.Collectors; @Service @TbRuleEngineComponent -@Slf4j public class DefaultTbCalculatedFieldConsumerService extends AbstractPartitionBasedConsumerService implements TbCalculatedFieldConsumerService { @Value("${queue.calculated_fields.poll_interval:25}") diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java index a3003ba6ff..e115e4f7b5 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java @@ -20,7 +20,6 @@ import com.google.common.util.concurrent.ListeningExecutorService; import com.google.common.util.concurrent.MoreExecutors; import jakarta.annotation.PostConstruct; import jakarta.annotation.PreDestroy; -import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; import org.springframework.beans.factory.annotation.Value; import org.springframework.context.ApplicationEventPublisher; @@ -118,7 +117,6 @@ import java.util.stream.Collectors; @Service @TbCoreComponent -@Slf4j public class DefaultTbCoreConsumerService extends AbstractConsumerService implements TbCoreConsumerService { @Value("${queue.core.poll-interval}") diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbEdgeConsumerService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbEdgeConsumerService.java index 2e4c15f8e8..d3dc2932f0 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbEdgeConsumerService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbEdgeConsumerService.java @@ -19,8 +19,6 @@ import com.google.common.util.concurrent.FutureCallback; import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; import com.google.common.util.concurrent.MoreExecutors; -import lombok.Data; -import lombok.extern.slf4j.Slf4j; import org.checkerframework.checker.nullness.qual.Nullable; import org.jetbrains.annotations.NotNull; import org.springframework.beans.factory.annotation.Value; @@ -45,12 +43,12 @@ import org.thingsboard.server.gen.transport.TransportProtos.ToEdgeMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToEdgeNotificationMsg; import org.thingsboard.server.queue.TbQueueConsumer; import org.thingsboard.server.queue.common.TbProtoQueueMsg; +import org.thingsboard.server.queue.common.consumer.MainQueueConsumerManager; import org.thingsboard.server.queue.discovery.QueueKey; import org.thingsboard.server.queue.discovery.event.PartitionChangeEvent; import org.thingsboard.server.queue.provider.TbCoreQueueFactory; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.edge.EdgeContextComponent; -import org.thingsboard.server.queue.common.consumer.MainQueueConsumerManager; import org.thingsboard.server.service.edge.rpc.EdgeRpcService; import org.thingsboard.server.service.queue.processing.AbstractConsumerService; import org.thingsboard.server.service.queue.processing.IdMsgPair; @@ -66,7 +64,6 @@ import java.util.concurrent.Future; import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; -@Slf4j @Service @TbCoreComponent public class DefaultTbEdgeConsumerService extends AbstractConsumerService implements TbEdgeConsumerService { diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbRuleEngineConsumerService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbRuleEngineConsumerService.java index 98f857750a..f51678310e 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbRuleEngineConsumerService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbRuleEngineConsumerService.java @@ -15,7 +15,6 @@ */ package org.thingsboard.server.service.queue; -import lombok.extern.slf4j.Slf4j; import org.springframework.context.ApplicationEventPublisher; import org.springframework.context.event.EventListener; import org.springframework.scheduling.annotation.Scheduled; @@ -65,7 +64,6 @@ import java.util.stream.Collectors; @Service @TbRuleEngineComponent -@Slf4j public class DefaultTbRuleEngineConsumerService extends AbstractPartitionBasedConsumerService implements TbRuleEngineConsumerService { private final TbRuleEngineConsumerContext ctx; diff --git a/application/src/main/java/org/thingsboard/server/service/queue/processing/AbstractConsumerService.java b/application/src/main/java/org/thingsboard/server/service/queue/processing/AbstractConsumerService.java index d12bd896ff..26b689df9d 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/processing/AbstractConsumerService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/processing/AbstractConsumerService.java @@ -17,7 +17,8 @@ package org.thingsboard.server.service.queue.processing; import jakarta.annotation.PreDestroy; import lombok.RequiredArgsConstructor; -import lombok.extern.slf4j.Slf4j; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import org.springframework.context.ApplicationEventPublisher; import org.thingsboard.common.util.ThingsBoardExecutors; import org.thingsboard.common.util.ThingsBoardThreadFactory; @@ -62,10 +63,11 @@ import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; -@Slf4j @RequiredArgsConstructor public abstract class AbstractConsumerService extends TbApplicationEventListener { + protected final Logger log = LoggerFactory.getLogger(getClass()); + protected final ActorSystemContext actorContext; protected final TbTenantProfileCache tenantProfileCache; protected final TbDeviceProfileCache deviceProfileCache; diff --git a/application/src/main/java/org/thingsboard/server/service/queue/processing/AbstractPartitionBasedConsumerService.java b/application/src/main/java/org/thingsboard/server/service/queue/processing/AbstractPartitionBasedConsumerService.java index 97aa81d41c..f42908bcb5 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/processing/AbstractPartitionBasedConsumerService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/processing/AbstractPartitionBasedConsumerService.java @@ -28,6 +28,8 @@ import org.thingsboard.server.service.profile.TbAssetProfileCache; import org.thingsboard.server.service.profile.TbDeviceProfileCache; import org.thingsboard.server.service.security.auth.jwt.settings.JwtSettingsService; +import java.util.ArrayList; +import java.util.List; import java.util.concurrent.locks.Lock; import java.util.concurrent.locks.ReentrantLock; @@ -35,7 +37,7 @@ public abstract class AbstractPartitionBasedConsumerService pendingEvents = new ArrayList<>(); public AbstractPartitionBasedConsumerService(ActorSystemContext actorContext, TbTenantProfileCache tenantProfileCache, @@ -61,8 +63,16 @@ public abstract class AbstractPartitionBasedConsumerService Date: Thu, 10 Apr 2025 18:00:22 +0300 Subject: [PATCH 28/47] UI: Add change detection when updating dashboard state to prevent rare cases when compiled template in markdown widget is not updated. --- .../home/components/dashboard-page/dashboard-page.component.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/ui-ngx/src/app/modules/home/components/dashboard-page/dashboard-page.component.ts b/ui-ngx/src/app/modules/home/components/dashboard-page/dashboard-page.component.ts index 9490db52a4..60d05d4c86 100644 --- a/ui-ngx/src/app/modules/home/components/dashboard-page/dashboard-page.component.ts +++ b/ui-ngx/src/app/modules/home/components/dashboard-page/dashboard-page.component.ts @@ -1118,6 +1118,7 @@ export class DashboardPageComponent extends PageComponent implements IDashboardC this.dashboardCtx.aliasController.dashboardStateChanged(); this.isRightLayoutOpened = openRightLayout ? true : false; this.updateLayouts(layoutsData); + this.cd.markForCheck(); } setTimeout(() => { this.mobileService.onDashboardLoaded(this.layouts.right.show, this.isRightLayoutOpened); From 9bf528eee584a1c2fc9bd835318983a981ed41fd Mon Sep 17 00:00:00 2001 From: Volodymyr Babak Date: Thu, 10 Apr 2025 18:05:21 +0300 Subject: [PATCH 29/47] Edge notification unique topic per edge must have the same group as topic name to avoid rebalance in case other edge disconnects --- .../server/queue/provider/KafkaMonolithQueueFactory.java | 5 +++-- .../server/queue/provider/KafkaTbCoreQueueFactory.java | 5 +++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaMonolithQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaMonolithQueueFactory.java index dcebe085b3..508cdff2a9 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaMonolithQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaMonolithQueueFactory.java @@ -493,9 +493,10 @@ public class KafkaMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEngi public TbQueueConsumer> createEdgeEventMsgConsumer(TenantId tenantId, EdgeId edgeId) { TbKafkaConsumerTemplate.TbKafkaConsumerTemplateBuilder> consumerBuilder = TbKafkaConsumerTemplate.builder(); consumerBuilder.settings(kafkaSettings); - consumerBuilder.topic(topicService.buildEdgeEventNotificationsTopicPartitionInfo(tenantId, edgeId).getTopic()); + String topic = topicService.buildEdgeEventNotificationsTopicPartitionInfo(tenantId, edgeId).getTopic(); + consumerBuilder.topic(topic); consumerBuilder.clientId("monolith-to-edge-event-consumer-" + serviceInfoProvider.getServiceId() + "-" + edgeConsumerCount.incrementAndGet()); - consumerBuilder.groupId(topicService.buildTopicName("monolith-edge-event-consumer")); + consumerBuilder.groupId(topic); consumerBuilder.decoder(msg -> new TbProtoQueueMsg<>(msg.getKey(), ToEdgeEventNotificationMsg.parseFrom(msg.getData()), msg.getHeaders())); consumerBuilder.admin(edgeEventAdmin); consumerBuilder.statsService(consumerStatsService); diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbCoreQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbCoreQueueFactory.java index e9c42b0022..14766ecbdb 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbCoreQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbCoreQueueFactory.java @@ -439,9 +439,10 @@ public class KafkaTbCoreQueueFactory implements TbCoreQueueFactory { public TbQueueConsumer> createEdgeEventMsgConsumer(TenantId tenantId, EdgeId edgeId) { TbKafkaConsumerTemplate.TbKafkaConsumerTemplateBuilder> consumerBuilder = TbKafkaConsumerTemplate.builder(); consumerBuilder.settings(kafkaSettings); - consumerBuilder.topic(topicService.buildEdgeEventNotificationsTopicPartitionInfo(tenantId, edgeId).getTopic()); + String topic = topicService.buildEdgeEventNotificationsTopicPartitionInfo(tenantId, edgeId).getTopic(); + consumerBuilder.topic(topic); consumerBuilder.clientId("tb-core-edge-event-consumer-" + serviceInfoProvider.getServiceId() + "-" + edgeConsumerCount.incrementAndGet()); - consumerBuilder.groupId(topicService.buildTopicName("tb-core-edge-event-consumer")); + consumerBuilder.groupId(topic); consumerBuilder.decoder(msg -> new TbProtoQueueMsg<>(msg.getKey(), ToEdgeEventNotificationMsg.parseFrom(msg.getData()), msg.getHeaders())); consumerBuilder.admin(edgeEventAdmin); consumerBuilder.statsService(consumerStatsService); From 2cfa9a0103fbe44a0e75ca5cac871722e9533b86 Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Fri, 11 Apr 2025 09:33:46 +0300 Subject: [PATCH 30/47] added handling of new messages to app actor --- .../main/java/org/thingsboard/server/actors/app/AppActor.java | 2 ++ .../java/org/thingsboard/server/actors/tenant/TenantActor.java | 1 + 2 files changed, 3 insertions(+) diff --git a/application/src/main/java/org/thingsboard/server/actors/app/AppActor.java b/application/src/main/java/org/thingsboard/server/actors/app/AppActor.java index 904547a2b6..d3f8f0d3ca 100644 --- a/application/src/main/java/org/thingsboard/server/actors/app/AppActor.java +++ b/application/src/main/java/org/thingsboard/server/actors/app/AppActor.java @@ -113,6 +113,8 @@ public class AppActor extends ContextAwareActor { case SESSION_TIMEOUT_MSG: ctx.broadcastToChildrenByType(msg, EntityType.TENANT); break; + case CF_ACTOR_INIT_MSG: + case CF_PROFILE_ENTITY_MSG: case CF_INIT_MSG: case CF_LINK_INIT_MSG: case CF_STATE_RESTORE_MSG: diff --git a/application/src/main/java/org/thingsboard/server/actors/tenant/TenantActor.java b/application/src/main/java/org/thingsboard/server/actors/tenant/TenantActor.java index 4e04962dcb..68d4082f7f 100644 --- a/application/src/main/java/org/thingsboard/server/actors/tenant/TenantActor.java +++ b/application/src/main/java/org/thingsboard/server/actors/tenant/TenantActor.java @@ -282,6 +282,7 @@ public class TenantActor extends RuleChainManagerActor { log.info("[{}] Failed to init CF Actor.", tenantId, e); } } + cfActor.tellWithHighPriority(msg); if (!ruleChainsInitialized) { log.info("Tenant {} is now managed by this service, initializing rule chains", tenantId); initRuleChains(); From 413cc12126796aea78fb6d98d6a7a2ff1113a6db Mon Sep 17 00:00:00 2001 From: Artem Dzhereleiko Date: Fri, 11 Apr 2025 09:34:44 +0300 Subject: [PATCH 31/47] UI: Handle error on apply preview upload and download --- .../scada-symbol/scada-symbol.component.ts | 96 ++++++++++--------- .../image/upload-image-dialog.component.html | 2 +- .../image/upload-image-dialog.component.ts | 63 +++++++----- 3 files changed, 90 insertions(+), 71 deletions(-) diff --git a/ui-ngx/src/app/modules/home/pages/scada-symbol/scada-symbol.component.ts b/ui-ngx/src/app/modules/home/pages/scada-symbol/scada-symbol.component.ts index c618380565..a2b47ee811 100644 --- a/ui-ngx/src/app/modules/home/pages/scada-symbol/scada-symbol.component.ts +++ b/ui-ngx/src/app/modules/home/pages/scada-symbol/scada-symbol.component.ts @@ -253,43 +253,47 @@ export class ScadaSymbolComponent extends PageComponent enterPreviewMode() { this.previewMetadata = this.scadaSymbolFormGroup.get('metadata').value; - this.symbolData.scadaSymbolContent = this.prepareScadaSymbolContent(this.previewMetadata); - this.previewScadaSymbolObjectSettings = { - behavior: {}, - properties: {} - }; - this.scadaPreviewFormGroup.patchValue({ - scadaSymbolObjectSettings: this.previewScadaSymbolObjectSettings - }, {emitEvent: false}); - this.scadaPreviewFormGroup.markAsPristine(); - const settings: ScadaSymbolWidgetSettings = {...scadaSymbolWidgetDefaultSettings, - ...{ + try { + this.symbolData.scadaSymbolContent = this.prepareScadaSymbolContent(this.previewMetadata); + this.previewScadaSymbolObjectSettings = { + behavior: {}, + properties: {} + }; + this.scadaPreviewFormGroup.patchValue({ + scadaSymbolObjectSettings: this.previewScadaSymbolObjectSettings + }, {emitEvent: false}); + this.scadaPreviewFormGroup.markAsPristine(); + const settings: ScadaSymbolWidgetSettings = {...scadaSymbolWidgetDefaultSettings, + ...{ simulated: true, scadaSymbolUrl: null, scadaSymbolContent: this.symbolData.scadaSymbolContent, scadaSymbolObjectSettings: this.previewScadaSymbolObjectSettings, padding: '0', background: colorBackground('rgba(0,0,0,0)') - } - }; - this.previewWidget = { - typeFullFqn: 'system.scada_symbol', - type: widgetType.rpc, - sizeX: this.previewMetadata.widgetSizeX || 3, - sizeY: this.previewMetadata.widgetSizeY || 3, - row: 0, - col: 0, - config: { - settings, - showTitle: false, - dropShadow: false, - padding: '0', - margin: '0', - backgroundColor: 'rgba(0,0,0,0)' - } - }; - this.previewWidgets = [this.previewWidget]; - this.previewMode = true; + } + }; + this.previewWidget = { + typeFullFqn: 'system.scada_symbol', + type: widgetType.rpc, + sizeX: this.previewMetadata.widgetSizeX || 3, + sizeY: this.previewMetadata.widgetSizeY || 3, + row: 0, + col: 0, + config: { + settings, + showTitle: false, + dropShadow: false, + padding: '0', + margin: '0', + backgroundColor: 'rgba(0,0,0,0)' + } + }; + this.previewWidgets = [this.previewWidget]; + this.previewMode = true; + } catch (e) { + this.store.dispatch(new ActionNotificationShow({ message: e.message, type: 'error' })); + } } exitPreviewMode() { @@ -379,19 +383,23 @@ export class ScadaSymbolComponent extends PageComponent metadata = parseScadaSymbolMetadataFromContent(this.origSymbolData.scadaSymbolContent); } const linkElement = document.createElement('a'); - const scadaSymbolContent = this.prepareScadaSymbolContent(metadata); - const blob = new Blob([scadaSymbolContent], { type: this.symbolData.imageResource.descriptor.mediaType }); - const url = URL.createObjectURL(blob); - linkElement.setAttribute('href', url); - linkElement.setAttribute('download', this.symbolData.imageResource.fileName); - const clickEvent = new MouseEvent('click', - { - view: window, - bubbles: true, - cancelable: false - } - ); - linkElement.dispatchEvent(clickEvent); + try { + const scadaSymbolContent = this.prepareScadaSymbolContent(metadata); + const blob = new Blob([scadaSymbolContent], { type: this.symbolData.imageResource.descriptor.mediaType }); + const url = URL.createObjectURL(blob); + linkElement.setAttribute('href', url); + linkElement.setAttribute('download', this.symbolData.imageResource.fileName); + const clickEvent = new MouseEvent('click', + { + view: window, + bubbles: true, + cancelable: false + } + ); + linkElement.dispatchEvent(clickEvent); + } catch (e) { + this.store.dispatch(new ActionNotificationShow({ message: e.message, type: 'error' })); + } } createWidget() { diff --git a/ui-ngx/src/app/shared/components/image/upload-image-dialog.component.html b/ui-ngx/src/app/shared/components/image/upload-image-dialog.component.html index bbaf7d88ae..11dffb1b25 100644 --- a/ui-ngx/src/app/shared/components/image/upload-image-dialog.component.html +++ b/ui-ngx/src/app/shared/components/image/upload-image-dialog.component.html @@ -15,7 +15,7 @@ limitations under the License. --> -

+

{{ ( uploadImage ? (isScada ? 'scada.upload-symbol' : 'image.upload-image') : (isScada ? 'scada.update-symbol' : 'image.update-image') ) | translate }}

diff --git a/ui-ngx/src/app/shared/components/image/upload-image-dialog.component.ts b/ui-ngx/src/app/shared/components/image/upload-image-dialog.component.ts index 86bf9c4cd8..779cc021d4 100644 --- a/ui-ngx/src/app/shared/components/image/upload-image-dialog.component.ts +++ b/ui-ngx/src/app/shared/components/image/upload-image-dialog.component.ts @@ -41,6 +41,7 @@ import { updateScadaSymbolMetadataInContent } from '@home/components/widget/lib/scada/scada-symbol.models'; import { takeUntilDestroyed } from '@angular/core/rxjs-interop'; +import { ActionNotificationShow } from '@core/notification/notification.actions'; export interface UploadImageDialogData { imageSubType: ResourceSubType; @@ -135,38 +136,48 @@ export class UploadImageDialogComponent extends upload(): void { this.submitted = true; let file: File = this.uploadImageFormGroup.get('file').value; - if (this.uploadImage) { - const title: string = this.uploadImageFormGroup.get('title').value; - if (this.isScada) { - if (!this.scadaSymbolMetadata) { - this.scadaSymbolMetadata = emptyMetadata(); - } - if (this.scadaSymbolMetadata.title !== title) { - this.scadaSymbolMetadata.title = title; + try { + if (this.uploadImage) { + const title: string = this.uploadImageFormGroup.get('title').value; + if (this.isScada) { + if (!this.scadaSymbolMetadata) { + this.scadaSymbolMetadata = emptyMetadata(); + } + if (this.scadaSymbolMetadata.title !== title) { + this.scadaSymbolMetadata.title = title; + } + const newContent = updateScadaSymbolMetadataInContent(this.scadaSymbolContent, this.scadaSymbolMetadata); + file = updateFileContent(file, newContent); } - const newContent = updateScadaSymbolMetadataInContent(this.scadaSymbolContent, this.scadaSymbolMetadata); - file = updateFileContent(file, newContent); - } - forkJoin([ - this.imageService.uploadImage(file, title, this.data.imageSubType), - blobToBase64(file) - ]).subscribe(([imageInfo, base64]) => { - this.dialogRef.close({image: Object.assign(imageInfo, {base64})}); - }); - } else { - if (this.isScada) { - blobToText(file).subscribe(scadaSymbolContent => { - this.dialogRef.close({scadaSymbolContent}); - }); - } else { - const image = this.data.image; forkJoin([ - this.imageService.updateImage(imageResourceType(image), image.resourceKey, file), + this.imageService.uploadImage(file, title, this.data.imageSubType), blobToBase64(file) ]).subscribe(([imageInfo, base64]) => { - this.dialogRef.close({image:Object.assign(imageInfo, {base64})}); + this.dialogRef.close({image: Object.assign(imageInfo, {base64})}); }); + } else { + if (this.isScada) { + blobToText(file).subscribe(scadaSymbolContent => { + this.dialogRef.close({scadaSymbolContent}); + }); + } else { + const image = this.data.image; + forkJoin([ + this.imageService.updateImage(imageResourceType(image), image.resourceKey, file), + blobToBase64(file) + ]).subscribe(([imageInfo, base64]) => { + this.dialogRef.close({image:Object.assign(imageInfo, {base64})}); + }); + } } + } catch (e) { + this.store.dispatch(new ActionNotificationShow({ + message: e.message, + type: 'error', + verticalPosition: 'bottom', + horizontalPosition: 'right', + target: 'uploadRoot' + })); } } } From a6cdc170cb9d45c94e22031de0c325f648f912e8 Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Fri, 11 Apr 2025 09:49:58 +0300 Subject: [PATCH 32/47] renamed new msgs --- .../org/thingsboard/server/actors/app/AppActor.java | 2 +- .../calculatedField/CalculatedFieldManagerActor.java | 10 +++++----- .../CalculatedFieldManagerMessageProcessor.java | 8 ++++---- .../thingsboard/server/actors/tenant/TenantActor.java | 6 +++--- .../service/cf/DefaultCalculatedFieldInitService.java | 6 +++--- .../org/thingsboard/server/common/msg/MsgType.java | 2 +- ...orInitMsg.java => CalculatedFieldCacheInitMsg.java} | 4 ++-- ...tyMsg.java => CalculatedFieldProfileEntityMsg.java} | 2 +- 8 files changed, 20 insertions(+), 20 deletions(-) rename common/message/src/main/java/org/thingsboard/server/common/msg/cf/{CalculatedFieldActorInitMsg.java => CalculatedFieldCacheInitMsg.java} (90%) rename common/message/src/main/java/org/thingsboard/server/common/msg/cf/{ProfileEntityMsg.java => CalculatedFieldProfileEntityMsg.java} (93%) diff --git a/application/src/main/java/org/thingsboard/server/actors/app/AppActor.java b/application/src/main/java/org/thingsboard/server/actors/app/AppActor.java index d3f8f0d3ca..fc9f344563 100644 --- a/application/src/main/java/org/thingsboard/server/actors/app/AppActor.java +++ b/application/src/main/java/org/thingsboard/server/actors/app/AppActor.java @@ -113,7 +113,7 @@ public class AppActor extends ContextAwareActor { case SESSION_TIMEOUT_MSG: ctx.broadcastToChildrenByType(msg, EntityType.TENANT); break; - case CF_ACTOR_INIT_MSG: + case CF_CACHE_INIT_MSG: case CF_PROFILE_ENTITY_MSG: case CF_INIT_MSG: case CF_LINK_INIT_MSG: diff --git a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerActor.java b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerActor.java index 511449c2a9..d5a3da03e4 100644 --- a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerActor.java +++ b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerActor.java @@ -22,12 +22,12 @@ import org.thingsboard.server.actors.TbActorException; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.msg.TbActorStopReason; import org.thingsboard.server.common.msg.ToCalculatedFieldSystemMsg; -import org.thingsboard.server.common.msg.cf.CalculatedFieldActorInitMsg; +import org.thingsboard.server.common.msg.cf.CalculatedFieldCacheInitMsg; import org.thingsboard.server.common.msg.cf.CalculatedFieldEntityLifecycleMsg; import org.thingsboard.server.common.msg.cf.CalculatedFieldInitMsg; import org.thingsboard.server.common.msg.cf.CalculatedFieldLinkInitMsg; import org.thingsboard.server.common.msg.cf.CalculatedFieldPartitionChangeMsg; -import org.thingsboard.server.common.msg.cf.ProfileEntityMsg; +import org.thingsboard.server.common.msg.cf.CalculatedFieldProfileEntityMsg; /** * Created by ashvayka on 15.03.18. @@ -67,11 +67,11 @@ public class CalculatedFieldManagerActor extends AbstractCalculatedFieldActor { case CF_PARTITIONS_CHANGE_MSG: processor.onPartitionChange((CalculatedFieldPartitionChangeMsg) msg); break; - case CF_ACTOR_INIT_MSG: - processor.onActorInitMsg((CalculatedFieldActorInitMsg) msg); + case CF_CACHE_INIT_MSG: + processor.onCacheInitMsg((CalculatedFieldCacheInitMsg) msg); break; case CF_PROFILE_ENTITY_MSG: - processor.onProfileEntityMsg((ProfileEntityMsg) msg); + processor.onProfileEntityMsg((CalculatedFieldProfileEntityMsg) msg); break; case CF_INIT_MSG: processor.onFieldInitMsg((CalculatedFieldInitMsg) msg); diff --git a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java index 9dad4e820e..e62c531b16 100644 --- a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java +++ b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java @@ -35,12 +35,12 @@ import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageDataIterable; -import org.thingsboard.server.common.msg.cf.CalculatedFieldActorInitMsg; +import org.thingsboard.server.common.msg.cf.CalculatedFieldCacheInitMsg; import org.thingsboard.server.common.msg.cf.CalculatedFieldEntityLifecycleMsg; import org.thingsboard.server.common.msg.cf.CalculatedFieldInitMsg; import org.thingsboard.server.common.msg.cf.CalculatedFieldLinkInitMsg; import org.thingsboard.server.common.msg.cf.CalculatedFieldPartitionChangeMsg; -import org.thingsboard.server.common.msg.cf.ProfileEntityMsg; +import org.thingsboard.server.common.msg.cf.CalculatedFieldProfileEntityMsg; import org.thingsboard.server.common.msg.plugin.ComponentLifecycleMsg; import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.common.msg.queue.TbCallback; @@ -116,14 +116,14 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware ctx.stop(ctx.getSelf()); } - public void onActorInitMsg(CalculatedFieldActorInitMsg msg) { + public void onCacheInitMsg(CalculatedFieldCacheInitMsg msg) { log.debug("[{}] Processing CF actor init message.", msg.getTenantId().getId()); initEntityProfileCache(); initCalculatedFields(); msg.getCallback().onSuccess(); } - public void onProfileEntityMsg(ProfileEntityMsg msg) { + public void onProfileEntityMsg(CalculatedFieldProfileEntityMsg msg) { log.debug("[{}] Processing profile entity message.", msg.getTenantId().getId()); entityProfileCache.add(msg.getProfileEntityId(), msg.getEntityId()); msg.getCallback().onSuccess(); diff --git a/application/src/main/java/org/thingsboard/server/actors/tenant/TenantActor.java b/application/src/main/java/org/thingsboard/server/actors/tenant/TenantActor.java index 68d4082f7f..a5be701d38 100644 --- a/application/src/main/java/org/thingsboard/server/actors/tenant/TenantActor.java +++ b/application/src/main/java/org/thingsboard/server/actors/tenant/TenantActor.java @@ -50,7 +50,7 @@ import org.thingsboard.server.common.msg.TbMsg; import org.thingsboard.server.common.msg.ToCalculatedFieldSystemMsg; import org.thingsboard.server.common.msg.aware.DeviceAwareMsg; import org.thingsboard.server.common.msg.aware.RuleChainAwareMsg; -import org.thingsboard.server.common.msg.cf.CalculatedFieldActorInitMsg; +import org.thingsboard.server.common.msg.cf.CalculatedFieldCacheInitMsg; import org.thingsboard.server.common.msg.cf.CalculatedFieldEntityLifecycleMsg; import org.thingsboard.server.common.msg.plugin.ComponentLifecycleMsg; import org.thingsboard.server.common.msg.queue.PartitionChangeMsg; @@ -177,7 +177,7 @@ public class TenantActor extends RuleChainManagerActor { case RULE_CHAIN_TO_RULE_CHAIN_MSG: onRuleChainMsg((RuleChainAwareMsg) msg); break; - case CF_ACTOR_INIT_MSG: + case CF_CACHE_INIT_MSG: case CF_PROFILE_ENTITY_MSG: case CF_INIT_MSG: case CF_LINK_INIT_MSG: @@ -277,7 +277,7 @@ public class TenantActor extends RuleChainManagerActor { () -> DefaultActorService.CF_MANAGER_DISPATCHER_NAME, () -> new CalculatedFieldManagerActorCreator(systemContext, tenantId), () -> true); - cfActor.tellWithHighPriority(new CalculatedFieldActorInitMsg(tenantId)); + cfActor.tellWithHighPriority(new CalculatedFieldCacheInitMsg(tenantId)); } catch (Exception e) { log.info("[{}] Failed to init CF Actor.", tenantId, e); } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldInitService.java b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldInitService.java index be27b5318b..826a5243cf 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldInitService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldInitService.java @@ -23,7 +23,7 @@ import org.springframework.stereotype.Service; import org.thingsboard.server.actors.ActorSystemContext; import org.thingsboard.server.common.data.ProfileEntityIdInfo; import org.thingsboard.server.common.data.page.PageDataIterable; -import org.thingsboard.server.common.msg.cf.ProfileEntityMsg; +import org.thingsboard.server.common.msg.cf.CalculatedFieldProfileEntityMsg; import org.thingsboard.server.dao.asset.AssetService; import org.thingsboard.server.dao.device.DeviceService; import org.thingsboard.server.queue.util.AfterStartUp; @@ -49,7 +49,7 @@ public class DefaultCalculatedFieldInitService implements CalculatedFieldInitSer for (ProfileEntityIdInfo idInfo : deviceIdInfos) { log.trace("Processing device record: {}", idInfo); try { - actorSystemContext.tell(new ProfileEntityMsg(idInfo.getTenantId(), idInfo.getProfileId(), idInfo.getEntityId())); + actorSystemContext.tell(new CalculatedFieldProfileEntityMsg(idInfo.getTenantId(), idInfo.getProfileId(), idInfo.getEntityId())); } catch (Exception e) { log.error("Failed to process device record: {}", idInfo, e); } @@ -58,7 +58,7 @@ public class DefaultCalculatedFieldInitService implements CalculatedFieldInitSer for (ProfileEntityIdInfo idInfo : assetIdInfos) { log.trace("Processing asset record: {}", idInfo); try { - actorSystemContext.tell(new ProfileEntityMsg(idInfo.getTenantId(), idInfo.getProfileId(), idInfo.getEntityId())); + actorSystemContext.tell(new CalculatedFieldProfileEntityMsg(idInfo.getTenantId(), idInfo.getProfileId(), idInfo.getEntityId())); } catch (Exception e) { log.error("Failed to process asset record: {}", idInfo, e); } diff --git a/common/message/src/main/java/org/thingsboard/server/common/msg/MsgType.java b/common/message/src/main/java/org/thingsboard/server/common/msg/MsgType.java index bfbd5fb0d6..5bada73c8c 100644 --- a/common/message/src/main/java/org/thingsboard/server/common/msg/MsgType.java +++ b/common/message/src/main/java/org/thingsboard/server/common/msg/MsgType.java @@ -136,7 +136,7 @@ public enum MsgType { EDGE_SYNC_RESPONSE_FROM_EDGE_SESSION_MSG, - CF_ACTOR_INIT_MSG, // Sent to init caches for CF actor; + CF_CACHE_INIT_MSG, // Sent to init caches for CF actor; CF_PROFILE_ENTITY_MSG, // Sent to init profile entities cache; CF_INIT_MSG, // Sent to init particular calculated field; CF_LINK_INIT_MSG, // Sent to init particular calculated field; diff --git a/common/message/src/main/java/org/thingsboard/server/common/msg/cf/CalculatedFieldActorInitMsg.java b/common/message/src/main/java/org/thingsboard/server/common/msg/cf/CalculatedFieldCacheInitMsg.java similarity index 90% rename from common/message/src/main/java/org/thingsboard/server/common/msg/cf/CalculatedFieldActorInitMsg.java rename to common/message/src/main/java/org/thingsboard/server/common/msg/cf/CalculatedFieldCacheInitMsg.java index 851c7709bb..bf2054dfcf 100644 --- a/common/message/src/main/java/org/thingsboard/server/common/msg/cf/CalculatedFieldActorInitMsg.java +++ b/common/message/src/main/java/org/thingsboard/server/common/msg/cf/CalculatedFieldCacheInitMsg.java @@ -21,13 +21,13 @@ import org.thingsboard.server.common.msg.MsgType; import org.thingsboard.server.common.msg.ToCalculatedFieldSystemMsg; @Data -public class CalculatedFieldActorInitMsg implements ToCalculatedFieldSystemMsg { +public class CalculatedFieldCacheInitMsg implements ToCalculatedFieldSystemMsg { private final TenantId tenantId; @Override public MsgType getMsgType() { - return MsgType.CF_ACTOR_INIT_MSG; + return MsgType.CF_CACHE_INIT_MSG; } } diff --git a/common/message/src/main/java/org/thingsboard/server/common/msg/cf/ProfileEntityMsg.java b/common/message/src/main/java/org/thingsboard/server/common/msg/cf/CalculatedFieldProfileEntityMsg.java similarity index 93% rename from common/message/src/main/java/org/thingsboard/server/common/msg/cf/ProfileEntityMsg.java rename to common/message/src/main/java/org/thingsboard/server/common/msg/cf/CalculatedFieldProfileEntityMsg.java index 81196c150e..72447af677 100644 --- a/common/message/src/main/java/org/thingsboard/server/common/msg/cf/ProfileEntityMsg.java +++ b/common/message/src/main/java/org/thingsboard/server/common/msg/cf/CalculatedFieldProfileEntityMsg.java @@ -22,7 +22,7 @@ import org.thingsboard.server.common.msg.MsgType; import org.thingsboard.server.common.msg.ToCalculatedFieldSystemMsg; @Data -public class ProfileEntityMsg implements ToCalculatedFieldSystemMsg { +public class CalculatedFieldProfileEntityMsg implements ToCalculatedFieldSystemMsg { private final TenantId tenantId; private final EntityId profileEntityId; From 45cd9af1637167155125789cbe7bfefbc957d8ad Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Fri, 11 Apr 2025 10:02:39 +0300 Subject: [PATCH 33/47] fixed ruel engine consumer --- .../queue/ruleengine/TbRuleEngineQueueConsumerManager.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/queue/ruleengine/TbRuleEngineQueueConsumerManager.java b/application/src/main/java/org/thingsboard/server/service/queue/ruleengine/TbRuleEngineQueueConsumerManager.java index c147836dab..879da5afc6 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/ruleengine/TbRuleEngineQueueConsumerManager.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/ruleengine/TbRuleEngineQueueConsumerManager.java @@ -73,8 +73,8 @@ public class TbRuleEngineQueueConsumerManager extends MainQueueConsumerManager { - Integer partitionId = topicPartitionInfo.getPartition().orElse(-1); + (queueConfig, tpi) -> { + Integer partitionId = tpi != null ? tpi.getPartition().orElse(-1) : null; return ctx.getQueueFactory().createToRuleEngineMsgConsumer(queueConfig, partitionId); }, consumerExecutor, scheduler, taskExecutor, null); From 84778afc0d8ffe4f4ec9524759600516d97ba1f6 Mon Sep 17 00:00:00 2001 From: Artem Dzhereleiko Date: Fri, 11 Apr 2025 10:12:46 +0300 Subject: [PATCH 34/47] UI: Change error position for upload dialog --- .../components/image/upload-image-dialog.component.html | 6 +++--- .../components/image/upload-image-dialog.component.ts | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/ui-ngx/src/app/shared/components/image/upload-image-dialog.component.html b/ui-ngx/src/app/shared/components/image/upload-image-dialog.component.html index 11dffb1b25..47950e53b7 100644 --- a/ui-ngx/src/app/shared/components/image/upload-image-dialog.component.html +++ b/ui-ngx/src/app/shared/components/image/upload-image-dialog.component.html @@ -15,7 +15,7 @@ limitations under the License. --> - +

{{ ( uploadImage ? (isScada ? 'scada.upload-symbol' : 'image.upload-image') : (isScada ? 'scada.update-symbol' : 'image.update-image') ) | translate }}

@@ -28,8 +28,8 @@
-
-
+
+
Date: Fri, 11 Apr 2025 12:02:00 +0300 Subject: [PATCH 35/47] Minor refactoring for EdqsStatsService --- .../server/edqs/stats/DefaultEdqsStatsService.java | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/stats/DefaultEdqsStatsService.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/stats/DefaultEdqsStatsService.java index 4853920c44..435c7f0884 100644 --- a/common/edqs/src/main/java/org/thingsboard/server/edqs/stats/DefaultEdqsStatsService.java +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/stats/DefaultEdqsStatsService.java @@ -53,13 +53,12 @@ public class DefaultEdqsStatsService implements EdqsStatsService { private final ConcurrentMap objectCounters = new ConcurrentHashMap<>(); private final ConcurrentMap timers = new ConcurrentHashMap<>(); private final ConcurrentMap counters = new ConcurrentHashMap<>(); - private final ConcurrentMap gauges = new ConcurrentHashMap<>(); @PostConstruct private void init() { - statsFactory.createGauge("edqsGauges", "stringPoolSize", TbStringPool.getPool(), Map::size); - statsFactory.createGauge("edqsGauges", "bytePoolSize", TbBytePool.getPool(), Map::size); - statsFactory.createGauge("edqsGauges", "tenantReposSize", DefaultEdqsRepository.getRepos(), Map::size); + statsFactory.createGauge("edqsMapGauges", "stringPoolSize", TbStringPool.getPool(), Map::size); + statsFactory.createGauge("edqsMapGauges", "bytePoolSize", TbBytePool.getPool(), Map::size); + statsFactory.createGauge("edqsMapGauges", "tenantReposSize", DefaultEdqsRepository.getRepos(), Map::size); } @Override @@ -124,10 +123,6 @@ public class DefaultEdqsStatsService implements EdqsStatsService { return counters.computeIfAbsent(name, __ -> statsFactory.createStatsCounter("edqsCounters", name)); } - private AtomicInteger getGauge(String name) { - return gauges.computeIfAbsent(name, __ -> statsFactory.createGauge("edqsGauges", name, new AtomicInteger())); - } - private AtomicInteger getObjectGauge(ObjectType objectType) { return objectCounters.computeIfAbsent(objectType, type -> statsFactory.createGauge("edqsGauges", "objectsCount", new AtomicInteger(), "objectType", type.name())); From 54c2af2d6820a6c53f9a28d20600c72b285638e8 Mon Sep 17 00:00:00 2001 From: Artem Dzhereleiko Date: Fri, 11 Apr 2025 12:10:54 +0300 Subject: [PATCH 36/47] UI: Fixed visability of table header in preview mode --- .../widget/action/manage-widget-actions.component.scss | 1 + 1 file changed, 1 insertion(+) diff --git a/ui-ngx/src/app/modules/home/components/widget/action/manage-widget-actions.component.scss b/ui-ngx/src/app/modules/home/components/widget/action/manage-widget-actions.component.scss index 66cbe675b2..e122794bb0 100644 --- a/ui-ngx/src/app/modules/home/components/widget/action/manage-widget-actions.component.scss +++ b/ui-ngx/src/app/modules/home/components/widget/action/manage-widget-actions.component.scss @@ -33,6 +33,7 @@ .table-container { overflow: auto; + z-index: 1; } } } From ad40848a25efa3c3c06b58ef563e4b327a205d66 Mon Sep 17 00:00:00 2001 From: ViacheslavKlimov Date: Fri, 11 Apr 2025 12:29:20 +0300 Subject: [PATCH 37/47] Fix DefaultTbCoreConsumerServiceTest --- .../server/service/queue/DefaultTbCoreConsumerService.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java index e115e4f7b5..fe2d291765 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java @@ -20,6 +20,7 @@ import com.google.common.util.concurrent.ListeningExecutorService; import com.google.common.util.concurrent.MoreExecutors; import jakarta.annotation.PostConstruct; import jakarta.annotation.PreDestroy; +import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; import org.springframework.beans.factory.annotation.Value; import org.springframework.context.ApplicationEventPublisher; @@ -116,6 +117,7 @@ import java.util.function.Function; import java.util.stream.Collectors; @Service +@Slf4j @TbCoreComponent public class DefaultTbCoreConsumerService extends AbstractConsumerService implements TbCoreConsumerService { From 48d9aca0abec1525e69ba500c91673c8efe56f66 Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Fri, 11 Apr 2025 12:29:45 +0300 Subject: [PATCH 38/47] minor fixes --- .../org/thingsboard/server/actors/app/AppActor.java | 2 +- .../calculatedField/CalculatedFieldManagerActor.java | 6 +++--- .../CalculatedFieldManagerMessageProcessor.java | 12 +++++++----- .../server/actors/tenant/TenantActor.java | 3 ++- .../cf/DefaultCalculatedFieldInitService.java | 8 ++++---- .../DefaultTbCalculatedFieldConsumerService.java | 5 ++--- application/src/main/resources/thingsboard.yml | 4 ++++ .../queue/DefaultTbCoreConsumerServiceTest.java | 4 ++++ .../server/dao/cf/CalculatedFieldService.java | 2 -- .../org/thingsboard/server/common/msg/MsgType.java | 2 +- ...java => CalculatedFieldInitProfileEntityMsg.java} | 4 ++-- .../queue/discovery/event/PartitionChangeEvent.java | 1 - .../queue/provider/KafkaMonolithQueueFactory.java | 3 +++ .../provider/KafkaTbRuleEngineQueueFactory.java | 3 +++ .../server/dao/cf/BaseCalculatedFieldService.java | 6 ------ 15 files changed, 36 insertions(+), 29 deletions(-) rename common/message/src/main/java/org/thingsboard/server/common/msg/cf/{CalculatedFieldProfileEntityMsg.java => CalculatedFieldInitProfileEntityMsg.java} (88%) diff --git a/application/src/main/java/org/thingsboard/server/actors/app/AppActor.java b/application/src/main/java/org/thingsboard/server/actors/app/AppActor.java index fc9f344563..a79a182fa1 100644 --- a/application/src/main/java/org/thingsboard/server/actors/app/AppActor.java +++ b/application/src/main/java/org/thingsboard/server/actors/app/AppActor.java @@ -114,7 +114,7 @@ public class AppActor extends ContextAwareActor { ctx.broadcastToChildrenByType(msg, EntityType.TENANT); break; case CF_CACHE_INIT_MSG: - case CF_PROFILE_ENTITY_MSG: + case CF_INIT_PROFILE_ENTITY_MSG: case CF_INIT_MSG: case CF_LINK_INIT_MSG: case CF_STATE_RESTORE_MSG: diff --git a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerActor.java b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerActor.java index d5a3da03e4..9f59a80e67 100644 --- a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerActor.java +++ b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerActor.java @@ -25,9 +25,9 @@ import org.thingsboard.server.common.msg.ToCalculatedFieldSystemMsg; import org.thingsboard.server.common.msg.cf.CalculatedFieldCacheInitMsg; import org.thingsboard.server.common.msg.cf.CalculatedFieldEntityLifecycleMsg; import org.thingsboard.server.common.msg.cf.CalculatedFieldInitMsg; +import org.thingsboard.server.common.msg.cf.CalculatedFieldInitProfileEntityMsg; import org.thingsboard.server.common.msg.cf.CalculatedFieldLinkInitMsg; import org.thingsboard.server.common.msg.cf.CalculatedFieldPartitionChangeMsg; -import org.thingsboard.server.common.msg.cf.CalculatedFieldProfileEntityMsg; /** * Created by ashvayka on 15.03.18. @@ -70,8 +70,8 @@ public class CalculatedFieldManagerActor extends AbstractCalculatedFieldActor { case CF_CACHE_INIT_MSG: processor.onCacheInitMsg((CalculatedFieldCacheInitMsg) msg); break; - case CF_PROFILE_ENTITY_MSG: - processor.onProfileEntityMsg((CalculatedFieldProfileEntityMsg) msg); + case CF_INIT_PROFILE_ENTITY_MSG: + processor.onProfileEntityMsg((CalculatedFieldInitProfileEntityMsg) msg); break; case CF_INIT_MSG: processor.onFieldInitMsg((CalculatedFieldInitMsg) msg); diff --git a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java index e62c531b16..a896bbd74b 100644 --- a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java +++ b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java @@ -38,9 +38,9 @@ import org.thingsboard.server.common.data.page.PageDataIterable; import org.thingsboard.server.common.msg.cf.CalculatedFieldCacheInitMsg; import org.thingsboard.server.common.msg.cf.CalculatedFieldEntityLifecycleMsg; import org.thingsboard.server.common.msg.cf.CalculatedFieldInitMsg; +import org.thingsboard.server.common.msg.cf.CalculatedFieldInitProfileEntityMsg; import org.thingsboard.server.common.msg.cf.CalculatedFieldLinkInitMsg; import org.thingsboard.server.common.msg.cf.CalculatedFieldPartitionChangeMsg; -import org.thingsboard.server.common.msg.cf.CalculatedFieldProfileEntityMsg; import org.thingsboard.server.common.msg.plugin.ComponentLifecycleMsg; import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.common.msg.queue.TbCallback; @@ -86,7 +86,7 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware protected TbActorCtx ctx; - @Value("${calculated_fields.init_fetch_pack_size:50000}") + @Value("${queue.calculated_fields.init_tenant_fetch_pack_size:1000}") @Getter private int initFetchPackSize; @@ -123,7 +123,7 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware msg.getCallback().onSuccess(); } - public void onProfileEntityMsg(CalculatedFieldProfileEntityMsg msg) { + public void onProfileEntityMsg(CalculatedFieldInitProfileEntityMsg msg) { log.debug("[{}] Processing profile entity message.", msg.getTenantId().getId()); entityProfileCache.add(msg.getProfileEntityId(), msg.getEntityId()); msg.getCallback().onSuccess(); @@ -538,7 +538,8 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware return switch (entityId.getEntityType()) { case ASSET -> assetProfileCache.get(tenantId, (AssetId) entityId).getId(); case DEVICE -> deviceProfileCache.get(tenantId, (DeviceId) entityId).getId(); - default -> null; + default -> + throw new IllegalArgumentException("'" + entityId.getEntityType() + "' is not profile entity." + entityId); }; } @@ -567,10 +568,11 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware public void initCalculatedFields() { PageDataIterable cfs = new PageDataIterable<>(pageLink -> cfDaoService.findCalculatedFieldsByTenantId(tenantId, pageLink), initFetchPackSize); cfs.forEach(cf -> { + log.trace("Processing calculated field record: {}", cf); try { onFieldInitMsg(new CalculatedFieldInitMsg(cf.getTenantId(), cf)); } catch (CalculatedFieldException e) { - throw new RuntimeException(e); + log.error("Failed to process calculated field record: {}", cf, e); } }); calculatedFields.values().forEach(cf -> { diff --git a/application/src/main/java/org/thingsboard/server/actors/tenant/TenantActor.java b/application/src/main/java/org/thingsboard/server/actors/tenant/TenantActor.java index a5be701d38..2a039327f2 100644 --- a/application/src/main/java/org/thingsboard/server/actors/tenant/TenantActor.java +++ b/application/src/main/java/org/thingsboard/server/actors/tenant/TenantActor.java @@ -178,7 +178,7 @@ public class TenantActor extends RuleChainManagerActor { onRuleChainMsg((RuleChainAwareMsg) msg); break; case CF_CACHE_INIT_MSG: - case CF_PROFILE_ENTITY_MSG: + case CF_INIT_PROFILE_ENTITY_MSG: case CF_INIT_MSG: case CF_LINK_INIT_MSG: case CF_STATE_RESTORE_MSG: @@ -202,6 +202,7 @@ public class TenantActor extends RuleChainManagerActor { } else { log.debug("[{}] CF Actor is not initialized. ToCalculatedFieldSystemMsg: [{}]", tenantId, msg); } + msg.getCallback().onSuccess(); return; } if (priority) { diff --git a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldInitService.java b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldInitService.java index 826a5243cf..79950f2e3f 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldInitService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldInitService.java @@ -23,7 +23,7 @@ import org.springframework.stereotype.Service; import org.thingsboard.server.actors.ActorSystemContext; import org.thingsboard.server.common.data.ProfileEntityIdInfo; import org.thingsboard.server.common.data.page.PageDataIterable; -import org.thingsboard.server.common.msg.cf.CalculatedFieldProfileEntityMsg; +import org.thingsboard.server.common.msg.cf.CalculatedFieldInitProfileEntityMsg; import org.thingsboard.server.dao.asset.AssetService; import org.thingsboard.server.dao.device.DeviceService; import org.thingsboard.server.queue.util.AfterStartUp; @@ -39,7 +39,7 @@ public class DefaultCalculatedFieldInitService implements CalculatedFieldInitSer private final DeviceService deviceService; private final ActorSystemContext actorSystemContext; - @Value("${calculated_fields.init_fetch_pack_size:50000}") + @Value("${queue.calculated_fields.init_fetch_pack_size:50000}") @Getter private int initFetchPackSize; @@ -49,7 +49,7 @@ public class DefaultCalculatedFieldInitService implements CalculatedFieldInitSer for (ProfileEntityIdInfo idInfo : deviceIdInfos) { log.trace("Processing device record: {}", idInfo); try { - actorSystemContext.tell(new CalculatedFieldProfileEntityMsg(idInfo.getTenantId(), idInfo.getProfileId(), idInfo.getEntityId())); + actorSystemContext.tell(new CalculatedFieldInitProfileEntityMsg(idInfo.getTenantId(), idInfo.getProfileId(), idInfo.getEntityId())); } catch (Exception e) { log.error("Failed to process device record: {}", idInfo, e); } @@ -58,7 +58,7 @@ public class DefaultCalculatedFieldInitService implements CalculatedFieldInitSer for (ProfileEntityIdInfo idInfo : assetIdInfos) { log.trace("Processing asset record: {}", idInfo); try { - actorSystemContext.tell(new CalculatedFieldProfileEntityMsg(idInfo.getTenantId(), idInfo.getProfileId(), idInfo.getEntityId())); + actorSystemContext.tell(new CalculatedFieldInitProfileEntityMsg(idInfo.getTenantId(), idInfo.getProfileId(), idInfo.getEntityId())); } catch (Exception e) { log.error("Failed to process asset record: {}", idInfo, e); } diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCalculatedFieldConsumerService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCalculatedFieldConsumerService.java index ea2b87e5e4..41bf76ff0e 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCalculatedFieldConsumerService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCalculatedFieldConsumerService.java @@ -131,6 +131,7 @@ public class DefaultTbCalculatedFieldConsumerService extends AbstractPartitionBa stateService.restore(queueKey, partitions); } }); + // eventConsumer's partitions will be updated by stateService // Cleanup old entities after corresponding consumers are stopped. // Any periodic tasks need to check that the entity is still managed by the current server before processing. @@ -174,9 +175,7 @@ public class DefaultTbCalculatedFieldConsumerService extends AbstractPartitionBa packSubmitFuture.cancel(true); log.info("Timeout to process message: {}", pendingMsgHolder.getMsg()); } - if (log.isDebugEnabled()) { - ctx.getAckMap().forEach((id, msg) -> log.debug("[{}] Timeout to process message: {}", id, msg.getValue())); - } + ctx.getAckMap().forEach((id, msg) -> log.debug("[{}] Timeout to process message: {}", id, msg.getValue())); ctx.getFailedMap().forEach((id, msg) -> log.warn("[{}] Failed to process message: {}", id, msg.getValue())); } consumer.commit(); diff --git a/application/src/main/resources/thingsboard.yml b/application/src/main/resources/thingsboard.yml index 716797c2d2..14f0437402 100644 --- a/application/src/main/resources/thingsboard.yml +++ b/application/src/main/resources/thingsboard.yml @@ -1848,6 +1848,10 @@ queue: pool_size: "${TB_QUEUE_CF_POOL_SIZE:8}" # RocksDB path for storing CF states rocks_db_path: "${TB_QUEUE_CF_ROCKS_DB_PATH:${user.home}/.rocksdb/cf_states}" + # The fetch size specifies how many rows will be returned + init_fetch_pack_size: "${TB_QUEUE_CF_FETCH_PACK_SIZE:50000}" + # The fetch size specifies how many rows will be returned + init_tenant_fetch_pack_size: "${TB_QUEUE_CF_TENANT_FETCH_PACK_SIZE:1000}" transport: # For high-priority notifications that require minimum latency and processing time notifications_topic: "${TB_QUEUE_TRANSPORT_NOTIFICATIONS_TOPIC:tb_transport.notifications}" diff --git a/application/src/test/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerServiceTest.java b/application/src/test/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerServiceTest.java index 26832f6176..4a5ecef970 100644 --- a/application/src/test/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerServiceTest.java +++ b/application/src/test/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerServiceTest.java @@ -24,6 +24,7 @@ import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.ArgumentCaptor; import org.mockito.Mock; import org.mockito.junit.jupiter.MockitoExtension; +import org.slf4j.Logger; import org.springframework.test.util.ReflectionTestUtils; import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.TenantId; @@ -51,6 +52,8 @@ public class DefaultTbCoreConsumerServiceTest { private TbCoreConsumerStats statsMock; @Mock private RuleEngineCallService ruleEngineCallServiceMock; + @Mock + private Logger logMock; @Mock private TbCallback tbCallbackMock; @@ -69,6 +72,7 @@ public class DefaultTbCoreConsumerServiceTest { executor = MoreExecutors.newDirectExecutorService(); ReflectionTestUtils.setField(defaultTbCoreConsumerServiceMock, "stateService", stateServiceMock); ReflectionTestUtils.setField(defaultTbCoreConsumerServiceMock, "deviceActivityEventsExecutor", executor); + ReflectionTestUtils.setField(defaultTbCoreConsumerServiceMock, "log", logMock); } @AfterEach diff --git a/common/dao-api/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldService.java b/common/dao-api/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldService.java index b6f43cda67..5101d6d57e 100644 --- a/common/dao-api/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldService.java +++ b/common/dao-api/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldService.java @@ -55,8 +55,6 @@ public interface CalculatedFieldService extends EntityDaoService { List findAllCalculatedFieldLinksByEntityId(TenantId tenantId, EntityId entityId); - List findAllCalculatedFieldLinksByTenantId(TenantId tenantId); - PageData findAllCalculatedFieldLinksByTenantId(TenantId tenantId, PageLink pageLink); PageData findAllCalculatedFieldLinks(PageLink pageLink); diff --git a/common/message/src/main/java/org/thingsboard/server/common/msg/MsgType.java b/common/message/src/main/java/org/thingsboard/server/common/msg/MsgType.java index 5bada73c8c..f1c404ce16 100644 --- a/common/message/src/main/java/org/thingsboard/server/common/msg/MsgType.java +++ b/common/message/src/main/java/org/thingsboard/server/common/msg/MsgType.java @@ -137,7 +137,7 @@ public enum MsgType { CF_CACHE_INIT_MSG, // Sent to init caches for CF actor; - CF_PROFILE_ENTITY_MSG, // Sent to init profile entities cache; + CF_INIT_PROFILE_ENTITY_MSG, // Sent to init profile entities cache; CF_INIT_MSG, // Sent to init particular calculated field; CF_LINK_INIT_MSG, // Sent to init particular calculated field; CF_STATE_RESTORE_MSG, // Sent to restore particular calculated field entity state; diff --git a/common/message/src/main/java/org/thingsboard/server/common/msg/cf/CalculatedFieldProfileEntityMsg.java b/common/message/src/main/java/org/thingsboard/server/common/msg/cf/CalculatedFieldInitProfileEntityMsg.java similarity index 88% rename from common/message/src/main/java/org/thingsboard/server/common/msg/cf/CalculatedFieldProfileEntityMsg.java rename to common/message/src/main/java/org/thingsboard/server/common/msg/cf/CalculatedFieldInitProfileEntityMsg.java index 72447af677..66cd2ac441 100644 --- a/common/message/src/main/java/org/thingsboard/server/common/msg/cf/CalculatedFieldProfileEntityMsg.java +++ b/common/message/src/main/java/org/thingsboard/server/common/msg/cf/CalculatedFieldInitProfileEntityMsg.java @@ -22,7 +22,7 @@ import org.thingsboard.server.common.msg.MsgType; import org.thingsboard.server.common.msg.ToCalculatedFieldSystemMsg; @Data -public class CalculatedFieldProfileEntityMsg implements ToCalculatedFieldSystemMsg { +public class CalculatedFieldInitProfileEntityMsg implements ToCalculatedFieldSystemMsg { private final TenantId tenantId; private final EntityId profileEntityId; @@ -30,7 +30,7 @@ public class CalculatedFieldProfileEntityMsg implements ToCalculatedFieldSystemM @Override public MsgType getMsgType() { - return MsgType.CF_PROFILE_ENTITY_MSG; + return MsgType.CF_INIT_PROFILE_ENTITY_MSG; } } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/event/PartitionChangeEvent.java b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/event/PartitionChangeEvent.java index 7c2d0b0240..32537edba5 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/event/PartitionChangeEvent.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/event/PartitionChangeEvent.java @@ -18,7 +18,6 @@ package org.thingsboard.server.queue.discovery.event; import lombok.Getter; import lombok.ToString; import org.thingsboard.server.common.data.DataConstants; -import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; import org.thingsboard.server.queue.discovery.QueueKey; diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaMonolithQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaMonolithQueueFactory.java index 87a9d6a697..0738f660df 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaMonolithQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaMonolithQueueFactory.java @@ -517,6 +517,9 @@ public class KafkaMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEngi @Override public TbQueueConsumer> createToCalculatedFieldMsgConsumer(TopicPartitionInfo tpi) { String queueName = DataConstants.CF_QUEUE_NAME; + if (tpi == null) { + throw new IllegalArgumentException("TopicPartitionInfo is required."); + } TenantId tenantId = tpi.getTenantId().orElse(TenantId.SYS_TENANT_ID); Integer partitionId = tpi.getPartition().orElseThrow(() -> new IllegalArgumentException("PartitionId is required.")); String groupId = topicService.buildConsumerGroupId("cf-", tenantId, queueName, partitionId); diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbRuleEngineQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbRuleEngineQueueFactory.java index fdf2b2f3f7..b4884ae72c 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbRuleEngineQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbRuleEngineQueueFactory.java @@ -318,6 +318,9 @@ public class KafkaTbRuleEngineQueueFactory implements TbRuleEngineQueueFactory { @Override public TbQueueConsumer> createToCalculatedFieldMsgConsumer(TopicPartitionInfo tpi) { String queueName = DataConstants.CF_QUEUE_NAME; + if (tpi == null) { + throw new IllegalArgumentException("TopicPartitionInfo is required."); + } TenantId tenantId = tpi.getTenantId().orElse(TenantId.SYS_TENANT_ID); Integer partitionId = tpi.getPartition().orElseThrow(() -> new IllegalArgumentException("PartitionId is required.")); String groupId = topicService.buildConsumerGroupId("cf-", tenantId, queueName, partitionId); diff --git a/dao/src/main/java/org/thingsboard/server/dao/cf/BaseCalculatedFieldService.java b/dao/src/main/java/org/thingsboard/server/dao/cf/BaseCalculatedFieldService.java index d3fea73849..0c5df18e80 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/cf/BaseCalculatedFieldService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/cf/BaseCalculatedFieldService.java @@ -182,12 +182,6 @@ public class BaseCalculatedFieldService extends AbstractEntityService implements return calculatedFieldLinkDao.findCalculatedFieldLinksByEntityId(tenantId, entityId); } - @Override - public List findAllCalculatedFieldLinksByTenantId(TenantId tenantId) { - log.trace("Executing findAllCalculatedFieldLinksByTenantId, tenantId [{}]", tenantId); - return calculatedFieldLinkDao.findCalculatedFieldLinksByTenantId(tenantId); - } - @Override public PageData findAllCalculatedFieldLinksByTenantId(TenantId tenantId, PageLink pageLink) { log.trace("Executing findAllCalculatedFieldLinksByTenantId, tenantId[{}] pageLink [{}]", tenantId, pageLink); From 6e7ea4a35280004a573debb7d8890df867a19691 Mon Sep 17 00:00:00 2001 From: Vladyslav_Prykhodko Date: Fri, 11 Apr 2025 12:41:02 +0300 Subject: [PATCH 39/47] UI: Remove echart from module map --- ui-ngx/src/app/modules/common/modules-map.ts | 4 ---- 1 file changed, 4 deletions(-) diff --git a/ui-ngx/src/app/modules/common/modules-map.ts b/ui-ngx/src/app/modules/common/modules-map.ts index 530a41f006..70432e8cdf 100644 --- a/ui-ngx/src/app/modules/common/modules-map.ts +++ b/ui-ngx/src/app/modules/common/modules-map.ts @@ -76,8 +76,6 @@ import * as TranslateCore from '@ngx-translate/core'; import * as MatDateTimePicker from '@mat-datetimepicker/core'; import _moment from 'moment'; import * as tslib from 'tslib'; -import * as Echarts from 'echarts'; -import * as EchartsCore from 'echarts/core'; import * as TbCore from '@core/public-api'; import * as TbShared from '@shared/public-api'; @@ -407,8 +405,6 @@ class ModulesMap implements IModulesMap { '@mat-datetimepicker/core': MatDateTimePicker, moment: _moment, tslib, - 'echarts': Echarts, - 'echarts/core': EchartsCore, '@core/public-api': TbCore, '@shared/public-api': TbShared, From cc5d9dbb2c53f997554307cce45fcd7e36ae848e Mon Sep 17 00:00:00 2001 From: Volodymyr Babak Date: Fri, 11 Apr 2025 12:44:09 +0300 Subject: [PATCH 40/47] Added sync of fat edge consumer group into per edge group. Fixed clean up edge topics --- .../edge/rpc/KafkaEdgeGrpcSession.java | 1 + .../ttl/KafkaEdgeTopicsCleanUpService.java | 13 ++++--- .../server/queue/TbEdgeQueueAdmin.java | 22 ++++++++++++ .../server/queue/kafka/TbKafkaAdmin.java | 36 +++++++++++++++---- .../provider/KafkaMonolithQueueFactory.java | 6 +++- .../provider/KafkaTbCoreQueueFactory.java | 6 +++- 6 files changed, 71 insertions(+), 13 deletions(-) create mode 100644 common/cluster-api/src/main/java/org/thingsboard/server/queue/TbEdgeQueueAdmin.java diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/KafkaEdgeGrpcSession.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/KafkaEdgeGrpcSession.java index 076e8eb8f6..37687f14a9 100644 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/KafkaEdgeGrpcSession.java +++ b/application/src/main/java/org/thingsboard/server/service/edge/rpc/KafkaEdgeGrpcSession.java @@ -141,6 +141,7 @@ public class KafkaEdgeGrpcSession extends EdgeGrpcSession { String topic = topicService.buildEdgeEventNotificationsTopicPartitionInfo(tenantId, edge.getId()).getTopic(); TbKafkaAdmin kafkaAdmin = new TbKafkaAdmin(kafkaSettings, kafkaTopicConfigs.getEdgeEventConfigs()); kafkaAdmin.deleteTopic(topic); + kafkaAdmin.deleteConsumerGroup(topic); } } diff --git a/application/src/main/java/org/thingsboard/server/service/ttl/KafkaEdgeTopicsCleanUpService.java b/application/src/main/java/org/thingsboard/server/service/ttl/KafkaEdgeTopicsCleanUpService.java index 73712542fd..6d06c85585 100644 --- a/application/src/main/java/org/thingsboard/server/service/ttl/KafkaEdgeTopicsCleanUpService.java +++ b/application/src/main/java/org/thingsboard/server/service/ttl/KafkaEdgeTopicsCleanUpService.java @@ -113,7 +113,7 @@ public class KafkaEdgeTopicsCleanUpService extends AbstractCleanUpService { .ifPresentOrElse(lastConnectTime -> { String topic = topicService.buildEdgeEventNotificationsTopicPartitionInfo(tenantId, edgeId).getTopic(); if (kafkaAdmin.isTopicEmpty(topic)) { - kafkaAdmin.deleteTopic(topic); + deleteTopicAndConsumerGroup(topic); log.info("[{}] Removed outdated topic {} for edge {} older than {}", tenantId, topic, edgeId, Date.from(Instant.ofEpochMilli(currentTimeMillis - ttlMillis))); } @@ -121,7 +121,7 @@ public class KafkaEdgeTopicsCleanUpService extends AbstractCleanUpService { Edge edge = edgeService.findEdgeById(tenantId, edgeId); if (edge == null) { String topic = topicService.buildEdgeEventNotificationsTopicPartitionInfo(tenantId, edgeId).getTopic(); - kafkaAdmin.deleteTopic(topic); + deleteTopicAndConsumerGroup(topic); log.info("[{}] Removed topic {} for deleted edge {}", tenantId, topic, edgeId); } }); @@ -132,12 +132,17 @@ public class KafkaEdgeTopicsCleanUpService extends AbstractCleanUpService { } else { for (EdgeId edgeId : edgeIds) { String topic = topicService.buildEdgeEventNotificationsTopicPartitionInfo(tenantId, edgeId).getTopic(); - kafkaAdmin.deleteTopic(topic); + deleteTopicAndConsumerGroup(topic); } log.info("[{}] Removed topics for not existing tenant and edges {}", tenantId, edgeIds); } } + private void deleteTopicAndConsumerGroup(String topic) { + kafkaAdmin.deleteTopic(topic); + kafkaAdmin.deleteConsumerGroup(topic); + } + private boolean isTopicExpired(long lastConnectTime, long ttlMillis, long currentTimeMillis) { return lastConnectTime + ttlMillis < currentTimeMillis; } @@ -146,7 +151,7 @@ public class KafkaEdgeTopicsCleanUpService extends AbstractCleanUpService { Map> tenantEdgeMap = new HashMap<>(); for (String topic : topics) { try { - String remaining = topic.substring(prefix.length()); + String remaining = topic.substring(prefix.length() + 1); String[] parts = remaining.split("\\."); TenantId tenantId = TenantId.fromUUID(UUID.fromString(parts[0])); EdgeId edgeId = new EdgeId(UUID.fromString(parts[1])); diff --git a/common/cluster-api/src/main/java/org/thingsboard/server/queue/TbEdgeQueueAdmin.java b/common/cluster-api/src/main/java/org/thingsboard/server/queue/TbEdgeQueueAdmin.java new file mode 100644 index 0000000000..9be50bb145 --- /dev/null +++ b/common/cluster-api/src/main/java/org/thingsboard/server/queue/TbEdgeQueueAdmin.java @@ -0,0 +1,22 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue; + +public interface TbEdgeQueueAdmin extends TbQueueAdmin { + void syncEdgeNotificationsOffsets(String fatGroupId, String newGroupId); + + void deleteConsumerGroup(String consumerGroupId); +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaAdmin.java b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaAdmin.java index 37f881b49c..1e0064a5c8 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaAdmin.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaAdmin.java @@ -26,6 +26,7 @@ import org.apache.kafka.clients.admin.TopicDescription; import org.apache.kafka.clients.consumer.OffsetAndMetadata; import org.apache.kafka.common.TopicPartition; import org.apache.kafka.common.errors.TopicExistsException; +import org.thingsboard.server.queue.TbEdgeQueueAdmin; import org.thingsboard.server.queue.TbQueueAdmin; import org.thingsboard.server.queue.util.PropertyUtils; @@ -43,7 +44,7 @@ import java.util.stream.Collectors; * Created by ashvayka on 24.09.18. */ @Slf4j -public class TbKafkaAdmin implements TbQueueAdmin { +public class TbKafkaAdmin implements TbQueueAdmin, TbEdgeQueueAdmin { private final TbKafkaSettings settings; private final Map topicConfigs; @@ -149,17 +150,38 @@ public class TbKafkaAdmin implements TbQueueAdmin { * */ public void syncOffsets(String fatGroupId, String newGroupId, Integer partitionId) { try { - syncOffsetsUnsafe(fatGroupId, newGroupId, partitionId); + log.info("syncOffsets [{}][{}][{}]", fatGroupId, newGroupId, partitionId); + if (partitionId == null) { + return; + } + syncOffsetsUnsafe(fatGroupId, newGroupId, "." + partitionId); } catch (Exception e) { log.warn("Failed to syncOffsets from {} to {} partitionId {}", fatGroupId, newGroupId, partitionId, e); } } - void syncOffsetsUnsafe(String fatGroupId, String newGroupId, Integer partitionId) throws ExecutionException, InterruptedException, TimeoutException { - log.info("syncOffsets [{}][{}][{}]", fatGroupId, newGroupId, partitionId); - if (partitionId == null) { - return; + /** + * Sync edge notifications offsets from a fat group to a single group per edge + * */ + public void syncEdgeNotificationsOffsets(String fatGroupId, String newGroupId) { + try { + log.info("syncEdgeNotificationsOffsets [{}][{}]", fatGroupId, newGroupId); + syncOffsetsUnsafe(fatGroupId, newGroupId, newGroupId); + } catch (Exception e) { + log.warn("Failed to syncEdgeNotificationsOffsets from {} to {}", fatGroupId, newGroupId, e); + } + } + + @Override + public void deleteConsumerGroup(String consumerGroupId) { + try { + settings.getAdminClient().deleteConsumerGroups(Collections.singletonList(consumerGroupId)); + } catch (Exception e) { + log.warn("Failed to delete consumer group {}", consumerGroupId, e); } + } + + void syncOffsetsUnsafe(String fatGroupId, String newGroupId, String topicSuffix) throws ExecutionException, InterruptedException, TimeoutException { Map oldOffsets = getConsumerGroupOffsets(fatGroupId); if (oldOffsets.isEmpty()) { return; @@ -167,7 +189,7 @@ public class TbKafkaAdmin implements TbQueueAdmin { for (var consumerOffset : oldOffsets.entrySet()) { var tp = consumerOffset.getKey(); - if (!tp.topic().endsWith("." + partitionId)) { + if (!tp.topic().endsWith(topicSuffix)) { continue; } var om = consumerOffset.getValue(); diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaMonolithQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaMonolithQueueFactory.java index 508cdff2a9..b47fcf5e6c 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaMonolithQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaMonolithQueueFactory.java @@ -44,6 +44,7 @@ import org.thingsboard.server.gen.transport.TransportProtos.ToUsageStatsServiceM import org.thingsboard.server.gen.transport.TransportProtos.ToVersionControlServiceMsg; import org.thingsboard.server.gen.transport.TransportProtos.TransportApiRequestMsg; import org.thingsboard.server.gen.transport.TransportProtos.TransportApiResponseMsg; +import org.thingsboard.server.queue.TbEdgeQueueAdmin; import org.thingsboard.server.queue.TbQueueAdmin; import org.thingsboard.server.queue.TbQueueConsumer; import org.thingsboard.server.queue.TbQueueProducer; @@ -101,7 +102,7 @@ public class KafkaMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEngi private final TbQueueAdmin vcAdmin; private final TbQueueAdmin housekeeperAdmin; private final TbQueueAdmin housekeeperReprocessingAdmin; - private final TbQueueAdmin edgeAdmin; + private final TbEdgeQueueAdmin edgeAdmin; private final TbQueueAdmin edgeEventAdmin; private final TbQueueAdmin cfAdmin; private final TbQueueAdmin cfStateAdmin; @@ -494,6 +495,9 @@ public class KafkaMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEngi TbKafkaConsumerTemplate.TbKafkaConsumerTemplateBuilder> consumerBuilder = TbKafkaConsumerTemplate.builder(); consumerBuilder.settings(kafkaSettings); String topic = topicService.buildEdgeEventNotificationsTopicPartitionInfo(tenantId, edgeId).getTopic(); + + edgeAdmin.syncEdgeNotificationsOffsets(topicService.buildTopicName("monolith-edge-event-consumer"), topic); + consumerBuilder.topic(topic); consumerBuilder.clientId("monolith-to-edge-event-consumer-" + serviceInfoProvider.getServiceId() + "-" + edgeConsumerCount.incrementAndGet()); consumerBuilder.groupId(topic); diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbCoreQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbCoreQueueFactory.java index 14766ecbdb..ea7c56f0aa 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbCoreQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/KafkaTbCoreQueueFactory.java @@ -42,6 +42,7 @@ import org.thingsboard.server.gen.transport.TransportProtos.ToUsageStatsServiceM import org.thingsboard.server.gen.transport.TransportProtos.ToVersionControlServiceMsg; import org.thingsboard.server.gen.transport.TransportProtos.TransportApiRequestMsg; import org.thingsboard.server.gen.transport.TransportProtos.TransportApiResponseMsg; +import org.thingsboard.server.queue.TbEdgeQueueAdmin; import org.thingsboard.server.queue.TbQueueAdmin; import org.thingsboard.server.queue.TbQueueConsumer; import org.thingsboard.server.queue.TbQueueProducer; @@ -99,7 +100,7 @@ public class KafkaTbCoreQueueFactory implements TbCoreQueueFactory { private final TbQueueAdmin vcAdmin; private final TbQueueAdmin housekeeperAdmin; private final TbQueueAdmin housekeeperReprocessingAdmin; - private final TbQueueAdmin edgeAdmin; + private final TbEdgeQueueAdmin edgeAdmin; private final TbQueueAdmin edgeEventAdmin; private final TbQueueAdmin cfAdmin; private final TbQueueAdmin edqsEventsAdmin; @@ -440,6 +441,9 @@ public class KafkaTbCoreQueueFactory implements TbCoreQueueFactory { TbKafkaConsumerTemplate.TbKafkaConsumerTemplateBuilder> consumerBuilder = TbKafkaConsumerTemplate.builder(); consumerBuilder.settings(kafkaSettings); String topic = topicService.buildEdgeEventNotificationsTopicPartitionInfo(tenantId, edgeId).getTopic(); + + edgeAdmin.syncEdgeNotificationsOffsets(topicService.buildTopicName("tb-core-edge-event-consumer"), topic); + consumerBuilder.topic(topic); consumerBuilder.clientId("tb-core-edge-event-consumer-" + serviceInfoProvider.getServiceId() + "-" + edgeConsumerCount.incrementAndGet()); consumerBuilder.groupId(topic); From 870dc1b4c46bee7314ed224d6ee5bfb17fe6cf79 Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Fri, 11 Apr 2025 12:44:37 +0300 Subject: [PATCH 41/47] fixed sql query --- .../server/dao/sql/device/DefaultNativeAssetRepository.java | 2 +- .../server/dao/sql/device/DefaultNativeDeviceRepository.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/device/DefaultNativeAssetRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/device/DefaultNativeAssetRepository.java index 6c2a8dc506..ec47da3499 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/device/DefaultNativeAssetRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/device/DefaultNativeAssetRepository.java @@ -51,7 +51,7 @@ public class DefaultNativeAssetRepository extends AbstractNativeRepository imple @Override public PageData findProfileEntityIdInfosByTenantId(UUID tenantId, Pageable pageable) { - String PROFILE_ASSET_ID_INFO_QUERY = String.format("SELECT tenant_id as tenantId, asset_profile_id as profileId, id as id FROM asset WHERE tenant_id = %s ORDER BY created_time ASC LIMIT %%s OFFSET %%s", tenantId); + String PROFILE_ASSET_ID_INFO_QUERY = String.format("SELECT tenant_id as tenantId, asset_profile_id as profileId, id as id FROM asset WHERE tenant_id = '%s' ORDER BY created_time ASC LIMIT %%s OFFSET %%s", tenantId); return find(COUNT_QUERY, PROFILE_ASSET_ID_INFO_QUERY, pageable, row -> { AssetId id = new AssetId((UUID) row.get("id")); AssetProfileId profileId = new AssetProfileId((UUID) row.get("profileId")); diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/device/DefaultNativeDeviceRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/device/DefaultNativeDeviceRepository.java index 1648bb2255..78ee2795b0 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/device/DefaultNativeDeviceRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/device/DefaultNativeDeviceRepository.java @@ -63,7 +63,7 @@ public class DefaultNativeDeviceRepository extends AbstractNativeRepository impl @Override public PageData findProfileEntityIdInfosByTenantId(UUID tenantId, Pageable pageable) { - String PROFILE_DEVICE_ID_INFO_QUERY = String.format("SELECT tenant_id as tenantId, device_profile_id as profileId, id as id FROM device WHERE tenant_id = %s ORDER BY created_time ASC LIMIT %%s OFFSET %%s", tenantId); + String PROFILE_DEVICE_ID_INFO_QUERY = String.format("SELECT tenant_id as tenantId, device_profile_id as profileId, id as id FROM device WHERE tenant_id = '%s' ORDER BY created_time ASC LIMIT %%s OFFSET %%s", tenantId); return find(COUNT_QUERY, PROFILE_DEVICE_ID_INFO_QUERY, pageable, row -> { DeviceId id = new DeviceId((UUID) row.get("id")); DeviceProfileId profileId = new DeviceProfileId((UUID) row.get("profileId")); From 1ad2acc27c5e4d0a67ef69bb32b7271e72d78eec Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Fri, 11 Apr 2025 13:43:17 +0300 Subject: [PATCH 42/47] changed log level and wrote description for yml variables --- .../server/service/cf/DefaultCalculatedFieldCache.java | 2 +- .../queue/DefaultTbCalculatedFieldConsumerService.java | 2 +- application/src/main/resources/thingsboard.yml | 4 ++-- .../service/queue/DefaultTbCoreConsumerServiceTest.java | 4 ---- 4 files changed, 4 insertions(+), 8 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldCache.java b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldCache.java index 219a261183..ed35d96cb7 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldCache.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldCache.java @@ -64,7 +64,7 @@ public class DefaultCalculatedFieldCache implements CalculatedFieldCache { private final ConcurrentMap> entityIdCalculatedFieldLinks = new ConcurrentHashMap<>(); private final ConcurrentMap calculatedFieldsCtx = new ConcurrentHashMap<>(); - @Value("${calculatedField.initFetchPackSize:50000}") + @Value("${queue.calculated_fields.init_fetch_pack_size:50000}") @Getter private int initFetchPackSize; diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCalculatedFieldConsumerService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCalculatedFieldConsumerService.java index 41bf76ff0e..f8c398b016 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCalculatedFieldConsumerService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCalculatedFieldConsumerService.java @@ -175,7 +175,7 @@ public class DefaultTbCalculatedFieldConsumerService extends AbstractPartitionBa packSubmitFuture.cancel(true); log.info("Timeout to process message: {}", pendingMsgHolder.getMsg()); } - ctx.getAckMap().forEach((id, msg) -> log.debug("[{}] Timeout to process message: {}", id, msg.getValue())); + ctx.getAckMap().forEach((id, msg) -> log.warn("[{}] Timeout to process message: {}", id, msg.getValue())); ctx.getFailedMap().forEach((id, msg) -> log.warn("[{}] Failed to process message: {}", id, msg.getValue())); } consumer.commit(); diff --git a/application/src/main/resources/thingsboard.yml b/application/src/main/resources/thingsboard.yml index 14f0437402..76f148c099 100644 --- a/application/src/main/resources/thingsboard.yml +++ b/application/src/main/resources/thingsboard.yml @@ -1848,9 +1848,9 @@ queue: pool_size: "${TB_QUEUE_CF_POOL_SIZE:8}" # RocksDB path for storing CF states rocks_db_path: "${TB_QUEUE_CF_ROCKS_DB_PATH:${user.home}/.rocksdb/cf_states}" - # The fetch size specifies how many rows will be returned + # The fetch size specifies how many rows will be fetched from the database per request for initial fetching init_fetch_pack_size: "${TB_QUEUE_CF_FETCH_PACK_SIZE:50000}" - # The fetch size specifies how many rows will be returned + # The fetch size specifies how many rows will be fetched from the database per request for per-tenant fetching init_tenant_fetch_pack_size: "${TB_QUEUE_CF_TENANT_FETCH_PACK_SIZE:1000}" transport: # For high-priority notifications that require minimum latency and processing time diff --git a/application/src/test/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerServiceTest.java b/application/src/test/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerServiceTest.java index 4a5ecef970..26832f6176 100644 --- a/application/src/test/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerServiceTest.java +++ b/application/src/test/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerServiceTest.java @@ -24,7 +24,6 @@ import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.ArgumentCaptor; import org.mockito.Mock; import org.mockito.junit.jupiter.MockitoExtension; -import org.slf4j.Logger; import org.springframework.test.util.ReflectionTestUtils; import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.TenantId; @@ -52,8 +51,6 @@ public class DefaultTbCoreConsumerServiceTest { private TbCoreConsumerStats statsMock; @Mock private RuleEngineCallService ruleEngineCallServiceMock; - @Mock - private Logger logMock; @Mock private TbCallback tbCallbackMock; @@ -72,7 +69,6 @@ public class DefaultTbCoreConsumerServiceTest { executor = MoreExecutors.newDirectExecutorService(); ReflectionTestUtils.setField(defaultTbCoreConsumerServiceMock, "stateService", stateServiceMock); ReflectionTestUtils.setField(defaultTbCoreConsumerServiceMock, "deviceActivityEventsExecutor", executor); - ReflectionTestUtils.setField(defaultTbCoreConsumerServiceMock, "log", logMock); } @AfterEach From 5f99c0bae2672351dbc357819a1f91ff884a8392 Mon Sep 17 00:00:00 2001 From: dashevchenko Date: Fri, 11 Apr 2025 15:32:55 +0300 Subject: [PATCH 43/47] added validation for entity name filter --- .../server/common/data/edqs/fields/EntityFields.java | 2 +- .../server/dao/entity/BaseEntityService.java | 10 ++++++++++ 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/EntityFields.java b/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/EntityFields.java index 1b0975542c..b1a22e6c8a 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/EntityFields.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/EntityFields.java @@ -147,7 +147,7 @@ public interface EntityFields { default String getAsString(String key) { return switch (key) { - case "createdTime" -> Long.toString(getCreatedTime()); + case "createdTime", "created_time" -> Long.toString(getCreatedTime()); case "title" -> getName(); case "type" -> getType(); case "label" -> getLabel(); diff --git a/dao/src/main/java/org/thingsboard/server/dao/entity/BaseEntityService.java b/dao/src/main/java/org/thingsboard/server/dao/entity/BaseEntityService.java index 4df33779ee..604cceff09 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/entity/BaseEntityService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/entity/BaseEntityService.java @@ -43,6 +43,7 @@ import org.thingsboard.server.common.data.query.EntityDataQuery; import org.thingsboard.server.common.data.query.EntityFilterType; import org.thingsboard.server.common.data.query.EntityKey; import org.thingsboard.server.common.data.query.EntityListFilter; +import org.thingsboard.server.common.data.query.EntityNameFilter; import org.thingsboard.server.common.data.query.EntityTypeFilter; import org.thingsboard.server.common.data.query.KeyFilter; import org.thingsboard.server.common.data.query.RelationsQueryFilter; @@ -61,6 +62,7 @@ import java.util.function.Function; import java.util.stream.Collectors; import static org.thingsboard.server.common.data.id.EntityId.NULL_UUID; +import static org.thingsboard.server.common.data.query.EntityFilterType.ENTITY_NAME; import static org.thingsboard.server.common.data.query.EntityFilterType.ENTITY_TYPE; import static org.thingsboard.server.dao.service.Validator.validateEntityDataPageLink; import static org.thingsboard.server.dao.service.Validator.validateId; @@ -250,6 +252,8 @@ public class BaseEntityService extends AbstractEntityService implements EntitySe validateRelationQuery((RelationsQueryFilter) query.getEntityFilter()); } else if (query.getEntityFilter().getType().equals(ENTITY_TYPE)) { validateEntityTypeQuery((EntityTypeFilter) query.getEntityFilter()); + } else if (query.getEntityFilter().getType().equals(ENTITY_NAME)) { + validateEntityNameQuery((EntityNameFilter) query.getEntityFilter()); } } @@ -264,6 +268,12 @@ public class BaseEntityService extends AbstractEntityService implements EntitySe } } + private static void validateEntityNameQuery(EntityNameFilter filter) { + if (filter.getEntityType() == null) { + throw new IncorrectParameterException("Entity type is required"); + } + } + private static void validateRelationQuery(RelationsQueryFilter queryFilter) { if (queryFilter.isMultiRoot() && queryFilter.getMultiRootEntitiesType() == null) { throw new IncorrectParameterException("Multi-root relation query filter should contain 'multiRootEntitiesType'"); From 53ffd00184cd49c2b45132d41f54074ec42c8cce Mon Sep 17 00:00:00 2001 From: dashevchenko Date: Fri, 11 Apr 2025 16:04:57 +0300 Subject: [PATCH 44/47] reverted "created_time" field support --- .../server/common/data/edqs/fields/EntityFields.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/EntityFields.java b/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/EntityFields.java index b1a22e6c8a..1b0975542c 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/EntityFields.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/EntityFields.java @@ -147,7 +147,7 @@ public interface EntityFields { default String getAsString(String key) { return switch (key) { - case "createdTime", "created_time" -> Long.toString(getCreatedTime()); + case "createdTime" -> Long.toString(getCreatedTime()); case "title" -> getName(); case "type" -> getType(); case "label" -> getLabel(); From 12739b60ceaa2c2e0ec8cc2f44979ef1ff796feb Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Fri, 11 Apr 2025 17:05:04 +0300 Subject: [PATCH 45/47] moved initFetchPackSize property to settings --- .../server/actors/ActorSystemContext.java | 6 ++++++ .../CalculatedFieldManagerMessageProcessor.java | 17 +++++++---------- .../TbQueueCalculatedFieldSettings.java | 2 ++ 3 files changed, 15 insertions(+), 10 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java b/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java index b5fa1e6fbd..26c82a33de 100644 --- a/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java +++ b/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java @@ -106,6 +106,7 @@ import org.thingsboard.server.dao.widget.WidgetsBundleService; import org.thingsboard.server.queue.discovery.DiscoveryService; import org.thingsboard.server.queue.discovery.PartitionService; import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; +import org.thingsboard.server.queue.settings.TbQueueCalculatedFieldSettings; import org.thingsboard.server.service.apiusage.TbApiUsageStateService; import org.thingsboard.server.service.cf.CalculatedFieldProcessingService; import org.thingsboard.server.service.cf.CalculatedFieldQueueService; @@ -459,6 +460,11 @@ public class ActorSystemContext { @Getter private DebugModeRateLimitsConfig debugModeRateLimitsConfig; + @Lazy + @Autowired(required = false) + @Getter + private TbQueueCalculatedFieldSettings calculatedFieldSettings; + /** * The following Service will be null if we operate in tb-core mode */ diff --git a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java index a896bbd74b..ba15bd10f2 100644 --- a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java +++ b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java @@ -15,9 +15,7 @@ */ package org.thingsboard.server.actors.calculatedField; -import lombok.Getter; import lombok.extern.slf4j.Slf4j; -import org.springframework.beans.factory.annotation.Value; import org.thingsboard.server.actors.ActorSystemContext; import org.thingsboard.server.actors.TbActorCtx; import org.thingsboard.server.actors.TbActorRef; @@ -47,6 +45,7 @@ import org.thingsboard.server.common.msg.queue.TbCallback; import org.thingsboard.server.dao.asset.AssetService; import org.thingsboard.server.dao.cf.CalculatedFieldService; import org.thingsboard.server.dao.device.DeviceService; +import org.thingsboard.server.queue.settings.TbQueueCalculatedFieldSettings; import org.thingsboard.server.service.cf.CalculatedFieldProcessingService; import org.thingsboard.server.service.cf.CalculatedFieldStateService; import org.thingsboard.server.service.cf.cache.TenantEntityProfileCache; @@ -82,14 +81,11 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware private final TbAssetProfileCache assetProfileCache; private final TbDeviceProfileCache deviceProfileCache; private final TenantEntityProfileCache entityProfileCache; + private final TbQueueCalculatedFieldSettings cfSettings; protected final TenantId tenantId; protected TbActorCtx ctx; - @Value("${queue.calculated_fields.init_tenant_fetch_pack_size:1000}") - @Getter - private int initFetchPackSize; - CalculatedFieldManagerMessageProcessor(ActorSystemContext systemContext, TenantId tenantId) { super(systemContext); this.cfExecService = systemContext.getCalculatedFieldProcessingService(); @@ -100,6 +96,7 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware this.assetProfileCache = systemContext.getAssetProfileCache(); this.deviceProfileCache = systemContext.getDeviceProfileCache(); this.entityProfileCache = new TenantEntityProfileCache(); + this.cfSettings = systemContext.getCalculatedFieldSettings(); this.tenantId = tenantId; } @@ -566,7 +563,7 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware } public void initCalculatedFields() { - PageDataIterable cfs = new PageDataIterable<>(pageLink -> cfDaoService.findCalculatedFieldsByTenantId(tenantId, pageLink), initFetchPackSize); + PageDataIterable cfs = new PageDataIterable<>(pageLink -> cfDaoService.findCalculatedFieldsByTenantId(tenantId, pageLink), cfSettings.getInitTenantFetchPackSize()); cfs.forEach(cf -> { log.trace("Processing calculated field record: {}", cf); try { @@ -578,14 +575,14 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware calculatedFields.values().forEach(cf -> { entityIdCalculatedFields.computeIfAbsent(cf.getEntityId(), id -> new CopyOnWriteArrayList<>()).add(cf); }); - PageDataIterable cfls = new PageDataIterable<>(pageLink -> cfDaoService.findAllCalculatedFieldLinksByTenantId(tenantId, pageLink), initFetchPackSize); + PageDataIterable cfls = new PageDataIterable<>(pageLink -> cfDaoService.findAllCalculatedFieldLinksByTenantId(tenantId, pageLink), cfSettings.getInitTenantFetchPackSize()); cfls.forEach(link -> { onLinkInitMsg(new CalculatedFieldLinkInitMsg(link.getTenantId(), link)); }); } private void initEntityProfileCache() { - PageDataIterable deviceIdInfos = new PageDataIterable<>(pageLink -> deviceService.findProfileEntityIdInfosByTenantId(tenantId, pageLink), initFetchPackSize); + PageDataIterable deviceIdInfos = new PageDataIterable<>(pageLink -> deviceService.findProfileEntityIdInfosByTenantId(tenantId, pageLink), cfSettings.getInitTenantFetchPackSize()); for (ProfileEntityIdInfo idInfo : deviceIdInfos) { log.trace("Processing device record: {}", idInfo); try { @@ -594,7 +591,7 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware log.error("Failed to process device record: {}", idInfo, e); } } - PageDataIterable assetIdInfos = new PageDataIterable<>(pageLink -> assetService.findProfileEntityIdInfosByTenantId(tenantId, pageLink), initFetchPackSize); + PageDataIterable assetIdInfos = new PageDataIterable<>(pageLink -> assetService.findProfileEntityIdInfosByTenantId(tenantId, pageLink), cfSettings.getInitTenantFetchPackSize()); for (ProfileEntityIdInfo idInfo : assetIdInfos) { log.trace("Processing asset record: {}", idInfo); try { diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/settings/TbQueueCalculatedFieldSettings.java b/common/queue/src/main/java/org/thingsboard/server/queue/settings/TbQueueCalculatedFieldSettings.java index c2de8eff4e..9a87e88416 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/settings/TbQueueCalculatedFieldSettings.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/settings/TbQueueCalculatedFieldSettings.java @@ -31,5 +31,7 @@ public class TbQueueCalculatedFieldSettings { @Value("${queue.calculated_fields.state_topic}") private String stateTopic; + @Value("${queue.calculated_fields.init_tenant_fetch_pack_size:1000}") + private int initTenantFetchPackSize; } From 68eb2b6a3a809a71343f1b1a7f2e0977afa35cf4 Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Fri, 11 Apr 2025 17:14:32 +0300 Subject: [PATCH 46/47] fixed tenant and customer update --- .../CalculatedFieldManagerMessageProcessor.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java index ba15bd10f2..ba1ca71bda 100644 --- a/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java +++ b/application/src/main/java/org/thingsboard/server/actors/calculatedField/CalculatedFieldManagerMessageProcessor.java @@ -535,8 +535,7 @@ public class CalculatedFieldManagerMessageProcessor extends AbstractContextAware return switch (entityId.getEntityType()) { case ASSET -> assetProfileCache.get(tenantId, (AssetId) entityId).getId(); case DEVICE -> deviceProfileCache.get(tenantId, (DeviceId) entityId).getId(); - default -> - throw new IllegalArgumentException("'" + entityId.getEntityType() + "' is not profile entity." + entityId); + default -> null; }; } From 469b3f81b76266513962a065d241a4c82fa129d1 Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Fri, 11 Apr 2025 17:29:31 +0300 Subject: [PATCH 47/47] removed sending partition change event --- .../java/org/thingsboard/server/actors/tenant/TenantActor.java | 1 - 1 file changed, 1 deletion(-) diff --git a/application/src/main/java/org/thingsboard/server/actors/tenant/TenantActor.java b/application/src/main/java/org/thingsboard/server/actors/tenant/TenantActor.java index 2a039327f2..f0d330726d 100644 --- a/application/src/main/java/org/thingsboard/server/actors/tenant/TenantActor.java +++ b/application/src/main/java/org/thingsboard/server/actors/tenant/TenantActor.java @@ -283,7 +283,6 @@ public class TenantActor extends RuleChainManagerActor { log.info("[{}] Failed to init CF Actor.", tenantId, e); } } - cfActor.tellWithHighPriority(msg); if (!ruleChainsInitialized) { log.info("Tenant {} is now managed by this service, initializing rule chains", tenantId); initRuleChains();