From e2ac2708b658c3802c9f15f9825bb621a3730f85 Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Mon, 6 Jan 2025 17:37:33 +0200 Subject: [PATCH 1/9] changed CF links config --- .../service/cf/CalculatedFieldCache.java | 2 + .../cf/DefaultCalculatedFieldCache.java | 37 ++++++++++++++----- ...efaultCalculatedFieldExecutionService.java | 28 +++++++++++++- ...CalculatedFieldAttributeUpdateRequest.java | 21 +++++++---- ...CalculatedFieldTelemetryUpdateRequest.java | 4 +- ...alculatedFieldTimeSeriesUpdateRequest.java | 16 +++++--- .../DefaultTelemetrySubscriptionService.java | 7 +--- .../server/dao/cf/CalculatedFieldService.java | 2 + .../BaseCalculatedFieldConfiguration.java | 17 +++++---- .../dao/cf/BaseCalculatedFieldService.java | 7 ++++ .../server/dao/cf/CalculatedFieldDao.java | 2 + .../dao/sql/cf/CalculatedFieldRepository.java | 3 ++ .../dao/sql/cf/JpaCalculatedFieldDao.java | 5 +++ 13 files changed, 114 insertions(+), 37 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldCache.java b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldCache.java index ad683c324c..aa25565a34 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldCache.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldCache.java @@ -30,6 +30,8 @@ public interface CalculatedFieldCache { CalculatedField getCalculatedField(TenantId tenantId, CalculatedFieldId calculatedFieldId); + List getCalculatedFieldsByEntityId(TenantId tenantId, EntityId entityId); + List getCalculatedFieldLinks(TenantId tenantId, CalculatedFieldId calculatedFieldId); List getCalculatedFieldLinksByEntityId(TenantId tenantId, EntityId entityId); diff --git a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldCache.java b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldCache.java index c4293b9edc..40d569b040 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldCache.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldCache.java @@ -56,6 +56,7 @@ public class DefaultCalculatedFieldCache implements CalculatedFieldCache { private final DeviceService deviceService; private final ConcurrentMap calculatedFields = new ConcurrentHashMap<>(); + private final ConcurrentMap> entityIdCalculatedFields = new ConcurrentHashMap<>(); private final ConcurrentMap> calculatedFieldLinks = new ConcurrentHashMap<>(); private final ConcurrentMap> entityIdCalculatedFieldLinks = new ConcurrentHashMap<>(); private final ConcurrentMap calculatedFieldsCtx = new ConcurrentHashMap<>(); @@ -65,10 +66,8 @@ public class DefaultCalculatedFieldCache implements CalculatedFieldCache { @Getter private int initFetchPackSize; - @PostConstruct public void init() { - // to discuss: fetch on start or fetch on demand PageDataIterable cfs = new PageDataIterable<>(calculatedFieldService::findAllCalculatedFields, initFetchPackSize); cfs.forEach(cf -> calculatedFields.putIfAbsent(cf.getId(), cf)); PageDataIterable cfls = new PageDataIterable<>(calculatedFieldService::findAllCalculatedFieldLinks, initFetchPackSize); @@ -97,19 +96,37 @@ public class DefaultCalculatedFieldCache implements CalculatedFieldCache { return calculatedField; } + @Override + public List getCalculatedFieldsByEntityId(TenantId tenantId, EntityId entityId) { + List cfs = entityIdCalculatedFields.get(entityId); + if (cfs == null) { + calculatedFieldFetchLock.lock(); + try { + cfs = entityIdCalculatedFields.get(entityId); + if (cfs == null) { + cfs = calculatedFieldService.findCalculatedFieldsByEntityId(tenantId, entityId); + entityIdCalculatedFields.put(entityId, cfs); + log.debug("[{}] Fetch calculated fields by entity into cache: {}", entityId, cfs); + } + } finally { + calculatedFieldFetchLock.unlock(); + } + } + log.trace("[{}] Found calculated fields by entity in cache: {}", entityId, cfs); + return cfs; + } + @Override public List getCalculatedFieldLinks(TenantId tenantId, CalculatedFieldId calculatedFieldId) { List cfLinks = calculatedFieldLinks.get(calculatedFieldId); - if (cfLinks == null || cfLinks.isEmpty()) { + if (cfLinks == null) { calculatedFieldFetchLock.lock(); try { cfLinks = calculatedFieldLinks.get(calculatedFieldId); - if (cfLinks == null || cfLinks.isEmpty()) { + if (cfLinks == null) { cfLinks = calculatedFieldService.findAllCalculatedFieldLinksById(tenantId, calculatedFieldId); - if (cfLinks != null) { - calculatedFieldLinks.put(calculatedFieldId, cfLinks); - log.debug("[{}] Fetch calculated field links into cache: {}", calculatedFieldId, cfLinks); - } + calculatedFieldLinks.put(calculatedFieldId, cfLinks); + log.debug("[{}] Fetch calculated field links into cache: {}", calculatedFieldId, cfLinks); } } finally { calculatedFieldFetchLock.unlock(); @@ -139,7 +156,6 @@ public class DefaultCalculatedFieldCache implements CalculatedFieldCache { return cfLinks; } - @Override public void updateCalculatedFieldLinks(TenantId tenantId, CalculatedFieldId calculatedFieldId) { log.debug("Update calculated field links per entity for calculated field: [{}]", calculatedFieldId); @@ -225,6 +241,9 @@ public class DefaultCalculatedFieldCache implements CalculatedFieldCache { CalculatedField oldCalculatedField = calculatedFields.remove(calculatedFieldId); log.debug("[{}] evict calculated field from cache: {}", calculatedFieldId, oldCalculatedField); calculatedFieldLinks.remove(calculatedFieldId); + log.debug("[{}] evict calculated field from cached calculated fields by entity id: {}", calculatedFieldId, oldCalculatedField); + entityIdCalculatedFields.forEach((entityId, calculatedFields) -> calculatedFields.removeIf(cf -> cf.getId().equals(calculatedFieldId))); + entityIdCalculatedFields.remove(oldCalculatedField.getEntityId()); log.debug("[{}] evict calculated field links from cache: {}", calculatedFieldId, oldCalculatedField); calculatedFieldsCtx.remove(calculatedFieldId); log.debug("[{}] evict calculated field ctx from cache: {}", calculatedFieldId, oldCalculatedField); diff --git a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java index f7bcb4e678..c0f7acf0a3 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java @@ -38,6 +38,7 @@ import org.thingsboard.server.cluster.TbClusterService; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.cf.CalculatedFieldLink; +import org.thingsboard.server.common.data.cf.CalculatedFieldLinkConfiguration; import org.thingsboard.server.common.data.cf.CalculatedFieldType; import org.thingsboard.server.common.data.cf.configuration.Argument; import org.thingsboard.server.common.data.cf.configuration.ArgumentType; @@ -139,6 +140,8 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas Math.max(4, Runtime.getRuntime().availableProcessors()), "calculated-field")); calculatedFieldCallbackExecutor = MoreExecutors.listeningDecorator(ThingsBoardExecutors.newWorkStealingPool( Math.max(4, Runtime.getRuntime().availableProcessors()), "calculated-field-callback")); + scheduledExecutor.submit(() -> rocksDBService.getAll() + .forEach((ctxId, ctx) -> states.put(JacksonUtil.fromString(ctxId, CalculatedFieldEntityCtxId.class), JacksonUtil.fromString(ctx, CalculatedFieldEntityCtx.class)))); } @PreDestroy @@ -337,11 +340,32 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas if (supportedReferencedEntities.contains(entityId.getEntityType())) { EntityId profileId = getProfileId(tenantId, entityId); + // process by profile + if (profileId != null) { + calculatedFieldCache.getCalculatedFieldsByEntityId(tenantId, profileId).forEach(cf -> { + CalculatedFieldLinkConfiguration linkConfiguration = cf.getConfiguration().getReferencedEntityConfig(profileId); + Map telemetryKeys = calculatedFieldTelemetryUpdateRequest.getTelemetryKeysFromLink(linkConfiguration); + Map updatedTelemetry = calculatedFieldTelemetryUpdateRequest.getKvEntries().stream() + .filter(entry -> telemetryKeys.containsKey(entry.getKey())) + .collect(Collectors.toMap( + entry -> getMappedKey(entry, telemetryKeys), + entry -> entry, + (v1, v2) -> v1 + )); + + if (!updatedTelemetry.isEmpty()) { + List previousCalculatedFieldIds = calculatedFieldTelemetryUpdateRequest.getPreviousCalculatedFieldIds(); + executeTelemetryUpdate(tenantId, entityId, cf.getId(), previousCalculatedFieldIds, updatedTelemetry); + } + }); + } + + // process by links getCalculatedFieldLinks(tenantId, entityId, profileId).forEach(link -> { CalculatedFieldId calculatedFieldId = link.getCalculatedFieldId(); - Map telemetryKeys = calculatedFieldTelemetryUpdateRequest.getTelemetryKeysFromLink(link); + Map telemetryKeys = calculatedFieldTelemetryUpdateRequest.getTelemetryKeysFromLink(link.getConfiguration()); Map updatedTelemetry = calculatedFieldTelemetryUpdateRequest.getKvEntries().stream() - .filter(entry -> telemetryKeys.containsValue(entry.getKey())) + .filter(entry -> telemetryKeys.containsKey(entry.getKey())) .collect(Collectors.toMap( entry -> getMappedKey(entry, telemetryKeys), entry -> entry, diff --git a/application/src/main/java/org/thingsboard/server/service/cf/telemetry/CalculatedFieldAttributeUpdateRequest.java b/application/src/main/java/org/thingsboard/server/service/cf/telemetry/CalculatedFieldAttributeUpdateRequest.java index c56217b2ce..25d2f57bd6 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/telemetry/CalculatedFieldAttributeUpdateRequest.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/telemetry/CalculatedFieldAttributeUpdateRequest.java @@ -15,10 +15,10 @@ */ package org.thingsboard.server.service.cf.telemetry; -import lombok.AllArgsConstructor; import lombok.Data; +import org.thingsboard.rule.engine.api.AttributesSaveRequest; import org.thingsboard.server.common.data.AttributeScope; -import org.thingsboard.server.common.data.cf.CalculatedFieldLink; +import org.thingsboard.server.common.data.cf.CalculatedFieldLinkConfiguration; import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; @@ -28,7 +28,6 @@ import java.util.List; import java.util.Map; @Data -@AllArgsConstructor public class CalculatedFieldAttributeUpdateRequest implements CalculatedFieldTelemetryUpdateRequest { private TenantId tenantId; @@ -37,12 +36,20 @@ public class CalculatedFieldAttributeUpdateRequest implements CalculatedFieldTel private List kvEntries; private List previousCalculatedFieldIds; + public CalculatedFieldAttributeUpdateRequest(AttributesSaveRequest request) { + this.tenantId = request.getTenantId(); + this.entityId = request.getEntityId(); + this.scope = request.getScope(); + this.kvEntries = request.getEntries(); + this.previousCalculatedFieldIds = request.getPreviousCalculatedFieldIds(); + } + @Override - public Map getTelemetryKeysFromLink(CalculatedFieldLink link) { + public Map getTelemetryKeysFromLink(CalculatedFieldLinkConfiguration linkConfiguration) { return switch (scope) { - case CLIENT_SCOPE -> link.getConfiguration().getClientAttributes(); - case SERVER_SCOPE -> link.getConfiguration().getServerAttributes(); - case SHARED_SCOPE -> link.getConfiguration().getSharedAttributes(); + case CLIENT_SCOPE -> linkConfiguration.getClientAttributes(); + case SERVER_SCOPE -> linkConfiguration.getServerAttributes(); + case SHARED_SCOPE -> linkConfiguration.getSharedAttributes(); }; } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/telemetry/CalculatedFieldTelemetryUpdateRequest.java b/application/src/main/java/org/thingsboard/server/service/cf/telemetry/CalculatedFieldTelemetryUpdateRequest.java index 98062a08db..29ee899ec9 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/telemetry/CalculatedFieldTelemetryUpdateRequest.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/telemetry/CalculatedFieldTelemetryUpdateRequest.java @@ -15,7 +15,7 @@ */ package org.thingsboard.server.service.cf.telemetry; -import org.thingsboard.server.common.data.cf.CalculatedFieldLink; +import org.thingsboard.server.common.data.cf.CalculatedFieldLinkConfiguration; import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; @@ -34,6 +34,6 @@ public interface CalculatedFieldTelemetryUpdateRequest { List getPreviousCalculatedFieldIds(); - Map getTelemetryKeysFromLink(CalculatedFieldLink link); + Map getTelemetryKeysFromLink(CalculatedFieldLinkConfiguration linkConfiguration); } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/telemetry/CalculatedFieldTimeSeriesUpdateRequest.java b/application/src/main/java/org/thingsboard/server/service/cf/telemetry/CalculatedFieldTimeSeriesUpdateRequest.java index bd2161dca1..6225286631 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/telemetry/CalculatedFieldTimeSeriesUpdateRequest.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/telemetry/CalculatedFieldTimeSeriesUpdateRequest.java @@ -15,9 +15,9 @@ */ package org.thingsboard.server.service.cf.telemetry; -import lombok.AllArgsConstructor; import lombok.Data; -import org.thingsboard.server.common.data.cf.CalculatedFieldLink; +import org.thingsboard.rule.engine.api.TimeseriesSaveRequest; +import org.thingsboard.server.common.data.cf.CalculatedFieldLinkConfiguration; import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; @@ -27,7 +27,6 @@ import java.util.List; import java.util.Map; @Data -@AllArgsConstructor public class CalculatedFieldTimeSeriesUpdateRequest implements CalculatedFieldTelemetryUpdateRequest { private TenantId tenantId; @@ -35,9 +34,16 @@ public class CalculatedFieldTimeSeriesUpdateRequest implements CalculatedFieldTe private List kvEntries; private List previousCalculatedFieldIds; + public CalculatedFieldTimeSeriesUpdateRequest(TimeseriesSaveRequest request) { + this.tenantId = request.getTenantId(); + this.entityId = request.getEntityId(); + this.kvEntries = request.getEntries(); + this.previousCalculatedFieldIds = request.getPreviousCalculatedFieldIds(); + } + @Override - public Map getTelemetryKeysFromLink(CalculatedFieldLink link) { - return link.getConfiguration().getTimeSeries(); + public Map getTelemetryKeysFromLink(CalculatedFieldLinkConfiguration linkConfiguration) { + return linkConfiguration.getTimeSeries(); } } diff --git a/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java b/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java index f873e48774..8773564e5d 100644 --- a/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java +++ b/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java @@ -154,9 +154,7 @@ public class DefaultTelemetrySubscriptionService extends AbstractSubscriptionSer if (request.isSaveLatest() && !request.isOnlyLatest()) { addEntityViewCallback(tenantId, entityId, request.getEntries()); } - // Use something very similar to addMainCallback. don't forget about tsCallBackExecutor. - //CalculatedFieldTimeSeriesUpdateRequest - add constructor that accepts the TimeseriesSaveRequest - addCallback(saveFuture, success -> calculatedFieldExecutionService.onTelemetryUpdate(new CalculatedFieldTimeSeriesUpdateRequest(tenantId, entityId, request.getEntries(), request.getPreviousCalculatedFieldIds())), tsCallBackExecutor); + addCallback(saveFuture, success -> calculatedFieldExecutionService.onTelemetryUpdate(new CalculatedFieldTimeSeriesUpdateRequest(request)), tsCallBackExecutor); return saveFuture; } @@ -172,8 +170,7 @@ public class DefaultTelemetrySubscriptionService extends AbstractSubscriptionSer ListenableFuture> saveFuture = attrService.save(request.getTenantId(), request.getEntityId(), request.getScope(), request.getEntries()); addMainCallback(saveFuture, request.getCallback()); addWsCallback(saveFuture, success -> onAttributesUpdate(request.getTenantId(), request.getEntityId(), request.getScope().name(), request.getEntries(), request.isNotifyDevice())); - //CalculatedFieldAttributeUpdateRequest - add constructor that accepts the AttributesSaveRequest - addCallback(saveFuture, success -> calculatedFieldExecutionService.onTelemetryUpdate(new CalculatedFieldAttributeUpdateRequest(request.getTenantId(), request.getEntityId(), request.getScope(), request.getEntries(), request.getPreviousCalculatedFieldIds())), tsCallBackExecutor); + addCallback(saveFuture, success -> calculatedFieldExecutionService.onTelemetryUpdate(new CalculatedFieldAttributeUpdateRequest(request)), tsCallBackExecutor); } @Override diff --git a/common/dao-api/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldService.java b/common/dao-api/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldService.java index 1e64fdac60..3a508a5c08 100644 --- a/common/dao-api/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldService.java +++ b/common/dao-api/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldService.java @@ -38,6 +38,8 @@ public interface CalculatedFieldService extends EntityDaoService { List findCalculatedFieldIdsByEntityId(TenantId tenantId, EntityId entityId); + List findCalculatedFieldsByEntityId(TenantId tenantId, EntityId entityId); + List findAllCalculatedFields(); PageData findAllCalculatedFields(PageLink pageLink); diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/BaseCalculatedFieldConfiguration.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/BaseCalculatedFieldConfiguration.java index 8c86b6c552..c3c4e32507 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/BaseCalculatedFieldConfiguration.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/BaseCalculatedFieldConfiguration.java @@ -68,19 +68,22 @@ public abstract class BaseCalculatedFieldConfiguration implements CalculatedFiel arguments.entrySet().stream() .filter(entry -> entry.getValue().getEntityId().equals(entityId)) .forEach(entry -> { - Argument argument = entry.getValue(); + Argument tergetArgument = entry.getValue(); String argumentKey = entry.getKey(); - switch (argument.getType()) { + switch (tergetArgument.getType()) { case ATTRIBUTE -> { - switch (argument.getScope()) { - case CLIENT_SCOPE -> linkConfiguration.getClientAttributes().put(entry.getKey(), argument.getKey()); - case SERVER_SCOPE -> linkConfiguration.getServerAttributes().put(entry.getKey(), argument.getKey()); - case SHARED_SCOPE -> linkConfiguration.getSharedAttributes().put(entry.getKey(), argument.getKey()); + switch (tergetArgument.getScope()) { + case CLIENT_SCOPE -> + linkConfiguration.getClientAttributes().put(tergetArgument.getKey(), argumentKey); + case SERVER_SCOPE -> + linkConfiguration.getServerAttributes().put(tergetArgument.getKey(), argumentKey); + case SHARED_SCOPE -> + linkConfiguration.getSharedAttributes().put(tergetArgument.getKey(), argumentKey); } } case TS_LATEST, TS_ROLLING -> - linkConfiguration.getTimeSeries().put(argumentKey, argument.getKey()); + linkConfiguration.getTimeSeries().put(tergetArgument.getKey(), argumentKey); } }); diff --git a/dao/src/main/java/org/thingsboard/server/dao/cf/BaseCalculatedFieldService.java b/dao/src/main/java/org/thingsboard/server/dao/cf/BaseCalculatedFieldService.java index 0849414a0e..e650aec35e 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/cf/BaseCalculatedFieldService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/cf/BaseCalculatedFieldService.java @@ -98,6 +98,13 @@ public class BaseCalculatedFieldService extends AbstractEntityService implements return calculatedFieldDao.findCalculatedFieldIdsByEntityId(tenantId, entityId); } + @Override + public List findCalculatedFieldsByEntityId(TenantId tenantId, EntityId entityId) { + log.trace("Executing findCalculatedFieldsByEntityId [{}]", entityId); + validateId(entityId.getId(), id -> INCORRECT_ENTITY_ID + id); + return calculatedFieldDao.findCalculatedFieldsByEntityId(tenantId, entityId); + } + @Override public List findAllCalculatedFields() { log.trace("Executing findAll"); diff --git a/dao/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldDao.java b/dao/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldDao.java index 5b3bcc2750..39663d0afc 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/cf/CalculatedFieldDao.java @@ -31,6 +31,8 @@ public interface CalculatedFieldDao extends Dao { List findCalculatedFieldIdsByEntityId(TenantId tenantId, EntityId entityId); + List findCalculatedFieldsByEntityId(TenantId tenantId, EntityId entityId); + List findAll(); PageData findAll(PageLink pageLink); diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/cf/CalculatedFieldRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/cf/CalculatedFieldRepository.java index 9aa0aee428..816fa1546c 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/cf/CalculatedFieldRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/cf/CalculatedFieldRepository.java @@ -16,6 +16,7 @@ package org.thingsboard.server.dao.sql.cf; import org.springframework.data.jpa.repository.JpaRepository; +import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.dao.model.sql.CalculatedFieldEntity; @@ -28,6 +29,8 @@ public interface CalculatedFieldRepository extends JpaRepository findCalculatedFieldIdsByTenantIdAndEntityId(UUID tenantId, UUID entityId); + List findAllByTenantIdAndEntityId(UUID tenantId, UUID entityId); + List findAllByTenantId(UUID tenantId); List removeAllByTenantIdAndEntityId(UUID tenantId, UUID entityId); diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/cf/JpaCalculatedFieldDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/cf/JpaCalculatedFieldDao.java index e3762f6157..20081299e8 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/cf/JpaCalculatedFieldDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/cf/JpaCalculatedFieldDao.java @@ -55,6 +55,11 @@ public class JpaCalculatedFieldDao extends JpaAbstractDao findCalculatedFieldsByEntityId(TenantId tenantId, EntityId entityId) { + return calculatedFieldRepository.findAllByTenantIdAndEntityId(tenantId.getId(), entityId.getId()); + } + @Override public List findAll() { return DaoUtil.convertDataList(calculatedFieldRepository.findAll()); From 03c3341265724341aea3668f6f376a7e90d842f3 Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Wed, 8 Jan 2025 12:39:49 +0200 Subject: [PATCH 2/9] added logic to send msgs to RE when not my partition --- .../server/controller/BaseController.java | 7 - .../cf/CalculatedFieldExecutionService.java | 2 + ...efaultCalculatedFieldExecutionService.java | 252 ++++++++++++++---- .../cf/ctx/CalculatedFieldEntityCtxId.java | 5 +- ...CalculatedFieldAttributeUpdateRequest.java | 6 +- ...alculatedFieldTimeSeriesUpdateRequest.java | 6 +- .../TbRuleEngineQueueConsumerManager.java | 9 +- .../BaseCalculatedFieldConfiguration.java | 14 +- .../server/common/util/ProtoUtils.java | 133 +++++++++ common/proto/src/main/proto/queue.proto | 29 +- .../dao/cf/BaseCalculatedFieldService.java | 1 + 11 files changed, 386 insertions(+), 78 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/controller/BaseController.java b/application/src/main/java/org/thingsboard/server/controller/BaseController.java index 4987096d17..139c61d710 100644 --- a/application/src/main/java/org/thingsboard/server/controller/BaseController.java +++ b/application/src/main/java/org/thingsboard/server/controller/BaseController.java @@ -366,9 +366,6 @@ public abstract class BaseController { @Autowired protected TbServiceInfoProvider serviceInfoProvider; - @Autowired - protected CalculatedFieldService calculatedFieldService; - @Autowired protected NotificationTargetService notificationTargetService; @@ -998,10 +995,6 @@ public abstract class BaseController { return null; } - protected CalculatedField checkCalculatedFieldId(CalculatedFieldId calculatedFieldId, Operation operation) throws ThingsboardException { - return checkEntityId(calculatedFieldId, calculatedFieldService::findById, operation); - } - protected MediaType parseMediaType(String contentType) { try { return MediaType.parseMediaType(contentType); diff --git a/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldExecutionService.java b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldExecutionService.java index e4b0a7ca1e..6d1d459b9b 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldExecutionService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldExecutionService.java @@ -25,6 +25,8 @@ public interface CalculatedFieldExecutionService { void onTelemetryUpdate(CalculatedFieldTelemetryUpdateRequest calculatedFieldTelemetryUpdateRequest); + void onTelemetryUpdateMsg(TransportProtos.TelemetryUpdateMsgProto proto); + void onCalculatedFieldStateMsg(TransportProtos.CalculatedFieldStateMsgProto proto, TbCallback callback); void onEntityProfileChangedMsg(TransportProtos.EntityProfileUpdateMsgProto proto, TbCallback callback); diff --git a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java index c0f7acf0a3..0de3136439 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java @@ -35,7 +35,9 @@ import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.common.util.ThingsBoardExecutors; import org.thingsboard.script.api.tbel.TbelInvokeService; import org.thingsboard.server.cluster.TbClusterService; +import org.thingsboard.server.common.data.AttributeScope; import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.StringUtils; import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.cf.CalculatedFieldLink; import org.thingsboard.server.common.data.cf.CalculatedFieldLinkConfiguration; @@ -51,6 +53,7 @@ import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.EntityIdFactory; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.kv.Aggregation; +import org.thingsboard.server.common.data.kv.AttributeKvEntry; import org.thingsboard.server.common.data.kv.BaseAttributeKvEntry; import org.thingsboard.server.common.data.kv.BaseReadTsKvQuery; import org.thingsboard.server.common.data.kv.BasicTsKvEntry; @@ -67,6 +70,7 @@ import org.thingsboard.server.common.msg.TbMsgMetaData; import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.common.msg.queue.TbCallback; import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; +import org.thingsboard.server.common.util.ProtoUtils; import org.thingsboard.server.dao.attributes.AttributesService; import org.thingsboard.server.dao.cf.CalculatedFieldService; import org.thingsboard.server.dao.timeseries.TimeseriesService; @@ -80,7 +84,9 @@ import org.thingsboard.server.service.cf.ctx.state.ScriptCalculatedFieldState; import org.thingsboard.server.service.cf.ctx.state.SimpleCalculatedFieldState; import org.thingsboard.server.service.cf.ctx.state.SingleValueArgumentEntry; import org.thingsboard.server.service.cf.ctx.state.TsRollingArgumentEntry; +import org.thingsboard.server.service.cf.telemetry.CalculatedFieldAttributeUpdateRequest; import org.thingsboard.server.service.cf.telemetry.CalculatedFieldTelemetryUpdateRequest; +import org.thingsboard.server.service.cf.telemetry.CalculatedFieldTimeSeriesUpdateRequest; import org.thingsboard.server.service.partition.AbstractPartitionBasedService; import org.thingsboard.server.service.profile.TbAssetProfileCache; import org.thingsboard.server.service.profile.TbDeviceProfileCache; @@ -102,6 +108,7 @@ import java.util.stream.Collectors; import static org.thingsboard.server.common.data.DataConstants.SCOPE; import static org.thingsboard.server.common.util.ProtoUtils.fromObjectProto; import static org.thingsboard.server.common.util.ProtoUtils.toObjectProto; +import static org.thingsboard.server.common.util.ProtoUtils.toTsKvProto; @Service @Slf4j @@ -177,8 +184,8 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas TopicPartitionInfo tpi; try { tpi = partitionService.resolve(ServiceType.TB_RULE_ENGINE, cf.getTenantId(), entityId); - if (addedPartitions.contains(tpi) && states.keySet().stream().noneMatch(ctxId -> ctxId.cfId().equals(cf.getId().getId()))) { - tpiTargetEntityMap.computeIfAbsent(tpi, k -> new ArrayList<>()).add(new CalculatedFieldEntityCtxId(cf.getId().getId(), entityId.getId())); + if (addedPartitions.contains(tpi) && states.keySet().stream().noneMatch(ctxId -> ctxId.cfId().equals(cf.getId()))) { + tpiTargetEntityMap.computeIfAbsent(tpi, k -> new ArrayList<>()).add(new CalculatedFieldEntityCtxId(cf.getId(), entityId)); } } catch (Exception e) { log.warn("Failed to resolve partition for CalculatedFieldEntityCtxId: entityId=[{}], tenantId=[{}]. Reason: {}", @@ -213,7 +220,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas return result; } - private void restoreState(UUID calculatedFieldId, UUID entityId) { + private void restoreState(CalculatedFieldId calculatedFieldId, EntityId entityId) { CalculatedFieldEntityCtxId ctxId = new CalculatedFieldEntityCtxId(calculatedFieldId, entityId); String storedState = rocksDBService.get(JacksonUtil.writeValueAsString(ctxId)); @@ -232,7 +239,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas } private void cleanupEntity(CalculatedFieldId calculatedFieldId) { - states.keySet().removeIf(ctxId -> ctxId.cfId().equals(calculatedFieldId.getId())); + states.keySet().removeIf(ctxId -> ctxId.cfId().equals(calculatedFieldId)); } @Override @@ -243,7 +250,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas log.info("Received CalculatedFieldMsgProto for processing: tenantId=[{}], calculatedFieldId=[{}]", tenantId, calculatedFieldId); if (proto.getDeleted()) { log.warn("Executing onCalculatedFieldDelete, calculatedFieldId=[{}]", calculatedFieldId); - onCalculatedFieldDelete(tenantId, calculatedFieldId, callback); + onCalculatedFieldDelete(calculatedFieldId, callback); callback.onSuccess(); } CalculatedField cf = calculatedFieldCache.getCalculatedField(tenantId, calculatedFieldId); @@ -293,7 +300,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas CalculatedField oldCalculatedField = calculatedFieldCache.getCalculatedField(updatedCalculatedField.getTenantId(), updatedCalculatedField.getId()); boolean shouldReinit = true; if (hasSignificantChanges(oldCalculatedField, updatedCalculatedField)) { - onCalculatedFieldDelete(updatedCalculatedField.getTenantId(), updatedCalculatedField.getId(), callback); + onCalculatedFieldDelete(updatedCalculatedField.getId(), callback); } else { callback.onSuccess(); shouldReinit = false; @@ -301,12 +308,12 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas return shouldReinit; } - private void onCalculatedFieldDelete(TenantId tenantId, CalculatedFieldId calculatedFieldId, TbCallback callback) { + private void onCalculatedFieldDelete(CalculatedFieldId calculatedFieldId, TbCallback callback) { try { cleanupEntity(calculatedFieldId); - states.keySet().removeIf(ctxId -> ctxId.cfId().equals(calculatedFieldId.getId())); + states.keySet().removeIf(ctxId -> ctxId.cfId().equals(calculatedFieldId)); List statesToRemove = states.keySet().stream() - .filter(ctxId -> ctxId.cfId().equals(calculatedFieldId.getId())) + .filter(ctxId -> ctxId.cfId().equals(calculatedFieldId)) .map(JacksonUtil::writeValueAsString) .toList(); rocksDBService.deleteAll(statesToRemove); @@ -334,61 +341,147 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas @Override public void onTelemetryUpdate(CalculatedFieldTelemetryUpdateRequest calculatedFieldTelemetryUpdateRequest) { try { - TenantId tenantId = calculatedFieldTelemetryUpdateRequest.getTenantId(); EntityId entityId = calculatedFieldTelemetryUpdateRequest.getEntityId(); if (supportedReferencedEntities.contains(entityId.getEntityType())) { - EntityId profileId = getProfileId(tenantId, entityId); - - // process by profile - if (profileId != null) { - calculatedFieldCache.getCalculatedFieldsByEntityId(tenantId, profileId).forEach(cf -> { - CalculatedFieldLinkConfiguration linkConfiguration = cf.getConfiguration().getReferencedEntityConfig(profileId); - Map telemetryKeys = calculatedFieldTelemetryUpdateRequest.getTelemetryKeysFromLink(linkConfiguration); - Map updatedTelemetry = calculatedFieldTelemetryUpdateRequest.getKvEntries().stream() - .filter(entry -> telemetryKeys.containsKey(entry.getKey())) - .collect(Collectors.toMap( - entry -> getMappedKey(entry, telemetryKeys), - entry -> entry, - (v1, v2) -> v1 - )); - - if (!updatedTelemetry.isEmpty()) { - List previousCalculatedFieldIds = calculatedFieldTelemetryUpdateRequest.getPreviousCalculatedFieldIds(); - executeTelemetryUpdate(tenantId, entityId, cf.getId(), previousCalculatedFieldIds, updatedTelemetry); - } + TenantId tenantId = calculatedFieldTelemetryUpdateRequest.getTenantId(); + Map> tpiStatesToUpdate = new HashMap<>(); + + updateTelemetryForEntity(calculatedFieldTelemetryUpdateRequest, tpiStatesToUpdate); + updateTelemetryForProfile(calculatedFieldTelemetryUpdateRequest, getProfileId(tenantId, entityId), tpiStatesToUpdate); + updateTelemetryForLinkedEntities(calculatedFieldTelemetryUpdateRequest, tpiStatesToUpdate); + + if (!tpiStatesToUpdate.isEmpty()) { + tpiStatesToUpdate.forEach((topicPartitionInfo, ctxIds) -> { + TransportProtos.TelemetryUpdateMsgProto telemetryUpdateMsgProto = buildTelemetryUpdateMsgProto(calculatedFieldTelemetryUpdateRequest, ctxIds); + clusterService.pushMsgToRuleEngine(topicPartitionInfo, UUID.randomUUID(), TransportProtos.ToRuleEngineMsg.newBuilder().setCfTelemetryUpdateMsg(telemetryUpdateMsgProto).build(), null); }); } + } + } catch (Exception e) { + log.trace("Failed to update telemetry.", e); + } + } - // process by links - getCalculatedFieldLinks(tenantId, entityId, profileId).forEach(link -> { + private void updateTelemetryForEntity(CalculatedFieldTelemetryUpdateRequest request, Map> tpiStates) { + updateTelemetryForEntity(request, request.getEntityId(), tpiStates); + } + + private void updateTelemetryForProfile(CalculatedFieldTelemetryUpdateRequest request, EntityId profileId, Map> tpiStates) { + updateTelemetryForEntity(request, profileId, tpiStates); + } + + private void updateTelemetryForEntity(CalculatedFieldTelemetryUpdateRequest request, EntityId targetEntity, Map> tpiStates) { + TenantId tenantId = request.getTenantId(); + EntityId entityId = request.getEntityId(); + + TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_RULE_ENGINE, tenantId, entityId); + if (tpi.isMyPartition()) { + if (targetEntity != null) { + calculatedFieldCache.getCalculatedFieldsByEntityId(tenantId, targetEntity).forEach(cf -> { + CalculatedFieldLinkConfiguration linkConfiguration = cf.getConfiguration().getReferencedEntityConfig(targetEntity); + mapAndProcessUpdatedTelemetry(tenantId, entityId, cf.getId(), request, linkConfiguration); + }); + } + } else { + List ctxIds = tpiStates.computeIfAbsent(tpi, k -> new ArrayList<>()); + calculatedFieldCache.getCalculatedFieldsByEntityId(tenantId, targetEntity).forEach(cf -> { + ctxIds.add(new CalculatedFieldEntityCtxId(cf.getId(), entityId)); + }); + } + } + + private void updateTelemetryForLinkedEntity(CalculatedFieldTelemetryUpdateRequest request, EntityId targetEntity, CalculatedFieldLink link, Map> tpiStates) { + TenantId tenantId = request.getTenantId(); + EntityId entityId = request.getEntityId(); + CalculatedFieldId calculatedFieldId = link.getCalculatedFieldId(); + + TopicPartitionInfo targetEntityTpi = partitionService.resolve(ServiceType.TB_RULE_ENGINE, tenantId, targetEntity); + if (targetEntityTpi.isMyPartition()) { + mapAndProcessUpdatedTelemetry(tenantId, entityId, calculatedFieldId, request, link.getConfiguration()); + } else { + List ctxIds = tpiStates.computeIfAbsent(targetEntityTpi, k -> new ArrayList<>()); + ctxIds.add(new CalculatedFieldEntityCtxId(calculatedFieldId, targetEntity)); + } + } + + private void updateTelemetryForLinkedEntities(CalculatedFieldTelemetryUpdateRequest request, Map> tpiStates) { + TenantId tenantId = request.getTenantId(); + EntityId entityId = request.getEntityId(); + + calculatedFieldCache.getCalculatedFieldLinksByEntityId(tenantId, entityId) + .forEach(link -> { CalculatedFieldId calculatedFieldId = link.getCalculatedFieldId(); - Map telemetryKeys = calculatedFieldTelemetryUpdateRequest.getTelemetryKeysFromLink(link.getConfiguration()); - Map updatedTelemetry = calculatedFieldTelemetryUpdateRequest.getKvEntries().stream() - .filter(entry -> telemetryKeys.containsKey(entry.getKey())) - .collect(Collectors.toMap( - entry -> getMappedKey(entry, telemetryKeys), - entry -> entry, - (v1, v2) -> v1 - )); - - if (!updatedTelemetry.isEmpty()) { - List previousCalculatedFieldIds = calculatedFieldTelemetryUpdateRequest.getPreviousCalculatedFieldIds(); - executeTelemetryUpdate(tenantId, entityId, calculatedFieldId, previousCalculatedFieldIds, updatedTelemetry); + EntityId targetEntityId = calculatedFieldCache.getCalculatedField(tenantId, calculatedFieldId).getEntityId(); + + if (isProfileEntity(targetEntityId)) { + calculatedFieldCache.getEntitiesByProfile(tenantId, targetEntityId).forEach(entityByProfile -> { + updateTelemetryForLinkedEntity(request, entityByProfile, link, tpiStates); + }); + } else { + updateTelemetryForLinkedEntity(request, targetEntityId, link, tpiStates); } }); - } - } catch (Exception e) { - log.trace("Failed to update telemetry.", e); + } + + private void mapAndProcessUpdatedTelemetry(TenantId tenantId, + EntityId entityId, + CalculatedFieldId calculatedFieldId, + CalculatedFieldTelemetryUpdateRequest request, + CalculatedFieldLinkConfiguration linkConfiguration) { + Map telemetryKeys = request.getTelemetryKeysFromLink(linkConfiguration); + Map updatedTelemetry = mapTelemetryKeys(telemetryKeys, request.getKvEntries()); + + if (!updatedTelemetry.isEmpty()) { + List previousCalculatedFieldIds = request.getPreviousCalculatedFieldIds(); + executeTelemetryUpdate(tenantId, entityId, calculatedFieldId, previousCalculatedFieldIds, updatedTelemetry); } } - private String getMappedKey(KvEntry entry, Map telemetry) { - return telemetry.entrySet().stream() - .filter(kvEntry -> kvEntry.getValue().equals(entry.getKey())) - .map(Map.Entry::getKey) - .findFirst() - .orElse(entry.getKey()); + private Map mapTelemetryKeys(Map telemetryKeys, List kvEntries) { + return kvEntries.stream() + .filter(entry -> telemetryKeys.containsKey(entry.getKey())) + .collect(Collectors.toMap( + entry -> telemetryKeys.getOrDefault(entry.getKey(), entry.getKey()), + entry -> entry, + (v1, v2) -> v1 + )); + } + + @Override + public void onTelemetryUpdateMsg(TransportProtos.TelemetryUpdateMsgProto proto) { + try { + TenantId tenantId = TenantId.fromUUID(new UUID(proto.getTenantIdMSB(), proto.getTenantIdLSB())); + + proto.getLinksList().forEach(ctxIdProto -> { + EntityId entityId = EntityIdFactory.getByTypeAndUuid( + ctxIdProto.getEntityType(), new UUID(ctxIdProto.getEntityIdMSB(), ctxIdProto.getEntityIdLSB())); + + List updatedTelemetry = proto.getUpdatedTelemetryList().stream() + .map(ProtoUtils::fromTelemetryProto) + .toList(); + + boolean attributesUpdated = StringUtils.isEmpty(proto.getScope()); + + CalculatedFieldTelemetryUpdateRequest request = attributesUpdated + ? new CalculatedFieldAttributeUpdateRequest( + tenantId, entityId, AttributeScope.valueOf(proto.getScope()), updatedTelemetry, + proto.getPreviousCalculatedFieldsList().stream() + .map(cfIdProto -> new CalculatedFieldId( + new UUID(cfIdProto.getCalculatedFieldIdMSB(), cfIdProto.getCalculatedFieldIdLSB()))) + .toList()) + : new CalculatedFieldTimeSeriesUpdateRequest( + tenantId, entityId, updatedTelemetry, + proto.getPreviousCalculatedFieldsList().stream() + .map(cfIdProto -> new CalculatedFieldId( + new UUID(cfIdProto.getCalculatedFieldIdMSB(), cfIdProto.getCalculatedFieldIdLSB()))) + .toList()); + + onTelemetryUpdate(request); + }); + } catch (Exception e) { + log.trace("Failed to process telemetry update msg: [{}]", proto, e); + } } private void executeTelemetryUpdate(TenantId tenantId, EntityId entityId, CalculatedFieldId calculatedFieldId, List previousCalculatedFieldIds, Map updatedTelemetry) { @@ -481,7 +574,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_RULE_ENGINE, tenantId, entityId); if (tpi.isMyPartition()) { log.warn("Executing clearState, calculatedFieldId=[{}], entityId=[{}]", calculatedFieldId, entityId); - CalculatedFieldEntityCtxId ctxId = new CalculatedFieldEntityCtxId(calculatedFieldId.getId(), entityId.getId()); + CalculatedFieldEntityCtxId ctxId = new CalculatedFieldEntityCtxId(calculatedFieldId, entityId); states.remove(ctxId); rocksDBService.delete(JacksonUtil.writeValueAsString(ctxId)); } else { @@ -537,7 +630,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_RULE_ENGINE, tenantId, entityId); if (tpi.isMyPartition()) { - CalculatedFieldEntityCtxId entityCtxId = new CalculatedFieldEntityCtxId(cfId.getId(), entityId.getId()); + CalculatedFieldEntityCtxId entityCtxId = new CalculatedFieldEntityCtxId(cfId, entityId); states.compute(entityCtxId, (ctxId, ctx) -> { CalculatedFieldEntityCtx calculatedFieldEntityCtx = ctx != null ? ctx : fetchCalculatedFieldEntityState(ctxId, calculatedFieldCtx.getCfType()); @@ -777,6 +870,57 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas } } + private TransportProtos.TelemetryUpdateMsgProto buildTelemetryUpdateMsgProto( + CalculatedFieldTelemetryUpdateRequest request, List links + ) { + TransportProtos.TelemetryUpdateMsgProto.Builder builder = TransportProtos.TelemetryUpdateMsgProto.newBuilder(); + + builder.setTenantIdMSB(request.getTenantId().getId().getMostSignificantBits()) + .setTenantIdLSB(request.getTenantId().getId().getLeastSignificantBits()); + + for (CalculatedFieldEntityCtxId link : links) { + builder.addLinks(toProto(link)); + } + + for (CalculatedFieldId calculatedFieldId : request.getPreviousCalculatedFieldIds()) { + builder.addPreviousCalculatedFields(toProto(calculatedFieldId)); + } + + if (request instanceof CalculatedFieldAttributeUpdateRequest attributeUpdateRequest) { + builder.setScope(attributeUpdateRequest.getScope().name()); + } + + for (KvEntry entry : request.getKvEntries()) { + TransportProtos.TelemetryProto.Builder telemetryBuilder = TransportProtos.TelemetryProto.newBuilder(); + if (request instanceof CalculatedFieldTimeSeriesUpdateRequest) { + telemetryBuilder.setTsKv(toTsKvProto((TsKvEntry) entry)); + } + if (request instanceof CalculatedFieldAttributeUpdateRequest attrRequest) { + telemetryBuilder.setAttrKv(ProtoUtils.toAttributeKvProto((AttributeKvEntry) entry, attrRequest.getScope())); + } + builder.addUpdatedTelemetry(telemetryBuilder.build()); + } + + return builder.build(); + } + + private TransportProtos.CalculatedFieldEntityCtxIdProto toProto(CalculatedFieldEntityCtxId ctxId) { + return TransportProtos.CalculatedFieldEntityCtxIdProto.newBuilder() + .setCalculatedFieldIdMSB(ctxId.cfId().getId().getMostSignificantBits()) + .setCalculatedFieldIdLSB(ctxId.cfId().getId().getLeastSignificantBits()) + .setEntityType(ctxId.entityId().getEntityType().name()) + .setEntityIdMSB(ctxId.entityId().getId().getMostSignificantBits()) + .setEntityIdLSB(ctxId.entityId().getId().getLeastSignificantBits()) + .build(); + } + + private TransportProtos.CalculatedFieldIdProto toProto(CalculatedFieldId cfId) { + return TransportProtos.CalculatedFieldIdProto.newBuilder() + .setCalculatedFieldIdMSB(cfId.getId().getMostSignificantBits()) + .setCalculatedFieldIdLSB(cfId.getId().getLeastSignificantBits()) + .build(); + } + private KvEntry createDefaultKvEntry(Argument argument) { String key = argument.getKey(); String defaultValue = argument.getDefaultValue(); diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/CalculatedFieldEntityCtxId.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/CalculatedFieldEntityCtxId.java index f7c451efee..5fb90a3e46 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/CalculatedFieldEntityCtxId.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/CalculatedFieldEntityCtxId.java @@ -15,7 +15,8 @@ */ package org.thingsboard.server.service.cf.ctx; -import java.util.UUID; +import org.thingsboard.server.common.data.id.CalculatedFieldId; +import org.thingsboard.server.common.data.id.EntityId; -public record CalculatedFieldEntityCtxId(UUID cfId, UUID entityId) { +public record CalculatedFieldEntityCtxId(CalculatedFieldId cfId, EntityId entityId) { } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/telemetry/CalculatedFieldAttributeUpdateRequest.java b/application/src/main/java/org/thingsboard/server/service/cf/telemetry/CalculatedFieldAttributeUpdateRequest.java index 25d2f57bd6..a83cc0fc25 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/telemetry/CalculatedFieldAttributeUpdateRequest.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/telemetry/CalculatedFieldAttributeUpdateRequest.java @@ -15,6 +15,7 @@ */ package org.thingsboard.server.service.cf.telemetry; +import lombok.AllArgsConstructor; import lombok.Data; import org.thingsboard.rule.engine.api.AttributesSaveRequest; import org.thingsboard.server.common.data.AttributeScope; @@ -22,18 +23,19 @@ import org.thingsboard.server.common.data.cf.CalculatedFieldLinkConfiguration; import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.common.data.kv.AttributeKvEntry; +import org.thingsboard.server.common.data.kv.KvEntry; import java.util.List; import java.util.Map; @Data +@AllArgsConstructor public class CalculatedFieldAttributeUpdateRequest implements CalculatedFieldTelemetryUpdateRequest { private TenantId tenantId; private EntityId entityId; private AttributeScope scope; - private List kvEntries; + private List kvEntries; private List previousCalculatedFieldIds; public CalculatedFieldAttributeUpdateRequest(AttributesSaveRequest request) { diff --git a/application/src/main/java/org/thingsboard/server/service/cf/telemetry/CalculatedFieldTimeSeriesUpdateRequest.java b/application/src/main/java/org/thingsboard/server/service/cf/telemetry/CalculatedFieldTimeSeriesUpdateRequest.java index 6225286631..507daf386e 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/telemetry/CalculatedFieldTimeSeriesUpdateRequest.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/telemetry/CalculatedFieldTimeSeriesUpdateRequest.java @@ -15,23 +15,25 @@ */ package org.thingsboard.server.service.cf.telemetry; +import lombok.AllArgsConstructor; import lombok.Data; import org.thingsboard.rule.engine.api.TimeseriesSaveRequest; import org.thingsboard.server.common.data.cf.CalculatedFieldLinkConfiguration; import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.common.data.kv.TsKvEntry; +import org.thingsboard.server.common.data.kv.KvEntry; import java.util.List; import java.util.Map; @Data +@AllArgsConstructor public class CalculatedFieldTimeSeriesUpdateRequest implements CalculatedFieldTelemetryUpdateRequest { private TenantId tenantId; private EntityId entityId; - private List kvEntries; + private List kvEntries; private List previousCalculatedFieldIds; public CalculatedFieldTimeSeriesUpdateRequest(TimeseriesSaveRequest request) { diff --git a/application/src/main/java/org/thingsboard/server/service/queue/ruleengine/TbRuleEngineQueueConsumerManager.java b/application/src/main/java/org/thingsboard/server/service/queue/ruleengine/TbRuleEngineQueueConsumerManager.java index c2823d3c00..243a3adbf7 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/ruleengine/TbRuleEngineQueueConsumerManager.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/ruleengine/TbRuleEngineQueueConsumerManager.java @@ -34,6 +34,7 @@ import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; import org.thingsboard.server.queue.TbQueueConsumer; import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.queue.discovery.QueueKey; +import org.thingsboard.server.service.cf.CalculatedFieldExecutionService; import org.thingsboard.server.service.queue.TbMsgPackCallback; import org.thingsboard.server.service.queue.TbMsgPackProcessingContext; import org.thingsboard.server.service.queue.TbRuleEngineConsumerStats; @@ -63,14 +64,18 @@ public class TbRuleEngineQueueConsumerManager extends MainQueueConsumerManager entry.getValue().getEntityId().equals(entityId)) .forEach(entry -> { - Argument tergetArgument = entry.getValue(); + Argument targetArgument = entry.getValue(); String argumentKey = entry.getKey(); - switch (tergetArgument.getType()) { + switch (targetArgument.getType()) { case ATTRIBUTE -> { - switch (tergetArgument.getScope()) { + switch (targetArgument.getScope()) { case CLIENT_SCOPE -> - linkConfiguration.getClientAttributes().put(tergetArgument.getKey(), argumentKey); + linkConfiguration.getClientAttributes().put(targetArgument.getKey(), argumentKey); case SERVER_SCOPE -> - linkConfiguration.getServerAttributes().put(tergetArgument.getKey(), argumentKey); + linkConfiguration.getServerAttributes().put(targetArgument.getKey(), argumentKey); case SHARED_SCOPE -> - linkConfiguration.getSharedAttributes().put(tergetArgument.getKey(), argumentKey); + linkConfiguration.getSharedAttributes().put(targetArgument.getKey(), argumentKey); } } case TS_LATEST, TS_ROLLING -> - linkConfiguration.getTimeSeries().put(tergetArgument.getKey(), argumentKey); + linkConfiguration.getTimeSeries().put(targetArgument.getKey(), argumentKey); } }); diff --git a/common/proto/src/main/java/org/thingsboard/server/common/util/ProtoUtils.java b/common/proto/src/main/java/org/thingsboard/server/common/util/ProtoUtils.java index ec17914fd8..d332bac64f 100644 --- a/common/proto/src/main/java/org/thingsboard/server/common/util/ProtoUtils.java +++ b/common/proto/src/main/java/org/thingsboard/server/common/util/ProtoUtils.java @@ -22,6 +22,7 @@ import lombok.extern.slf4j.Slf4j; import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.server.common.data.ApiUsageState; import org.thingsboard.server.common.data.ApiUsageStateValue; +import org.thingsboard.server.common.data.AttributeScope; import org.thingsboard.server.common.data.Device; import org.thingsboard.server.common.data.DeviceProfile; import org.thingsboard.server.common.data.DeviceProfileProvisionType; @@ -58,12 +59,14 @@ import org.thingsboard.server.common.data.id.TenantProfileId; import org.thingsboard.server.common.data.kv.AttributeKey; import org.thingsboard.server.common.data.kv.AttributeKvEntry; import org.thingsboard.server.common.data.kv.BaseAttributeKvEntry; +import org.thingsboard.server.common.data.kv.BasicTsKvEntry; import org.thingsboard.server.common.data.kv.BooleanDataEntry; import org.thingsboard.server.common.data.kv.DoubleDataEntry; import org.thingsboard.server.common.data.kv.JsonDataEntry; import org.thingsboard.server.common.data.kv.KvEntry; import org.thingsboard.server.common.data.kv.LongDataEntry; import org.thingsboard.server.common.data.kv.StringDataEntry; +import org.thingsboard.server.common.data.kv.TsKvEntry; import org.thingsboard.server.common.data.plugin.ComponentLifecycleEvent; import org.thingsboard.server.common.data.rpc.RpcError; import org.thingsboard.server.common.data.rpc.ToDeviceRpcRequestBody; @@ -627,6 +630,136 @@ public class ProtoUtils { return new BaseAttributeKvEntry(entry, proto.getLastUpdateTs(), proto.hasVersion() ? proto.getVersion() : null); } + public static KvEntry fromProto(TransportProtos.TsKvProto proto) { + TransportProtos.KeyValueProto kvProto = proto.getKv(); + String key = kvProto.getKey(); + KvEntry entry = switch (kvProto.getType()) { + case BOOLEAN_V -> new BooleanDataEntry(key, kvProto.getBoolV()); + case LONG_V -> new LongDataEntry(key, kvProto.getLongV()); + case DOUBLE_V -> new DoubleDataEntry(key, kvProto.getDoubleV()); + case STRING_V -> new StringDataEntry(key, kvProto.getStringV()); + case JSON_V -> new JsonDataEntry(key, kvProto.getJsonV()); + default -> null; + }; + return new BasicTsKvEntry(proto.getTs(), entry, proto.hasVersion() ? proto.getVersion() : null); + } + + public static KvEntry fromTelemetryProto(TransportProtos.TelemetryProto telemetryProto) { + if (telemetryProto.hasAttrKv()) { + return fromProto(telemetryProto.getAttrKv().getValue()); + } else if (telemetryProto.hasTsKv()) { + return fromProto(telemetryProto.getTsKv()); + } else { + throw new IllegalArgumentException("Unsupported TelemetryProto type: " + telemetryProto); + } + } + + public static TransportProtos.AttributeKey toAttributeKeyProto(String key, AttributeScope scope) { + TransportProtos.AttributeKey.Builder builder = TransportProtos.AttributeKey.newBuilder(); + builder.setAttributeKey(key); + switch (scope) { + case CLIENT_SCOPE: + builder.setScope(TransportProtos.AttributeScopeProto.CLIENT_SCOPE); + break; + case SERVER_SCOPE: + builder.setScope(TransportProtos.AttributeScopeProto.SERVER_SCOPE); + break; + case SHARED_SCOPE: + builder.setScope(TransportProtos.AttributeScopeProto.SHARED_SCOPE); + break; + default: + throw new IllegalArgumentException("Unsupported attribute scope: " + scope); + } + return builder.build(); + } + + public static TransportProtos.AttributeKvProto toAttributeKvProto(AttributeKvEntry attributeKvEntry, AttributeScope scope) { + return TransportProtos.AttributeKvProto.newBuilder() + .setKey(ProtoUtils.toAttributeKeyProto(attributeKvEntry.getKey(), scope)) + .setValue(ProtoUtils.toAttributeValueProto(attributeKvEntry)) + .build(); + } + + public static TransportProtos.AttributeValueProto toAttributeValueProto(AttributeKvEntry attributeKvEntry) { + TransportProtos.AttributeValueProto.Builder builder = TransportProtos.AttributeValueProto.newBuilder(); + builder.setLastUpdateTs(attributeKvEntry.getLastUpdateTs()); + switch (attributeKvEntry.getDataType()) { + case BOOLEAN: + builder.setType(TransportProtos.KeyValueType.BOOLEAN_V) + .setHasV(true) + .setBoolV(attributeKvEntry.getBooleanValue().orElse(false)); + break; + case LONG: + builder.setType(TransportProtos.KeyValueType.LONG_V) + .setHasV(true) + .setLongV(attributeKvEntry.getLongValue().orElse(0L)); + break; + case DOUBLE: + builder.setType(TransportProtos.KeyValueType.DOUBLE_V) + .setHasV(true) + .setDoubleV(attributeKvEntry.getDoubleValue().orElse(0.0)); + break; + case STRING: + builder.setType(TransportProtos.KeyValueType.STRING_V) + .setHasV(true) + .setStringV(attributeKvEntry.getStrValue().orElse("")); + break; + case JSON: + builder.setType(TransportProtos.KeyValueType.JSON_V) + .setHasV(true) + .setJsonV(attributeKvEntry.getJsonValue().orElse("{}")); + break; + default: + builder.setHasV(false); + throw new IllegalArgumentException("Unsupported AttributeKvEntry data type: " + attributeKvEntry.getDataType()); + } + if (attributeKvEntry.getKey() != null) { + builder.setKey(attributeKvEntry.getKey()); + } + if (attributeKvEntry.getVersion() != null) { + builder.setVersion(attributeKvEntry.getVersion()); + } + return builder.build(); + } + + public static TransportProtos.TsKvProto toTsKvProto(TsKvEntry tsKvEntry) { + return TransportProtos.TsKvProto.newBuilder() + .setTs(tsKvEntry.getTs()) + .setKv(toKeyValueProto(tsKvEntry)) + .setVersion(tsKvEntry.getVersion()) + .build(); + } + + public static TransportProtos.KeyValueProto toKeyValueProto(KvEntry kvEntry) { + TransportProtos.KeyValueProto.Builder builder = TransportProtos.KeyValueProto.newBuilder(); + builder.setKey(kvEntry.getKey()); + switch (kvEntry.getDataType()) { + case BOOLEAN: + builder.setType(TransportProtos.KeyValueType.BOOLEAN_V) + .setBoolV(kvEntry.getBooleanValue().orElse(false)); + break; + case LONG: + builder.setType(TransportProtos.KeyValueType.LONG_V) + .setLongV(kvEntry.getLongValue().orElse(0L)); + break; + case DOUBLE: + builder.setType(TransportProtos.KeyValueType.DOUBLE_V) + .setDoubleV(kvEntry.getDoubleValue().orElse(0.0)); + break; + case STRING: + builder.setType(TransportProtos.KeyValueType.STRING_V) + .setStringV(kvEntry.getStrValue().orElse("")); + break; + case JSON: + builder.setType(TransportProtos.KeyValueType.JSON_V) + .setJsonV(kvEntry.getJsonValue().orElse("{}")); + break; + default: + throw new IllegalArgumentException("Unsupported KvEntry data type: " + kvEntry.getDataType()); + } + return builder.build(); + } + public static TransportProtos.DeviceProto toProto(Device device) { var builder = TransportProtos.DeviceProto.newBuilder() .setTenantIdMSB(device.getTenantId().getId().getMostSignificantBits()) diff --git a/common/proto/src/main/proto/queue.proto b/common/proto/src/main/proto/queue.proto index 56f347f311..685ca47719 100644 --- a/common/proto/src/main/proto/queue.proto +++ b/common/proto/src/main/proto/queue.proto @@ -183,6 +183,18 @@ message TsKvListProto { repeated KeyValueProto kv = 2; } +message AttributeKvProto { + AttributeKey key = 1; + AttributeValueProto value = 2; +} + +message TelemetryProto { + oneof proto { + AttributeKvProto attrKv = 1; + TsKvProto tsKv = 2; + } +} + message DeviceInfoProto { int64 tenantIdMSB = 1; int64 tenantIdLSB = 2; @@ -809,11 +821,21 @@ message ProfileEntityMsgProto { bool deleted = 10; } -message ToServerB { +message TelemetryUpdateMsgProto { int64 tenantIdMSB = 1; int64 tenantIdLSB = 2; - repeated CfIdEntityIdPair links = 3; - value = 4; + repeated CalculatedFieldEntityCtxIdProto links = 3; + repeated CalculatedFieldIdProto previousCalculatedFields = 4; + string scope = 5; + repeated TelemetryProto updatedTelemetry = 6; +} + +message CalculatedFieldEntityCtxIdProto { + int64 calculatedFieldIdMSB = 1; + int64 calculatedFieldIdLSB = 2; + string entityType = 3; + int64 entityIdMSB = 4; + int64 entityIdLSB = 5; } message CalculatedFieldStateMsgProto { @@ -1655,6 +1677,7 @@ message ToRuleEngineMsg { bytes tbMsg = 3; repeated string relationTypes = 4; string failureMessage = 5; + TelemetryUpdateMsgProto cfTelemetryUpdateMsg = 6; } message ToRuleEngineNotificationMsg { diff --git a/dao/src/main/java/org/thingsboard/server/dao/cf/BaseCalculatedFieldService.java b/dao/src/main/java/org/thingsboard/server/dao/cf/BaseCalculatedFieldService.java index e650aec35e..36bc3d038a 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/cf/BaseCalculatedFieldService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/cf/BaseCalculatedFieldService.java @@ -247,6 +247,7 @@ public class BaseCalculatedFieldService extends AbstractEntityService implements private List buildCalculatedFieldLinks(TenantId tenantId, CalculatedField calculatedField) { CalculatedFieldConfiguration cfConfig = calculatedField.getConfiguration(); return cfConfig.getReferencedEntities().stream() + .filter(referencedEntity -> !referencedEntity.equals(calculatedField.getEntityId())) .map(referencedEntityId -> { CalculatedFieldLink link = new CalculatedFieldLink(); link.setTenantId(tenantId); From 46180e33d70761ea18f93238420e8ed1cf97f31e Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Wed, 8 Jan 2025 17:20:20 +0200 Subject: [PATCH 3/9] cache refactoring --- .../service/cf/CalculatedFieldCache.java | 16 +- .../cf/DefaultCalculatedFieldCache.java | 132 +++++++-------- ...efaultCalculatedFieldExecutionService.java | 156 ++++++++++-------- .../processing/AbstractConsumerService.java | 4 +- .../BaseCalculatedFieldConfiguration.java | 21 +++ .../CalculatedFieldConfiguration.java | 7 + common/proto/src/main/proto/queue.proto | 11 +- .../dao/cf/BaseCalculatedFieldService.java | 18 +- .../dao/sql/cf/CalculatedFieldRepository.java | 3 +- .../dao/sql/cf/JpaCalculatedFieldDao.java | 2 +- 10 files changed, 195 insertions(+), 175 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldCache.java b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldCache.java index aa25565a34..7394c95f08 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldCache.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldCache.java @@ -28,20 +28,24 @@ import java.util.Set; public interface CalculatedFieldCache { - CalculatedField getCalculatedField(TenantId tenantId, CalculatedFieldId calculatedFieldId); + CalculatedField getCalculatedField(CalculatedFieldId calculatedFieldId); - List getCalculatedFieldsByEntityId(TenantId tenantId, EntityId entityId); + List getCalculatedFieldsByEntityId(EntityId entityId); - List getCalculatedFieldLinks(TenantId tenantId, CalculatedFieldId calculatedFieldId); + List getCalculatedFieldLinks(CalculatedFieldId calculatedFieldId); - List getCalculatedFieldLinksByEntityId(TenantId tenantId, EntityId entityId); + List getCalculatedFieldLinksByEntityId(EntityId entityId); - void updateCalculatedFieldLinks(TenantId tenantId, CalculatedFieldId calculatedFieldId); + void updateCalculatedFieldLinks(CalculatedFieldId calculatedFieldId); - CalculatedFieldCtx getCalculatedFieldCtx(TenantId tenantId, CalculatedFieldId calculatedFieldId, TbelInvokeService tbelInvokeService); + CalculatedFieldCtx getCalculatedFieldCtx(CalculatedFieldId calculatedFieldId, TbelInvokeService tbelInvokeService); Set getEntitiesByProfile(TenantId tenantId, EntityId entityId); + void addCalculatedField(TenantId tenantId, CalculatedFieldId calculatedFieldId); + + void updateCalculatedField(TenantId tenantId, CalculatedFieldId calculatedFieldId); + void evict(CalculatedFieldId calculatedFieldId); } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldCache.java b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldCache.java index 40d569b040..71fc3eff67 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldCache.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldCache.java @@ -24,6 +24,7 @@ import org.springframework.stereotype.Service; import org.thingsboard.script.api.tbel.TbelInvokeService; import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.cf.CalculatedFieldLink; +import org.thingsboard.server.common.data.cf.configuration.CalculatedFieldConfiguration; import org.thingsboard.server.common.data.id.AssetProfileId; import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.DeviceProfileId; @@ -70,98 +71,44 @@ public class DefaultCalculatedFieldCache implements CalculatedFieldCache { public void init() { PageDataIterable cfs = new PageDataIterable<>(calculatedFieldService::findAllCalculatedFields, initFetchPackSize); cfs.forEach(cf -> calculatedFields.putIfAbsent(cf.getId(), cf)); + calculatedFields.values().forEach(cf -> + entityIdCalculatedFields.computeIfAbsent(cf.getEntityId(), id -> new ArrayList<>()).add(cf) + ); PageDataIterable cfls = new PageDataIterable<>(calculatedFieldService::findAllCalculatedFieldLinks, initFetchPackSize); cfls.forEach(link -> calculatedFieldLinks.computeIfAbsent(link.getCalculatedFieldId(), id -> new ArrayList<>()).add(link)); + calculatedFieldLinks.values().stream() + .flatMap(List::stream) + .forEach(link -> + entityIdCalculatedFieldLinks.computeIfAbsent(link.getEntityId(), id -> new ArrayList<>()).add(link) + ); } @Override - public CalculatedField getCalculatedField(TenantId tenantId, CalculatedFieldId calculatedFieldId) { - CalculatedField calculatedField = calculatedFields.get(calculatedFieldId); - if (calculatedField == null) { - calculatedFieldFetchLock.lock(); - try { - calculatedField = calculatedFields.get(calculatedFieldId); - if (calculatedField == null) { - calculatedField = calculatedFieldService.findById(tenantId, calculatedFieldId); - if (calculatedField != null) { - calculatedFields.put(calculatedFieldId, calculatedField); - log.debug("[{}] Fetch calculated field into cache: {}", calculatedFieldId, calculatedField); - } - } - } finally { - calculatedFieldFetchLock.unlock(); - } - } - log.trace("[{}] Found calculated field in cache: {}", calculatedFieldId, calculatedField); - return calculatedField; + public CalculatedField getCalculatedField(CalculatedFieldId calculatedFieldId) { + return calculatedFields.get(calculatedFieldId); } @Override - public List getCalculatedFieldsByEntityId(TenantId tenantId, EntityId entityId) { - List cfs = entityIdCalculatedFields.get(entityId); - if (cfs == null) { - calculatedFieldFetchLock.lock(); - try { - cfs = entityIdCalculatedFields.get(entityId); - if (cfs == null) { - cfs = calculatedFieldService.findCalculatedFieldsByEntityId(tenantId, entityId); - entityIdCalculatedFields.put(entityId, cfs); - log.debug("[{}] Fetch calculated fields by entity into cache: {}", entityId, cfs); - } - } finally { - calculatedFieldFetchLock.unlock(); - } - } - log.trace("[{}] Found calculated fields by entity in cache: {}", entityId, cfs); - return cfs; + public List getCalculatedFieldsByEntityId(EntityId entityId) { + return entityIdCalculatedFields.getOrDefault(entityId, new ArrayList<>()); } @Override - public List getCalculatedFieldLinks(TenantId tenantId, CalculatedFieldId calculatedFieldId) { - List cfLinks = calculatedFieldLinks.get(calculatedFieldId); - if (cfLinks == null) { - calculatedFieldFetchLock.lock(); - try { - cfLinks = calculatedFieldLinks.get(calculatedFieldId); - if (cfLinks == null) { - cfLinks = calculatedFieldService.findAllCalculatedFieldLinksById(tenantId, calculatedFieldId); - calculatedFieldLinks.put(calculatedFieldId, cfLinks); - log.debug("[{}] Fetch calculated field links into cache: {}", calculatedFieldId, cfLinks); - } - } finally { - calculatedFieldFetchLock.unlock(); - } - } - log.trace("[{}] Found calculated field links in cache: {}", calculatedFieldId, cfLinks); - return cfLinks; + public List getCalculatedFieldLinks(CalculatedFieldId calculatedFieldId) { + return calculatedFieldLinks.getOrDefault(calculatedFieldId, new ArrayList<>()); } @Override - public List getCalculatedFieldLinksByEntityId(TenantId tenantId, EntityId entityId) { - List cfLinks = entityIdCalculatedFieldLinks.get(entityId); - if (cfLinks == null) { - calculatedFieldFetchLock.lock(); - try { - cfLinks = entityIdCalculatedFieldLinks.get(entityId); - if (cfLinks == null) { - cfLinks = calculatedFieldService.findAllCalculatedFieldLinksByEntityId(tenantId, entityId); - entityIdCalculatedFieldLinks.put(entityId, cfLinks); - log.debug("[{}] Fetch calculated field links by entity id into cache: {}", entityId, cfLinks); - } - } finally { - calculatedFieldFetchLock.unlock(); - } - } - log.trace("[{}] Found calculated field links by entity id in cache: {}", entityId, cfLinks); - return cfLinks; + public List getCalculatedFieldLinksByEntityId(EntityId entityId) { + return entityIdCalculatedFieldLinks.getOrDefault(entityId, new ArrayList<>()); } @Override - public void updateCalculatedFieldLinks(TenantId tenantId, CalculatedFieldId calculatedFieldId) { + public void updateCalculatedFieldLinks(CalculatedFieldId calculatedFieldId) { log.debug("Update calculated field links per entity for calculated field: [{}]", calculatedFieldId); calculatedFieldFetchLock.lock(); try { - List cfLinks = getCalculatedFieldLinks(tenantId, calculatedFieldId); + List cfLinks = getCalculatedFieldLinks(calculatedFieldId); if (cfLinks != null && !cfLinks.isEmpty()) { cfLinks.forEach(link -> { entityIdCalculatedFieldLinks.compute(link.getEntityId(), (id, existingList) -> { @@ -181,14 +128,14 @@ public class DefaultCalculatedFieldCache implements CalculatedFieldCache { } @Override - public CalculatedFieldCtx getCalculatedFieldCtx(TenantId tenantId, CalculatedFieldId calculatedFieldId, TbelInvokeService tbelInvokeService) { + public CalculatedFieldCtx getCalculatedFieldCtx(CalculatedFieldId calculatedFieldId, TbelInvokeService tbelInvokeService) { CalculatedFieldCtx ctx = calculatedFieldsCtx.get(calculatedFieldId); if (ctx == null) { calculatedFieldFetchLock.lock(); try { ctx = calculatedFieldsCtx.get(calculatedFieldId); if (ctx == null) { - CalculatedField calculatedField = getCalculatedField(tenantId, calculatedFieldId); + CalculatedField calculatedField = getCalculatedField(calculatedFieldId); if (calculatedField != null) { ctx = new CalculatedFieldCtx(calculatedField, tbelInvokeService); calculatedFieldsCtx.put(calculatedFieldId, ctx); @@ -236,6 +183,42 @@ public class DefaultCalculatedFieldCache implements CalculatedFieldCache { return entities; } + @Override + public void addCalculatedField(TenantId tenantId, CalculatedFieldId calculatedFieldId) { + calculatedFieldFetchLock.lock(); + try { + CalculatedField calculatedField = calculatedFieldService.findById(tenantId, calculatedFieldId); + EntityId cfEntityId = calculatedField.getEntityId(); + + calculatedFields.put(calculatedFieldId, calculatedField); + + entityIdCalculatedFields.computeIfAbsent(cfEntityId, entityId -> new ArrayList<>()).add(calculatedField); + + CalculatedFieldConfiguration configuration = calculatedField.getConfiguration(); + calculatedFieldLinks.put(calculatedFieldId, configuration.buildCalculatedFieldLinks(tenantId, cfEntityId, calculatedFieldId)); + + configuration.getReferencedEntities().stream() + .filter(referencedEntityId -> !referencedEntityId.equals(cfEntityId)) + .forEach(referencedEntityId -> { + entityIdCalculatedFieldLinks.computeIfAbsent(referencedEntityId, entityId -> new ArrayList<>()) + .add(configuration.buildCalculatedFieldLink(tenantId, referencedEntityId, calculatedFieldId)); + }); + } finally { + calculatedFieldFetchLock.unlock(); + } + } + + @Override + public void updateCalculatedField(TenantId tenantId, CalculatedFieldId calculatedFieldId) { + calculatedFieldFetchLock.lock(); + try { + evict(calculatedFieldId); + addCalculatedField(tenantId, calculatedFieldId); + } finally { + calculatedFieldFetchLock.unlock(); + } + } + @Override public void evict(CalculatedFieldId calculatedFieldId) { CalculatedField oldCalculatedField = calculatedFields.remove(calculatedFieldId); @@ -243,7 +226,6 @@ public class DefaultCalculatedFieldCache implements CalculatedFieldCache { calculatedFieldLinks.remove(calculatedFieldId); log.debug("[{}] evict calculated field from cached calculated fields by entity id: {}", calculatedFieldId, oldCalculatedField); entityIdCalculatedFields.forEach((entityId, calculatedFields) -> calculatedFields.removeIf(cf -> cf.getId().equals(calculatedFieldId))); - entityIdCalculatedFields.remove(oldCalculatedField.getEntityId()); log.debug("[{}] evict calculated field links from cache: {}", calculatedFieldId, oldCalculatedField); calculatedFieldsCtx.remove(calculatedFieldId); log.debug("[{}] evict calculated field ctx from cache: {}", calculatedFieldId, oldCalculatedField); diff --git a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java index 0de3136439..dd11c799c2 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java @@ -92,6 +92,7 @@ import org.thingsboard.server.service.profile.TbAssetProfileCache; import org.thingsboard.server.service.profile.TbDeviceProfileCache; import java.util.ArrayList; +import java.util.Collections; import java.util.EnumSet; import java.util.HashMap; import java.util.List; @@ -253,7 +254,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas onCalculatedFieldDelete(calculatedFieldId, callback); callback.onSuccess(); } - CalculatedField cf = calculatedFieldCache.getCalculatedField(tenantId, calculatedFieldId); + CalculatedField cf = calculatedFieldCache.getCalculatedField(calculatedFieldId); if (proto.getUpdated()) { log.info("Executing onCalculatedFieldUpdate, calculatedFieldId=[{}]", calculatedFieldId); boolean shouldReinit = onCalculatedFieldUpdate(cf, callback); @@ -263,7 +264,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas } if (cf != null) { EntityId entityId = cf.getEntityId(); - CalculatedFieldCtx calculatedFieldCtx = calculatedFieldCache.getCalculatedFieldCtx(tenantId, calculatedFieldId, tbelInvokeService); + CalculatedFieldCtx calculatedFieldCtx = calculatedFieldCache.getCalculatedFieldCtx(calculatedFieldId, tbelInvokeService); switch (entityId.getEntityType()) { case ASSET, DEVICE -> { log.info("Initializing state for entity: tenantId=[{}], entityId=[{}]", tenantId, entityId); @@ -297,7 +298,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas } private boolean onCalculatedFieldUpdate(CalculatedField updatedCalculatedField, TbCallback callback) { - CalculatedField oldCalculatedField = calculatedFieldCache.getCalculatedField(updatedCalculatedField.getTenantId(), updatedCalculatedField.getId()); + CalculatedField oldCalculatedField = calculatedFieldCache.getCalculatedField(updatedCalculatedField.getId()); boolean shouldReinit = true; if (hasSignificantChanges(oldCalculatedField, updatedCalculatedField)) { onCalculatedFieldDelete(updatedCalculatedField.getId(), callback); @@ -345,17 +346,27 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas if (supportedReferencedEntities.contains(entityId.getEntityType())) { TenantId tenantId = calculatedFieldTelemetryUpdateRequest.getTenantId(); - Map> tpiStatesToUpdate = new HashMap<>(); + TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_RULE_ENGINE, tenantId, entityId); - updateTelemetryForEntity(calculatedFieldTelemetryUpdateRequest, tpiStatesToUpdate); - updateTelemetryForProfile(calculatedFieldTelemetryUpdateRequest, getProfileId(tenantId, entityId), tpiStatesToUpdate); - updateTelemetryForLinkedEntities(calculatedFieldTelemetryUpdateRequest, tpiStatesToUpdate); + if (tpi.isMyPartition()) { - if (!tpiStatesToUpdate.isEmpty()) { - tpiStatesToUpdate.forEach((topicPartitionInfo, ctxIds) -> { - TransportProtos.TelemetryUpdateMsgProto telemetryUpdateMsgProto = buildTelemetryUpdateMsgProto(calculatedFieldTelemetryUpdateRequest, ctxIds); - clusterService.pushMsgToRuleEngine(topicPartitionInfo, UUID.randomUUID(), TransportProtos.ToRuleEngineMsg.newBuilder().setCfTelemetryUpdateMsg(telemetryUpdateMsgProto).build(), null); - }); + processCalculatedFields(calculatedFieldTelemetryUpdateRequest, entityId); + processCalculatedFields(calculatedFieldTelemetryUpdateRequest, getProfileId(tenantId, entityId)); + + Map> tpiStatesToUpdate = new HashMap<>(); + processCalculatedFieldLinks(calculatedFieldTelemetryUpdateRequest, tpiStatesToUpdate); + if (!tpiStatesToUpdate.isEmpty()) { + tpiStatesToUpdate.forEach((topicPartitionInfo, ctxIds) -> { + TransportProtos.TelemetryUpdateMsgProto telemetryUpdateMsgProto = buildTelemetryUpdateMsgProto(calculatedFieldTelemetryUpdateRequest, ctxIds); + clusterService.pushMsgToRuleEngine(topicPartitionInfo, UUID.randomUUID(), TransportProtos.ToRuleEngineMsg.newBuilder() + .setCfTelemetryUpdateMsg(telemetryUpdateMsgProto).build(), null); + }); + } + } else { + TransportProtos.TelemetryUpdateMsgProto telemetryUpdateMsgProto = buildTelemetryUpdateMsgProto(calculatedFieldTelemetryUpdateRequest); + clusterService.pushMsgToRuleEngine(tpi, UUID.randomUUID(), TransportProtos.ToRuleEngineMsg.newBuilder() + .setCfTelemetryUpdateMsg(telemetryUpdateMsgProto).build(), null); + // Forward this request to a correct server based on entity id. } } } catch (Exception e) { @@ -363,30 +374,14 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas } } - private void updateTelemetryForEntity(CalculatedFieldTelemetryUpdateRequest request, Map> tpiStates) { - updateTelemetryForEntity(request, request.getEntityId(), tpiStates); - } - - private void updateTelemetryForProfile(CalculatedFieldTelemetryUpdateRequest request, EntityId profileId, Map> tpiStates) { - updateTelemetryForEntity(request, profileId, tpiStates); - } - - private void updateTelemetryForEntity(CalculatedFieldTelemetryUpdateRequest request, EntityId targetEntity, Map> tpiStates) { + private void processCalculatedFields(CalculatedFieldTelemetryUpdateRequest request, EntityId cfTargetEntityId) { TenantId tenantId = request.getTenantId(); EntityId entityId = request.getEntityId(); - TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_RULE_ENGINE, tenantId, entityId); - if (tpi.isMyPartition()) { - if (targetEntity != null) { - calculatedFieldCache.getCalculatedFieldsByEntityId(tenantId, targetEntity).forEach(cf -> { - CalculatedFieldLinkConfiguration linkConfiguration = cf.getConfiguration().getReferencedEntityConfig(targetEntity); - mapAndProcessUpdatedTelemetry(tenantId, entityId, cf.getId(), request, linkConfiguration); - }); - } - } else { - List ctxIds = tpiStates.computeIfAbsent(tpi, k -> new ArrayList<>()); - calculatedFieldCache.getCalculatedFieldsByEntityId(tenantId, targetEntity).forEach(cf -> { - ctxIds.add(new CalculatedFieldEntityCtxId(cf.getId(), entityId)); + if (cfTargetEntityId != null) { + calculatedFieldCache.getCalculatedFieldsByEntityId(cfTargetEntityId).forEach(cf -> { + CalculatedFieldLinkConfiguration linkConfiguration = cf.getConfiguration().getReferencedEntityConfig(cfTargetEntityId); + mapAndProcessUpdatedTelemetry(tenantId, entityId, cf.getId(), request, linkConfiguration); }); } } @@ -405,14 +400,14 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas } } - private void updateTelemetryForLinkedEntities(CalculatedFieldTelemetryUpdateRequest request, Map> tpiStates) { + private void processCalculatedFieldLinks(CalculatedFieldTelemetryUpdateRequest request, Map> tpiStates) { TenantId tenantId = request.getTenantId(); EntityId entityId = request.getEntityId(); - calculatedFieldCache.getCalculatedFieldLinksByEntityId(tenantId, entityId) + calculatedFieldCache.getCalculatedFieldLinksByEntityId(entityId) .forEach(link -> { CalculatedFieldId calculatedFieldId = link.getCalculatedFieldId(); - EntityId targetEntityId = calculatedFieldCache.getCalculatedField(tenantId, calculatedFieldId).getEntityId(); + EntityId targetEntityId = calculatedFieldCache.getCalculatedField(calculatedFieldId).getEntityId(); if (isProfileEntity(targetEntityId)) { calculatedFieldCache.getEntitiesByProfile(tenantId, targetEntityId).forEach(entityByProfile -> { @@ -451,33 +446,22 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas @Override public void onTelemetryUpdateMsg(TransportProtos.TelemetryUpdateMsgProto proto) { try { - TenantId tenantId = TenantId.fromUUID(new UUID(proto.getTenantIdMSB(), proto.getTenantIdLSB())); + CalculatedFieldTelemetryUpdateRequest request = fromProto(proto); + + if (proto.getLinksList().isEmpty()) { + onTelemetryUpdate(request); + return; + } proto.getLinksList().forEach(ctxIdProto -> { - EntityId entityId = EntityIdFactory.getByTypeAndUuid( - ctxIdProto.getEntityType(), new UUID(ctxIdProto.getEntityIdMSB(), ctxIdProto.getEntityIdLSB())); - - List updatedTelemetry = proto.getUpdatedTelemetryList().stream() - .map(ProtoUtils::fromTelemetryProto) - .toList(); - - boolean attributesUpdated = StringUtils.isEmpty(proto.getScope()); - - CalculatedFieldTelemetryUpdateRequest request = attributesUpdated - ? new CalculatedFieldAttributeUpdateRequest( - tenantId, entityId, AttributeScope.valueOf(proto.getScope()), updatedTelemetry, - proto.getPreviousCalculatedFieldsList().stream() - .map(cfIdProto -> new CalculatedFieldId( - new UUID(cfIdProto.getCalculatedFieldIdMSB(), cfIdProto.getCalculatedFieldIdLSB()))) - .toList()) - : new CalculatedFieldTimeSeriesUpdateRequest( - tenantId, entityId, updatedTelemetry, - proto.getPreviousCalculatedFieldsList().stream() - .map(cfIdProto -> new CalculatedFieldId( - new UUID(cfIdProto.getCalculatedFieldIdMSB(), cfIdProto.getCalculatedFieldIdLSB()))) - .toList()); + TenantId tenantId = request.getTenantId(); + EntityId entityId = request.getEntityId(); + CalculatedFieldId calculatedFieldId = new CalculatedFieldId(new UUID(ctxIdProto.getCalculatedFieldIdMSB(), ctxIdProto.getCalculatedFieldIdLSB())); - onTelemetryUpdate(request); + CalculatedFieldLinkConfiguration linkConfiguration + = calculatedFieldCache.getCalculatedField(calculatedFieldId).getConfiguration().getReferencedEntityConfig(entityId); + + mapAndProcessUpdatedTelemetry(tenantId, entityId, calculatedFieldId, request, linkConfiguration); }); } catch (Exception e) { log.trace("Failed to process telemetry update msg: [{}]", proto, e); @@ -486,8 +470,8 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas private void executeTelemetryUpdate(TenantId tenantId, EntityId entityId, CalculatedFieldId calculatedFieldId, List previousCalculatedFieldIds, Map updatedTelemetry) { log.info("Received telemetry update msg: tenantId=[{}], entityId=[{}], calculatedFieldId=[{}]", tenantId, entityId, calculatedFieldId); - CalculatedField calculatedField = calculatedFieldCache.getCalculatedField(tenantId, calculatedFieldId); - CalculatedFieldCtx calculatedFieldCtx = calculatedFieldCache.getCalculatedFieldCtx(tenantId, calculatedFieldId, tbelInvokeService); + CalculatedField calculatedField = calculatedFieldCache.getCalculatedField(calculatedFieldId); + CalculatedFieldCtx calculatedFieldCtx = calculatedFieldCache.getCalculatedFieldCtx(calculatedFieldId, tbelInvokeService); Map argumentValues = updatedTelemetry.entrySet().stream() .collect(Collectors.toMap(Map.Entry::getKey, entry -> ArgumentEntry.createSingleValueArgument(entry.getValue()))); @@ -524,7 +508,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas Map argumentsMap = proto.getArgumentsMap().entrySet().stream() .collect(Collectors.toMap(Map.Entry::getKey, entry -> fromArgumentEntryProto(entry.getValue()))); - CalculatedFieldCtx calculatedFieldCtx = calculatedFieldCache.getCalculatedFieldCtx(tenantId, calculatedFieldId, tbelInvokeService); + CalculatedFieldCtx calculatedFieldCtx = calculatedFieldCache.getCalculatedFieldCtx(calculatedFieldId, tbelInvokeService); updateOrInitializeState(calculatedFieldCtx, entityId, argumentsMap, previousCalculatedFieldIds); } catch (Exception e) { log.trace("Failed to process calculated field update state msg: [{}]", proto, e); @@ -559,7 +543,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas if (proto.getDeleted()) { log.info("Executing profile entity deleted msg, tenantId=[{}], entityId=[{}]", tenantId, entityId); - getCalculatedFieldLinks(tenantId, entityId, profileId) + getCalculatedFieldLinks(entityId, profileId) .forEach(link -> clearState(tenantId, link.getCalculatedFieldId(), entityId)); } else { log.info("Executing profile entity added msg, tenantId=[{}], entityId=[{}]", tenantId, entityId); @@ -585,7 +569,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas private void initializeStateForEntityByProfile(TenantId tenantId, EntityId entityId, EntityId profileId, TbCallback callback) { calculatedFieldService.findCalculatedFieldIdsByEntityId(tenantId, profileId) .stream() - .map(cfId -> calculatedFieldCache.getCalculatedFieldCtx(tenantId, cfId, tbelInvokeService)) + .map(cfId -> calculatedFieldCache.getCalculatedFieldCtx(cfId, tbelInvokeService)) .forEach(cfCtx -> initializeStateForEntity(cfCtx, entityId, callback)); } @@ -722,10 +706,10 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas } } - private List getCalculatedFieldLinks(TenantId tenantId, EntityId entityId, EntityId profileId) { - List links = new ArrayList<>(calculatedFieldCache.getCalculatedFieldLinksByEntityId(tenantId, entityId)); + private List getCalculatedFieldLinks(EntityId entityId, EntityId profileId) { + List links = new ArrayList<>(calculatedFieldCache.getCalculatedFieldLinksByEntityId(entityId)); if (profileId != null) { - links.addAll(calculatedFieldCache.getCalculatedFieldLinksByEntityId(tenantId, profileId)); + links.addAll(calculatedFieldCache.getCalculatedFieldLinksByEntityId(profileId)); } return links; } @@ -870,13 +854,22 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas } } + private TransportProtos.TelemetryUpdateMsgProto buildTelemetryUpdateMsgProto(CalculatedFieldTelemetryUpdateRequest request) { + return buildTelemetryUpdateMsgProto(request, Collections.emptyList()); + } + + ; + private TransportProtos.TelemetryUpdateMsgProto buildTelemetryUpdateMsgProto( CalculatedFieldTelemetryUpdateRequest request, List links ) { TransportProtos.TelemetryUpdateMsgProto.Builder builder = TransportProtos.TelemetryUpdateMsgProto.newBuilder(); builder.setTenantIdMSB(request.getTenantId().getId().getMostSignificantBits()) - .setTenantIdLSB(request.getTenantId().getId().getLeastSignificantBits()); + .setTenantIdLSB(request.getTenantId().getId().getLeastSignificantBits()) + .setEntityType(request.getEntityId().getEntityType().name()) + .setEntityIdMSB(request.getEntityId().getId().getMostSignificantBits()) + .setEntityIdLSB(request.getEntityId().getId().getLeastSignificantBits()); for (CalculatedFieldEntityCtxId link : links) { builder.addLinks(toProto(link)); @@ -904,6 +897,31 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas return builder.build(); } + private CalculatedFieldTelemetryUpdateRequest fromProto(TransportProtos.TelemetryUpdateMsgProto proto) { + TenantId tenantId = TenantId.fromUUID(new UUID(proto.getTenantIdMSB(), proto.getTenantIdLSB())); + EntityId entityId = EntityIdFactory.getByTypeAndUuid(proto.getEntityType(), new UUID(proto.getEntityIdMSB(), proto.getEntityIdLSB())); + + List updatedTelemetry = proto.getUpdatedTelemetryList().stream() + .map(ProtoUtils::fromTelemetryProto) + .toList(); + + boolean attributesUpdated = StringUtils.isEmpty(proto.getScope()); + + return attributesUpdated + ? new CalculatedFieldAttributeUpdateRequest( + tenantId, entityId, AttributeScope.valueOf(proto.getScope()), updatedTelemetry, + proto.getPreviousCalculatedFieldsList().stream() + .map(cfIdProto -> new CalculatedFieldId( + new UUID(cfIdProto.getCalculatedFieldIdMSB(), cfIdProto.getCalculatedFieldIdLSB()))) + .toList()) + : new CalculatedFieldTimeSeriesUpdateRequest( + tenantId, entityId, updatedTelemetry, + proto.getPreviousCalculatedFieldsList().stream() + .map(cfIdProto -> new CalculatedFieldId( + new UUID(cfIdProto.getCalculatedFieldIdMSB(), cfIdProto.getCalculatedFieldIdLSB()))) + .toList()); + } + private TransportProtos.CalculatedFieldEntityCtxIdProto toProto(CalculatedFieldEntityCtxId ctxId) { return TransportProtos.CalculatedFieldEntityCtxIdProto.newBuilder() .setCalculatedFieldIdMSB(ctxId.cfId().getId().getMostSignificantBits()) diff --git a/application/src/main/java/org/thingsboard/server/service/queue/processing/AbstractConsumerService.java b/application/src/main/java/org/thingsboard/server/service/queue/processing/AbstractConsumerService.java index 2aaed13ec1..dac35bfc5c 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/processing/AbstractConsumerService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/processing/AbstractConsumerService.java @@ -194,7 +194,9 @@ public abstract class AbstractConsumerService buildCalculatedFieldLinks(TenantId tenantId, EntityId cfEntityId, CalculatedFieldId calculatedFieldId) { + return getReferencedEntities().stream() + .filter(referencedEntity -> !referencedEntity.equals(cfEntityId)) + .map(referencedEntityId -> buildCalculatedFieldLink(tenantId, referencedEntityId, calculatedFieldId)) + .collect(Collectors.toList()); + } + + @Override + public CalculatedFieldLink buildCalculatedFieldLink(TenantId tenantId, EntityId referencedEntityId, CalculatedFieldId calculatedFieldId) { + CalculatedFieldLink link = new CalculatedFieldLink(); + link.setTenantId(tenantId); + link.setEntityId(referencedEntityId); + link.setCalculatedFieldId(calculatedFieldId); + link.setConfiguration(getReferencedEntityConfig(referencedEntityId)); + return link; + } + @Override public JsonNode calculatedFieldConfigToJson(EntityType entityType, UUID entityId) { ObjectNode configNode = mapper.createObjectNode(); diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/CalculatedFieldConfiguration.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/CalculatedFieldConfiguration.java index 5c428bd628..ac94ade134 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/CalculatedFieldConfiguration.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/CalculatedFieldConfiguration.java @@ -20,9 +20,12 @@ import com.fasterxml.jackson.annotation.JsonSubTypes; import com.fasterxml.jackson.annotation.JsonTypeInfo; import com.fasterxml.jackson.databind.JsonNode; import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.cf.CalculatedFieldLink; import org.thingsboard.server.common.data.cf.CalculatedFieldLinkConfiguration; import org.thingsboard.server.common.data.cf.CalculatedFieldType; +import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.TenantId; import java.util.List; import java.util.Map; @@ -57,4 +60,8 @@ public interface CalculatedFieldConfiguration { @JsonIgnore JsonNode calculatedFieldConfigToJson(EntityType entityType, UUID entityId); + List buildCalculatedFieldLinks(TenantId tenantId, EntityId cfEntityId, CalculatedFieldId calculatedFieldId); + + CalculatedFieldLink buildCalculatedFieldLink(TenantId tenantId, EntityId referencedEntityId, CalculatedFieldId calculatedFieldId); + } diff --git a/common/proto/src/main/proto/queue.proto b/common/proto/src/main/proto/queue.proto index 685ca47719..8a7c2d8c03 100644 --- a/common/proto/src/main/proto/queue.proto +++ b/common/proto/src/main/proto/queue.proto @@ -824,10 +824,13 @@ message ProfileEntityMsgProto { message TelemetryUpdateMsgProto { int64 tenantIdMSB = 1; int64 tenantIdLSB = 2; - repeated CalculatedFieldEntityCtxIdProto links = 3; - repeated CalculatedFieldIdProto previousCalculatedFields = 4; - string scope = 5; - repeated TelemetryProto updatedTelemetry = 6; + string entityType = 3; + int64 entityIdMSB = 4; + int64 entityIdLSB = 5; + repeated CalculatedFieldEntityCtxIdProto links = 6; + repeated CalculatedFieldIdProto previousCalculatedFields = 7; + string scope = 8; + repeated TelemetryProto updatedTelemetry = 9; } message CalculatedFieldEntityCtxIdProto { diff --git a/dao/src/main/java/org/thingsboard/server/dao/cf/BaseCalculatedFieldService.java b/dao/src/main/java/org/thingsboard/server/dao/cf/BaseCalculatedFieldService.java index 36bc3d038a..9c81d91f64 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/cf/BaseCalculatedFieldService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/cf/BaseCalculatedFieldService.java @@ -38,7 +38,6 @@ import org.thingsboard.server.dao.service.DataValidator; import java.util.List; import java.util.Optional; -import java.util.stream.Collectors; import static org.thingsboard.server.dao.service.Validator.validateId; import static org.thingsboard.server.dao.service.Validator.validatePageLink; @@ -240,23 +239,8 @@ public class BaseCalculatedFieldService extends AbstractEntityService implements } private void createOrUpdateCalculatedFieldLink(TenantId tenantId, CalculatedField calculatedField) { - List links = buildCalculatedFieldLinks(tenantId, calculatedField); + List links = calculatedField.getConfiguration().buildCalculatedFieldLinks(tenantId, calculatedField.getEntityId(), calculatedField.getId()); links.forEach(link -> saveCalculatedFieldLink(tenantId, link)); } - private List buildCalculatedFieldLinks(TenantId tenantId, CalculatedField calculatedField) { - CalculatedFieldConfiguration cfConfig = calculatedField.getConfiguration(); - return cfConfig.getReferencedEntities().stream() - .filter(referencedEntity -> !referencedEntity.equals(calculatedField.getEntityId())) - .map(referencedEntityId -> { - CalculatedFieldLink link = new CalculatedFieldLink(); - link.setTenantId(tenantId); - link.setEntityId(referencedEntityId); - link.setCalculatedFieldId(calculatedField.getId()); - link.setConfiguration(cfConfig.getReferencedEntityConfig(referencedEntityId)); - return link; - }) - .collect(Collectors.toList()); - } - } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/cf/CalculatedFieldRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/cf/CalculatedFieldRepository.java index 816fa1546c..bed6f2d3a2 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/cf/CalculatedFieldRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/cf/CalculatedFieldRepository.java @@ -16,7 +16,6 @@ package org.thingsboard.server.dao.sql.cf; import org.springframework.data.jpa.repository.JpaRepository; -import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.dao.model.sql.CalculatedFieldEntity; @@ -29,7 +28,7 @@ public interface CalculatedFieldRepository extends JpaRepository findCalculatedFieldIdsByTenantIdAndEntityId(UUID tenantId, UUID entityId); - List findAllByTenantIdAndEntityId(UUID tenantId, UUID entityId); + List findAllByTenantIdAndEntityId(UUID tenantId, UUID entityId); List findAllByTenantId(UUID tenantId); diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/cf/JpaCalculatedFieldDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/cf/JpaCalculatedFieldDao.java index 20081299e8..cdcffdd440 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/cf/JpaCalculatedFieldDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/cf/JpaCalculatedFieldDao.java @@ -57,7 +57,7 @@ public class JpaCalculatedFieldDao extends JpaAbstractDao findCalculatedFieldsByEntityId(TenantId tenantId, EntityId entityId) { - return calculatedFieldRepository.findAllByTenantIdAndEntityId(tenantId.getId(), entityId.getId()); + return DaoUtil.convertDataList(calculatedFieldRepository.findAllByTenantIdAndEntityId(tenantId.getId(), entityId.getId())); } @Override From 5203ef7422f32a0219a64470298e774c8b683ab8 Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Thu, 9 Jan 2025 14:45:42 +0200 Subject: [PATCH 4/9] added calculated field state service --- .../service/cf/CalculatedFieldCache.java | 2 - .../cf/CalculatedFieldExecutionService.java | 2 - .../cf/DefaultCalculatedFieldCache.java | 51 +-- ...efaultCalculatedFieldExecutionService.java | 325 ++++++------------ .../server/service/cf/RocksDBService.java | 13 - .../queue/DefaultTbCoreConsumerService.java | 14 - .../server/common/util/ProtoUtils.java | 40 --- common/proto/src/main/proto/queue.proto | 43 +-- 8 files changed, 119 insertions(+), 371 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldCache.java b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldCache.java index 7394c95f08..bf5dc8d42f 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldCache.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldCache.java @@ -36,8 +36,6 @@ public interface CalculatedFieldCache { List getCalculatedFieldLinksByEntityId(EntityId entityId); - void updateCalculatedFieldLinks(CalculatedFieldId calculatedFieldId); - CalculatedFieldCtx getCalculatedFieldCtx(CalculatedFieldId calculatedFieldId, TbelInvokeService tbelInvokeService); Set getEntitiesByProfile(TenantId tenantId, EntityId entityId); diff --git a/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldExecutionService.java b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldExecutionService.java index 6d1d459b9b..8ba1f6dfed 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldExecutionService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldExecutionService.java @@ -27,8 +27,6 @@ public interface CalculatedFieldExecutionService { void onTelemetryUpdateMsg(TransportProtos.TelemetryUpdateMsgProto proto); - void onCalculatedFieldStateMsg(TransportProtos.CalculatedFieldStateMsgProto proto, TbCallback callback); - void onEntityProfileChangedMsg(TransportProtos.EntityProfileUpdateMsgProto proto, TbCallback callback); void onProfileEntityMsg(TransportProtos.ProfileEntityMsgProto proto, TbCallback callback); diff --git a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldCache.java b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldCache.java index 71fc3eff67..f762ae3530 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldCache.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldCache.java @@ -36,12 +36,12 @@ import org.thingsboard.server.dao.cf.CalculatedFieldService; import org.thingsboard.server.dao.device.DeviceService; import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldCtx; -import java.util.ArrayList; import java.util.HashSet; import java.util.List; import java.util.Set; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.CopyOnWriteArrayList; import java.util.concurrent.locks.Lock; import java.util.concurrent.locks.ReentrantLock; @@ -72,14 +72,14 @@ public class DefaultCalculatedFieldCache implements CalculatedFieldCache { PageDataIterable cfs = new PageDataIterable<>(calculatedFieldService::findAllCalculatedFields, initFetchPackSize); cfs.forEach(cf -> calculatedFields.putIfAbsent(cf.getId(), cf)); calculatedFields.values().forEach(cf -> - entityIdCalculatedFields.computeIfAbsent(cf.getEntityId(), id -> new ArrayList<>()).add(cf) + entityIdCalculatedFields.computeIfAbsent(cf.getEntityId(), id -> new CopyOnWriteArrayList<>()).add(cf) ); PageDataIterable cfls = new PageDataIterable<>(calculatedFieldService::findAllCalculatedFieldLinks, initFetchPackSize); - cfls.forEach(link -> calculatedFieldLinks.computeIfAbsent(link.getCalculatedFieldId(), id -> new ArrayList<>()).add(link)); + cfls.forEach(link -> calculatedFieldLinks.computeIfAbsent(link.getCalculatedFieldId(), id -> new CopyOnWriteArrayList<>()).add(link)); calculatedFieldLinks.values().stream() .flatMap(List::stream) .forEach(link -> - entityIdCalculatedFieldLinks.computeIfAbsent(link.getEntityId(), id -> new ArrayList<>()).add(link) + entityIdCalculatedFieldLinks.computeIfAbsent(link.getEntityId(), id -> new CopyOnWriteArrayList<>()).add(link) ); } @@ -90,41 +90,17 @@ public class DefaultCalculatedFieldCache implements CalculatedFieldCache { @Override public List getCalculatedFieldsByEntityId(EntityId entityId) { - return entityIdCalculatedFields.getOrDefault(entityId, new ArrayList<>()); + return entityIdCalculatedFields.getOrDefault(entityId, new CopyOnWriteArrayList<>()); } @Override public List getCalculatedFieldLinks(CalculatedFieldId calculatedFieldId) { - return calculatedFieldLinks.getOrDefault(calculatedFieldId, new ArrayList<>()); + return calculatedFieldLinks.getOrDefault(calculatedFieldId, new CopyOnWriteArrayList<>()); } @Override public List getCalculatedFieldLinksByEntityId(EntityId entityId) { - return entityIdCalculatedFieldLinks.getOrDefault(entityId, new ArrayList<>()); - } - - @Override - public void updateCalculatedFieldLinks(CalculatedFieldId calculatedFieldId) { - log.debug("Update calculated field links per entity for calculated field: [{}]", calculatedFieldId); - calculatedFieldFetchLock.lock(); - try { - List cfLinks = getCalculatedFieldLinks(calculatedFieldId); - if (cfLinks != null && !cfLinks.isEmpty()) { - cfLinks.forEach(link -> { - entityIdCalculatedFieldLinks.compute(link.getEntityId(), (id, existingList) -> { - if (existingList == null) { - existingList = new ArrayList<>(); - } else if (!(existingList instanceof ArrayList)) { - existingList = new ArrayList<>(existingList); - } - existingList.add(link); - return existingList; - }); - }); - } - } finally { - calculatedFieldFetchLock.unlock(); - } + return entityIdCalculatedFieldLinks.getOrDefault(entityId, new CopyOnWriteArrayList<>()); } @Override @@ -192,7 +168,7 @@ public class DefaultCalculatedFieldCache implements CalculatedFieldCache { calculatedFields.put(calculatedFieldId, calculatedField); - entityIdCalculatedFields.computeIfAbsent(cfEntityId, entityId -> new ArrayList<>()).add(calculatedField); + entityIdCalculatedFields.computeIfAbsent(cfEntityId, entityId -> new CopyOnWriteArrayList<>()).add(calculatedField); CalculatedFieldConfiguration configuration = calculatedField.getConfiguration(); calculatedFieldLinks.put(calculatedFieldId, configuration.buildCalculatedFieldLinks(tenantId, cfEntityId, calculatedFieldId)); @@ -200,7 +176,7 @@ public class DefaultCalculatedFieldCache implements CalculatedFieldCache { configuration.getReferencedEntities().stream() .filter(referencedEntityId -> !referencedEntityId.equals(cfEntityId)) .forEach(referencedEntityId -> { - entityIdCalculatedFieldLinks.computeIfAbsent(referencedEntityId, entityId -> new ArrayList<>()) + entityIdCalculatedFieldLinks.computeIfAbsent(referencedEntityId, entityId -> new CopyOnWriteArrayList<>()) .add(configuration.buildCalculatedFieldLink(tenantId, referencedEntityId, calculatedFieldId)); }); } finally { @@ -210,13 +186,8 @@ public class DefaultCalculatedFieldCache implements CalculatedFieldCache { @Override public void updateCalculatedField(TenantId tenantId, CalculatedFieldId calculatedFieldId) { - calculatedFieldFetchLock.lock(); - try { - evict(calculatedFieldId); - addCalculatedField(tenantId, calculatedFieldId); - } finally { - calculatedFieldFetchLock.unlock(); - } + evict(calculatedFieldId); + addCalculatedField(tenantId, calculatedFieldId); } @Override diff --git a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java index dd11c799c2..e54c1bd5c9 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java @@ -77,6 +77,7 @@ import org.thingsboard.server.dao.timeseries.TimeseriesService; import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.service.cf.ctx.CalculatedFieldEntityCtx; import org.thingsboard.server.service.cf.ctx.CalculatedFieldEntityCtxId; +import org.thingsboard.server.service.cf.ctx.CalculatedFieldStateService; import org.thingsboard.server.service.cf.ctx.state.ArgumentEntry; import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldCtx; import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldState; @@ -107,8 +108,6 @@ import java.util.function.Consumer; import java.util.stream.Collectors; import static org.thingsboard.server.common.data.DataConstants.SCOPE; -import static org.thingsboard.server.common.util.ProtoUtils.fromObjectProto; -import static org.thingsboard.server.common.util.ProtoUtils.toObjectProto; import static org.thingsboard.server.common.util.ProtoUtils.toTsKvProto; @Service @@ -122,7 +121,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas private final CalculatedFieldCache calculatedFieldCache; private final AttributesService attributesService; private final TimeseriesService timeseriesService; - private final RocksDBService rocksDBService; + private final CalculatedFieldStateService stateService; private final TbClusterService clusterService; private final TbelInvokeService tbelInvokeService; @@ -148,8 +147,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas Math.max(4, Runtime.getRuntime().availableProcessors()), "calculated-field")); calculatedFieldCallbackExecutor = MoreExecutors.listeningDecorator(ThingsBoardExecutors.newWorkStealingPool( Math.max(4, Runtime.getRuntime().availableProcessors()), "calculated-field-callback")); - scheduledExecutor.submit(() -> rocksDBService.getAll() - .forEach((ctxId, ctx) -> states.put(JacksonUtil.fromString(ctxId, CalculatedFieldEntityCtxId.class), JacksonUtil.fromString(ctx, CalculatedFieldEntityCtx.class)))); + scheduledExecutor.submit(() -> states.putAll(stateService.restoreStates())); } @PreDestroy @@ -223,10 +221,9 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas private void restoreState(CalculatedFieldId calculatedFieldId, EntityId entityId) { CalculatedFieldEntityCtxId ctxId = new CalculatedFieldEntityCtxId(calculatedFieldId, entityId); - String storedState = rocksDBService.get(JacksonUtil.writeValueAsString(ctxId)); + CalculatedFieldEntityCtx restoredCtx = stateService.restoreState(ctxId); - if (storedState != null) { - CalculatedFieldEntityCtx restoredCtx = JacksonUtil.fromString(storedState, CalculatedFieldEntityCtx.class); + if (restoredCtx != null) { states.put(ctxId, restoredCtx); log.info("Restored state for CalculatedField [{}]", calculatedFieldId); } else { @@ -251,18 +248,21 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas log.info("Received CalculatedFieldMsgProto for processing: tenantId=[{}], calculatedFieldId=[{}]", tenantId, calculatedFieldId); if (proto.getDeleted()) { log.warn("Executing onCalculatedFieldDelete, calculatedFieldId=[{}]", calculatedFieldId); + calculatedFieldCache.evict(calculatedFieldId); onCalculatedFieldDelete(calculatedFieldId, callback); callback.onSuccess(); } - CalculatedField cf = calculatedFieldCache.getCalculatedField(calculatedFieldId); + CalculatedField cf = calculatedFieldService.findById(tenantId, calculatedFieldId); if (proto.getUpdated()) { log.info("Executing onCalculatedFieldUpdate, calculatedFieldId=[{}]", calculatedFieldId); + calculatedFieldCache.updateCalculatedField(tenantId, calculatedFieldId); boolean shouldReinit = onCalculatedFieldUpdate(cf, callback); if (!shouldReinit) { return; } } if (cf != null) { + calculatedFieldCache.addCalculatedField(tenantId, calculatedFieldId); EntityId entityId = cf.getEntityId(); CalculatedFieldCtx calculatedFieldCtx = calculatedFieldCache.getCalculatedFieldCtx(calculatedFieldId, tbelInvokeService); switch (entityId.getEntityType()) { @@ -312,12 +312,13 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas private void onCalculatedFieldDelete(CalculatedFieldId calculatedFieldId, TbCallback callback) { try { cleanupEntity(calculatedFieldId); - states.keySet().removeIf(ctxId -> ctxId.cfId().equals(calculatedFieldId)); - List statesToRemove = states.keySet().stream() - .filter(ctxId -> ctxId.cfId().equals(calculatedFieldId)) - .map(JacksonUtil::writeValueAsString) - .toList(); - rocksDBService.deleteAll(statesToRemove); + states.keySet().removeIf(ctxId -> { + if (ctxId.cfId().equals(calculatedFieldId)) { + stateService.removeState(ctxId); + return true; + } + return false; + }); } catch (Exception e) { log.trace("Failed to delete calculated field: [{}]", calculatedFieldId, e); callback.onFailure(e); @@ -366,7 +367,6 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas TransportProtos.TelemetryUpdateMsgProto telemetryUpdateMsgProto = buildTelemetryUpdateMsgProto(calculatedFieldTelemetryUpdateRequest); clusterService.pushMsgToRuleEngine(tpi, UUID.randomUUID(), TransportProtos.ToRuleEngineMsg.newBuilder() .setCfTelemetryUpdateMsg(telemetryUpdateMsgProto).build(), null); - // Forward this request to a correct server based on entity id. } } } catch (Exception e) { @@ -386,20 +386,6 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas } } - private void updateTelemetryForLinkedEntity(CalculatedFieldTelemetryUpdateRequest request, EntityId targetEntity, CalculatedFieldLink link, Map> tpiStates) { - TenantId tenantId = request.getTenantId(); - EntityId entityId = request.getEntityId(); - CalculatedFieldId calculatedFieldId = link.getCalculatedFieldId(); - - TopicPartitionInfo targetEntityTpi = partitionService.resolve(ServiceType.TB_RULE_ENGINE, tenantId, targetEntity); - if (targetEntityTpi.isMyPartition()) { - mapAndProcessUpdatedTelemetry(tenantId, entityId, calculatedFieldId, request, link.getConfiguration()); - } else { - List ctxIds = tpiStates.computeIfAbsent(targetEntityTpi, k -> new ArrayList<>()); - ctxIds.add(new CalculatedFieldEntityCtxId(calculatedFieldId, targetEntity)); - } - } - private void processCalculatedFieldLinks(CalculatedFieldTelemetryUpdateRequest request, Map> tpiStates) { TenantId tenantId = request.getTenantId(); EntityId entityId = request.getEntityId(); @@ -411,14 +397,28 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas if (isProfileEntity(targetEntityId)) { calculatedFieldCache.getEntitiesByProfile(tenantId, targetEntityId).forEach(entityByProfile -> { - updateTelemetryForLinkedEntity(request, entityByProfile, link, tpiStates); + processCalculatedFieldLink(request, entityByProfile, link, tpiStates); }); } else { - updateTelemetryForLinkedEntity(request, targetEntityId, link, tpiStates); + processCalculatedFieldLink(request, targetEntityId, link, tpiStates); } }); } + private void processCalculatedFieldLink(CalculatedFieldTelemetryUpdateRequest request, EntityId targetEntity, CalculatedFieldLink link, Map> tpiStates) { + TenantId tenantId = request.getTenantId(); + EntityId entityId = request.getEntityId(); + CalculatedFieldId calculatedFieldId = link.getCalculatedFieldId(); + + TopicPartitionInfo targetEntityTpi = partitionService.resolve(ServiceType.TB_RULE_ENGINE, tenantId, targetEntity); + if (targetEntityTpi.isMyPartition()) { + mapAndProcessUpdatedTelemetry(tenantId, entityId, calculatedFieldId, request, link.getConfiguration()); + } else { + List ctxIds = tpiStates.computeIfAbsent(targetEntityTpi, k -> new ArrayList<>()); + ctxIds.add(new CalculatedFieldEntityCtxId(calculatedFieldId, targetEntity)); + } + } + private void mapAndProcessUpdatedTelemetry(TenantId tenantId, EntityId entityId, CalculatedFieldId calculatedFieldId, @@ -490,31 +490,6 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas } } - @Override - public void onCalculatedFieldStateMsg(TransportProtos.CalculatedFieldStateMsgProto proto, TbCallback callback) { - try { - TenantId tenantId = TenantId.fromUUID(new UUID(proto.getTenantIdMSB(), proto.getTenantIdLSB())); - CalculatedFieldId calculatedFieldId = new CalculatedFieldId(new UUID(proto.getCalculatedFieldIdMSB(), proto.getCalculatedFieldIdLSB())); - EntityId entityId = EntityIdFactory.getByTypeAndUuid(proto.getEntityType(), new UUID(proto.getEntityIdMSB(), proto.getEntityIdLSB())); - log.info("Received CalculatedFieldStateMsgProto for processing: tenantId=[{}], calculatedFieldId=[{}], entityId=[{}]", tenantId, calculatedFieldId, entityId); - if (proto.getClear()) { - clearState(tenantId, calculatedFieldId, entityId); - return; - } - - List previousCalculatedFieldIds = proto.getPreviousCalculatedFieldsList().stream() - .map(cfIdProto -> new CalculatedFieldId(new UUID(cfIdProto.getCalculatedFieldIdMSB(), cfIdProto.getCalculatedFieldIdLSB()))) - .collect(Collectors.toCollection(ArrayList::new)); - Map argumentsMap = proto.getArgumentsMap().entrySet().stream() - .collect(Collectors.toMap(Map.Entry::getKey, entry -> fromArgumentEntryProto(entry.getValue()))); - - CalculatedFieldCtx calculatedFieldCtx = calculatedFieldCache.getCalculatedFieldCtx(calculatedFieldId, tbelInvokeService); - updateOrInitializeState(calculatedFieldCtx, entityId, argumentsMap, previousCalculatedFieldIds); - } catch (Exception e) { - log.trace("Failed to process calculated field update state msg: [{}]", proto, e); - } - } - @Override public void onEntityProfileChangedMsg(TransportProtos.EntityProfileUpdateMsgProto proto, TbCallback callback) { try { @@ -522,12 +497,15 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas EntityId entityId = EntityIdFactory.getByTypeAndUuid(proto.getEntityType(), new UUID(proto.getEntityIdMSB(), proto.getEntityIdLSB())); EntityId oldProfileId = EntityIdFactory.getByTypeAndUuid(proto.getEntityProfileType(), new UUID(proto.getOldProfileIdMSB(), proto.getOldProfileIdLSB())); EntityId newProfileId = EntityIdFactory.getByTypeAndUuid(proto.getEntityProfileType(), new UUID(proto.getNewProfileIdMSB(), proto.getNewProfileIdLSB())); - log.info("Received EntityProfileUpdateMsgProto for processing: tenantId=[{}], entityId=[{}]", tenantId, entityId); - - calculatedFieldService.findCalculatedFieldIdsByEntityId(tenantId, oldProfileId) - .forEach(cfId -> clearState(tenantId, cfId, entityId)); - initializeStateForEntityByProfile(tenantId, entityId, newProfileId, callback); + TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_RULE_ENGINE, tenantId, entityId); + if (tpi.isMyPartition()) { + log.info("Received EntityProfileUpdateMsgProto for processing: tenantId=[{}], entityId=[{}]", tenantId, entityId); + calculatedFieldCache.getCalculatedFieldsByEntityId(oldProfileId).forEach(cf -> clearState(cf.getId(), entityId)); + initializeStateForEntityByProfile(entityId, newProfileId, callback); + } else { + clusterService.pushMsgToRuleEngine(tpi, UUID.randomUUID(), TransportProtos.ToRuleEngineMsg.newBuilder().setEntityProfileUpdateMsg(proto).build(), null); + } } catch (Exception e) { log.trace("Failed to process entity type update msg: [{}]", proto, e); } @@ -539,37 +517,39 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas TenantId tenantId = TenantId.fromUUID(new UUID(proto.getTenantIdMSB(), proto.getTenantIdLSB())); EntityId entityId = EntityIdFactory.getByTypeAndUuid(proto.getEntityType(), new UUID(proto.getEntityIdMSB(), proto.getEntityIdLSB())); EntityId profileId = EntityIdFactory.getByTypeAndUuid(proto.getEntityProfileType(), new UUID(proto.getProfileIdMSB(), proto.getProfileIdLSB())); - log.info("Received ProfileEntityMsgProto for processing: tenantId=[{}], entityId=[{}]", tenantId, entityId); - if (proto.getDeleted()) { - log.info("Executing profile entity deleted msg, tenantId=[{}], entityId=[{}]", tenantId, entityId); - getCalculatedFieldLinks(entityId, profileId) - .forEach(link -> clearState(tenantId, link.getCalculatedFieldId(), entityId)); + TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_RULE_ENGINE, tenantId, entityId); + if (tpi.isMyPartition()) { + log.info("Received ProfileEntityMsgProto for processing: tenantId=[{}], entityId=[{}]", tenantId, entityId); + if (proto.getDeleted()) { + log.info("Executing profile entity deleted msg, tenantId=[{}], entityId=[{}]", tenantId, entityId); + + calculatedFieldCache.getCalculatedFieldsByEntityId(entityId).forEach(cf -> clearState(cf.getId(), entityId)); + calculatedFieldCache.getCalculatedFieldsByEntityId(profileId).forEach(cf -> clearState(cf.getId(), entityId)); + } else { + log.info("Executing profile entity added msg, tenantId=[{}], entityId=[{}]", tenantId, entityId); + initializeStateForEntityByProfile(entityId, profileId, callback); + } } else { - log.info("Executing profile entity added msg, tenantId=[{}], entityId=[{}]", tenantId, entityId); - initializeStateForEntityByProfile(tenantId, entityId, profileId, callback); + clusterService.pushMsgToRuleEngine(tpi, UUID.randomUUID(), TransportProtos.ToRuleEngineMsg.newBuilder().setProfileEntityMsg(proto).build(), null); } + + } catch (Exception e) { log.trace("Failed to process profile entity msg: [{}]", proto, e); } } - private void clearState(TenantId tenantId, CalculatedFieldId calculatedFieldId, EntityId entityId) { - TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_RULE_ENGINE, tenantId, entityId); - if (tpi.isMyPartition()) { - log.warn("Executing clearState, calculatedFieldId=[{}], entityId=[{}]", calculatedFieldId, entityId); - CalculatedFieldEntityCtxId ctxId = new CalculatedFieldEntityCtxId(calculatedFieldId, entityId); - states.remove(ctxId); - rocksDBService.delete(JacksonUtil.writeValueAsString(ctxId)); - } else { - sendClearCalculatedFieldStateMsg(tenantId, calculatedFieldId, entityId); - } + private void clearState(CalculatedFieldId calculatedFieldId, EntityId entityId) { + log.warn("Executing clearState, calculatedFieldId=[{}], entityId=[{}]", calculatedFieldId, entityId); + CalculatedFieldEntityCtxId ctxId = new CalculatedFieldEntityCtxId(calculatedFieldId, entityId); + states.remove(ctxId); + stateService.removeState(ctxId); } - private void initializeStateForEntityByProfile(TenantId tenantId, EntityId entityId, EntityId profileId, TbCallback callback) { - calculatedFieldService.findCalculatedFieldIdsByEntityId(tenantId, profileId) - .stream() - .map(cfId -> calculatedFieldCache.getCalculatedFieldCtx(cfId, tbelInvokeService)) + private void initializeStateForEntityByProfile(EntityId entityId, EntityId profileId, TbCallback callback) { + calculatedFieldCache.getCalculatedFieldsByEntityId(profileId).stream() + .map(cf -> calculatedFieldCache.getCalculatedFieldCtx(cf.getId(), tbelInvokeService)) .forEach(cfCtx -> initializeStateForEntity(cfCtx, entityId, callback)); } @@ -607,65 +587,58 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas } private void updateOrInitializeState(CalculatedFieldCtx calculatedFieldCtx, EntityId entityId, Map argumentValues, List previousCalculatedFieldIds) { - TenantId tenantId = calculatedFieldCtx.getTenantId(); CalculatedFieldId cfId = calculatedFieldCtx.getCfId(); Map argumentsMap = new HashMap<>(argumentValues); - TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_RULE_ENGINE, tenantId, entityId); - if (tpi.isMyPartition()) { - - CalculatedFieldEntityCtxId entityCtxId = new CalculatedFieldEntityCtxId(cfId, entityId); + CalculatedFieldEntityCtxId entityCtxId = new CalculatedFieldEntityCtxId(cfId, entityId); - states.compute(entityCtxId, (ctxId, ctx) -> { - CalculatedFieldEntityCtx calculatedFieldEntityCtx = ctx != null ? ctx : fetchCalculatedFieldEntityState(ctxId, calculatedFieldCtx.getCfType()); + states.compute(entityCtxId, (ctxId, ctx) -> { + CalculatedFieldEntityCtx calculatedFieldEntityCtx = ctx != null ? ctx : fetchCalculatedFieldEntityState(ctxId, calculatedFieldCtx.getCfType()); - CompletableFuture updateFuture = new CompletableFuture<>(); + CompletableFuture updateFuture = new CompletableFuture<>(); - Consumer performUpdateState = (state) -> { - if (state.updateState(argumentsMap)) { - calculatedFieldEntityCtx.setState(state); - rocksDBService.put(JacksonUtil.writeValueAsString(entityCtxId), JacksonUtil.writeValueAsString(calculatedFieldEntityCtx)); - Map arguments = state.getArguments(); - boolean allArgsPresent = arguments.keySet().containsAll(calculatedFieldCtx.getArguments().keySet()) && - !arguments.containsValue(SingleValueArgumentEntry.EMPTY) && !arguments.containsValue(TsRollingArgumentEntry.EMPTY); - if (allArgsPresent) { - performCalculation(calculatedFieldCtx, state, entityId, previousCalculatedFieldIds); - } - log.info("Successfully updated state: calculatedFieldId=[{}], entityId=[{}]", calculatedFieldCtx.getCfId(), entityId); + Consumer performUpdateState = (state) -> { + if (state.updateState(argumentsMap)) { + calculatedFieldEntityCtx.setState(state); + stateService.persistState(entityCtxId, calculatedFieldEntityCtx); + Map arguments = state.getArguments(); + boolean allArgsPresent = arguments.keySet().containsAll(calculatedFieldCtx.getArguments().keySet()) && + !arguments.containsValue(SingleValueArgumentEntry.EMPTY) && !arguments.containsValue(TsRollingArgumentEntry.EMPTY); + if (allArgsPresent) { + performCalculation(calculatedFieldCtx, state, entityId, previousCalculatedFieldIds); } - updateFuture.complete(null); - }; + log.info("Successfully updated state: calculatedFieldId=[{}], entityId=[{}]", calculatedFieldCtx.getCfId(), entityId); + } + updateFuture.complete(null); + }; - CalculatedFieldState state = calculatedFieldEntityCtx.getState(); + CalculatedFieldState state = calculatedFieldEntityCtx.getState(); - boolean allKeysPresent = argumentsMap.keySet().containsAll(calculatedFieldCtx.getArguments().keySet()); - boolean requiresTsRollingUpdate = calculatedFieldCtx.getArguments().values().stream() - .anyMatch(argument -> ArgumentType.TS_ROLLING.equals(argument.getType()) && state.getArguments().get(argument.getKey()) == null); + boolean allKeysPresent = argumentsMap.keySet().containsAll(calculatedFieldCtx.getArguments().keySet()); + boolean requiresTsRollingUpdate = calculatedFieldCtx.getArguments().values().stream() + .anyMatch(argument -> ArgumentType.TS_ROLLING.equals(argument.getType()) && state.getArguments().get(argument.getKey()) == null); - if (!allKeysPresent || requiresTsRollingUpdate) { - Map missingArguments = calculatedFieldCtx.getArguments().entrySet().stream() - .filter(entry -> !argumentsMap.containsKey(entry.getKey()) || (ArgumentType.TS_ROLLING.equals(entry.getValue().getType()) && state.getArguments().get(entry.getKey()) == null)) - .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); + if (!allKeysPresent || requiresTsRollingUpdate) { + Map missingArguments = calculatedFieldCtx.getArguments().entrySet().stream() + .filter(entry -> !argumentsMap.containsKey(entry.getKey()) || (ArgumentType.TS_ROLLING.equals(entry.getValue().getType()) && state.getArguments().get(entry.getKey()) == null)) + .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); - fetchArguments(calculatedFieldCtx.getTenantId(), entityId, missingArguments, argumentsMap::putAll) - .addListener(() -> performUpdateState.accept(state), - calculatedFieldCallbackExecutor); - } else { - performUpdateState.accept(state); - } + fetchArguments(calculatedFieldCtx.getTenantId(), entityId, missingArguments, argumentsMap::putAll) + .addListener(() -> performUpdateState.accept(state), + calculatedFieldCallbackExecutor); + } else { + performUpdateState.accept(state); + } - try { - updateFuture.join(); - } catch (Exception e) { - log.trace("Failed to update state for ctxId [{}].", ctxId, e); - throw new RuntimeException("Failed to update or initialize state.", e); - } + try { + updateFuture.join(); + } catch (Exception e) { + log.trace("Failed to update state for ctxId [{}].", ctxId, e); + throw new RuntimeException("Failed to update or initialize state.", e); + } - return calculatedFieldEntityCtx; - }); - } else { - sendUpdateCalculatedFieldStateMsg(tenantId, cfId, entityId, previousCalculatedFieldIds, argumentsMap); - } + return calculatedFieldEntityCtx; + }); } private void performCalculation(CalculatedFieldCtx calculatedFieldCtx, CalculatedFieldState state, EntityId entityId, List previousCalculatedFieldIds) { @@ -706,14 +679,6 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas } } - private List getCalculatedFieldLinks(EntityId entityId, EntityId profileId) { - List links = new ArrayList<>(calculatedFieldCache.getCalculatedFieldLinksByEntityId(entityId)); - if (profileId != null) { - links.addAll(calculatedFieldCache.getCalculatedFieldLinksByEntityId(profileId)); - } - return links; - } - private ListenableFuture fetchArguments(TenantId tenantId, EntityId entityId, Map necessaryArguments, Consumer> onComplete) { Map argumentValues = new HashMap<>(); List> futures = new ArrayList<>(); @@ -777,89 +742,10 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas return Futures.transform(tsRollingFuture, tsRolling -> tsRolling == null ? TsRollingArgumentEntry.EMPTY : ArgumentEntry.createTsRollingArgument(tsRolling), calculatedFieldCallbackExecutor); } - private void sendUpdateCalculatedFieldStateMsg(TenantId tenantId, CalculatedFieldId calculatedFieldId, EntityId entityId, List previousCalculatedFieldIds, Map argumentValues) { - TransportProtos.CalculatedFieldStateMsgProto.Builder msgBuilder = createBaseCalculatedFieldStateMsg(tenantId, calculatedFieldId, entityId); - if (argumentValues != null) { - argumentValues.forEach((key, argumentEntry) -> msgBuilder.putArguments(key, toArgumentEntryProto(argumentEntry))); - } - if (previousCalculatedFieldIds != null) { - previousCalculatedFieldIds.forEach(cfId -> msgBuilder.addPreviousCalculatedFields( - TransportProtos.CalculatedFieldIdProto.newBuilder() - .setCalculatedFieldIdMSB(cfId.getId().getMostSignificantBits()) - .setCalculatedFieldIdLSB(cfId.getId().getLeastSignificantBits()) - .build() - )); - } - - log.info("Sending calculated field state msg from entityId [{}]", entityId); - clusterService.pushMsgToCore(tenantId, calculatedFieldId, TransportProtos.ToCoreMsg.newBuilder().setCalculatedFieldStateMsg(msgBuilder).build(), null); - } - - private void sendClearCalculatedFieldStateMsg(TenantId tenantId, CalculatedFieldId calculatedFieldId, EntityId entityId) { - TransportProtos.CalculatedFieldStateMsgProto msg = createBaseCalculatedFieldStateMsg(tenantId, calculatedFieldId, entityId) - .setClear(true) - .build(); - - clusterService.pushMsgToCore(tenantId, calculatedFieldId, TransportProtos.ToCoreMsg.newBuilder().setCalculatedFieldStateMsg(msg).build(), null); - } - - private TransportProtos.CalculatedFieldStateMsgProto.Builder createBaseCalculatedFieldStateMsg( - TenantId tenantId, - CalculatedFieldId calculatedFieldId, - EntityId entityId - ) { - return TransportProtos.CalculatedFieldStateMsgProto.newBuilder() - .setTenantIdMSB(tenantId.getId().getMostSignificantBits()) - .setTenantIdLSB(tenantId.getId().getLeastSignificantBits()) - .setCalculatedFieldIdMSB(calculatedFieldId.getId().getMostSignificantBits()) - .setCalculatedFieldIdLSB(calculatedFieldId.getId().getLeastSignificantBits()) - .setEntityType(entityId.getEntityType().name()) - .setEntityIdMSB(entityId.getId().getMostSignificantBits()) - .setEntityIdLSB(entityId.getId().getLeastSignificantBits()); - } - - private TransportProtos.ArgumentEntryProto toArgumentEntryProto(ArgumentEntry argumentEntry) { - TransportProtos.ArgumentEntryProto.Builder argumentProtoBuilder = TransportProtos.ArgumentEntryProto.newBuilder(); - - if (argumentEntry instanceof TsRollingArgumentEntry tsRollingArgumentEntry) { - TransportProtos.TsRollingProto.Builder tsRollingProtoBuilder = TransportProtos.TsRollingProto.newBuilder(); - tsRollingArgumentEntry.getTsRecords().forEach((ts, value) -> - tsRollingProtoBuilder.putTsRecords(ts, toObjectProto(value)) - ); - argumentProtoBuilder.setTsRecords(tsRollingProtoBuilder.build()); - } else if (argumentEntry instanceof SingleValueArgumentEntry singleValueArgumentEntry) { - argumentProtoBuilder.setSingleValue( - TransportProtos.SingleValueProto.newBuilder() - .setTs(singleValueArgumentEntry.getTs()) - .setValue(toObjectProto(singleValueArgumentEntry.getValue())) - .build() - ); - } - - return argumentProtoBuilder.build(); - } - - private ArgumentEntry fromArgumentEntryProto(TransportProtos.ArgumentEntryProto entryProto) { - if (entryProto.hasTsRecords()) { - TsRollingArgumentEntry tsRollingArgumentEntry = new TsRollingArgumentEntry(); - entryProto.getTsRecords().getTsRecordsMap().forEach((ts, objectProto) -> - tsRollingArgumentEntry.getTsRecords().put(ts, fromObjectProto(objectProto)) - ); - return tsRollingArgumentEntry; - } else if (entryProto.hasSingleValue()) { - TransportProtos.SingleValueProto singleValueProto = entryProto.getSingleValue(); - return new SingleValueArgumentEntry(singleValueProto.getTs(), fromObjectProto(singleValueProto.getValue()), singleValueProto.getVersion()); - } else { - throw new IllegalArgumentException("Unsupported ArgumentEntryProto type"); - } - } - private TransportProtos.TelemetryUpdateMsgProto buildTelemetryUpdateMsgProto(CalculatedFieldTelemetryUpdateRequest request) { return buildTelemetryUpdateMsgProto(request, Collections.emptyList()); } - ; - private TransportProtos.TelemetryUpdateMsgProto buildTelemetryUpdateMsgProto( CalculatedFieldTelemetryUpdateRequest request, List links ) { @@ -952,11 +838,12 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas } private CalculatedFieldEntityCtx fetchCalculatedFieldEntityState(CalculatedFieldEntityCtxId entityCtxId, CalculatedFieldType cfType) { - String stateStr = rocksDBService.get(JacksonUtil.writeValueAsString(entityCtxId)); - if (stateStr == null) { + CalculatedFieldEntityCtx state = stateService.restoreState(entityCtxId); + + if (state == null) { return new CalculatedFieldEntityCtx(entityCtxId, createStateByType(cfType)); } - return JacksonUtil.fromString(stateStr, CalculatedFieldEntityCtx.class); + return state; } private ObjectNode createJsonPayload(CalculatedFieldResult calculatedFieldResult) { diff --git a/application/src/main/java/org/thingsboard/server/service/cf/RocksDBService.java b/application/src/main/java/org/thingsboard/server/service/cf/RocksDBService.java index d6b2980042..3aed65eced 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/RocksDBService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/RocksDBService.java @@ -19,7 +19,6 @@ import lombok.extern.slf4j.Slf4j; import org.rocksdb.RocksDB; import org.rocksdb.RocksDBException; import org.rocksdb.RocksIterator; -import org.rocksdb.WriteBatch; import org.rocksdb.WriteOptions; import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; import org.springframework.stereotype.Service; @@ -27,7 +26,6 @@ import org.thingsboard.server.utils.RocksDBConfig; import java.nio.charset.StandardCharsets; import java.util.HashMap; -import java.util.List; import java.util.Map; @Service @@ -59,17 +57,6 @@ public class RocksDBService { } } - public void deleteAll(List keys) { - try (WriteBatch batch = new WriteBatch()) { - for (String key : keys) { - batch.delete(key.getBytes(StandardCharsets.UTF_8)); - } - db.write(writeOptions, batch); - } catch (RocksDBException e) { - log.error("Failed to delete data from RocksDB", e); - } - } - public String get(String key) { try { byte[] value = db.get(key.getBytes(StandardCharsets.UTF_8)); diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java index 6baa75b3ef..c598540ff2 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java @@ -326,8 +326,6 @@ public class DefaultTbCoreConsumerService extends AbstractConsumerService future = calculatedFieldsExecutor.submit(() -> calculatedFieldExecutionService.onCalculatedFieldStateMsg(calculatedFieldStateMsgProto, callback)); - DonAsynchron.withCallback(future, - __ -> callback.onSuccess(), - t -> { - log.warn("[{}] Failed to process calculated field state message for entityId [{}]", tenantId.getId(), calculatedFieldId.getId(), t); - callback.onFailure(t); - }); - } - private void forwardToNotificationSchedulerService(TransportProtos.NotificationSchedulerServiceMsg msg, TbCallback callback) { TenantId tenantId = toTenantId(msg.getTenantIdMSB(), msg.getTenantIdLSB()); NotificationRequestId notificationRequestId = new NotificationRequestId(new UUID(msg.getRequestIdMSB(), msg.getRequestIdLSB())); diff --git a/common/proto/src/main/java/org/thingsboard/server/common/util/ProtoUtils.java b/common/proto/src/main/java/org/thingsboard/server/common/util/ProtoUtils.java index d332bac64f..073f47d59b 100644 --- a/common/proto/src/main/java/org/thingsboard/server/common/util/ProtoUtils.java +++ b/common/proto/src/main/java/org/thingsboard/server/common/util/ProtoUtils.java @@ -1316,46 +1316,6 @@ public class ProtoUtils { return builder.build(); } - public static TransportProtos.ObjectProto toObjectProto(Object value) { - if (value == null) { - throw new IllegalArgumentException("Cannot convert null to ObjectProto"); - } - - TransportProtos.ObjectProto.Builder builder = TransportProtos.ObjectProto.newBuilder(); - - if (value instanceof String) { - builder.setStringValue((String) value); - } else if (value instanceof Integer) { - builder.setIntValue((Integer) value); - } else if (value instanceof Long) { - builder.setLongValue((Long) value); - } else if (value instanceof Double) { - builder.setDoubleValue((Double) value); - } else if (value instanceof Boolean) { - builder.setBoolValue((Boolean) value); - } else { - throw new IllegalArgumentException("Unsupported value type: " + value.getClass().getName()); - } - - return builder.build(); - } - - public static Object fromObjectProto(TransportProtos.ObjectProto proto) { - try { - return switch (proto.getValueCase()) { - case STRINGVALUE -> proto.getStringValue(); - case INTVALUE -> proto.getIntValue(); - case LONGVALUE -> proto.getLongValue(); - case DOUBLEVALUE -> proto.getDoubleValue(); - case BOOLVALUE -> proto.getBoolValue(); - case VALUE_NOT_SET -> throw new IllegalArgumentException("Value not set in ObjectProto"); - }; - } catch (Exception e) { - log.error("Failed to deserialize ObjectProto: [{}]", proto, e); - return null; - } - } - private static boolean isNotNull(Object obj) { return obj != null; } diff --git a/common/proto/src/main/proto/queue.proto b/common/proto/src/main/proto/queue.proto index 8a7c2d8c03..1036d5ba67 100644 --- a/common/proto/src/main/proto/queue.proto +++ b/common/proto/src/main/proto/queue.proto @@ -841,51 +841,11 @@ message CalculatedFieldEntityCtxIdProto { int64 entityIdLSB = 5; } -message CalculatedFieldStateMsgProto { - int64 tenantIdMSB = 1; - int64 tenantIdLSB = 2; - int64 calculatedFieldIdMSB = 3; - int64 calculatedFieldIdLSB = 4; - string entityType = 5; - int64 entityIdMSB = 6; - int64 entityIdLSB = 7; - bool clear = 8; - repeated CalculatedFieldIdProto previousCalculatedFields = 9; - map arguments = 10; -} - message CalculatedFieldIdProto { int64 calculatedFieldIdMSB = 1; int64 calculatedFieldIdLSB = 2; } -message ArgumentEntryProto { - oneof entry_type { - TsRollingProto tsRecords = 1; - SingleValueProto singleValue = 2; - } -} - -message TsRollingProto { - map tsRecords = 1; -} - -message SingleValueProto { - int64 ts = 1; - ObjectProto value = 2; - int64 version = 3; -} - -message ObjectProto { - oneof value { - string stringValue = 1; - int32 intValue = 2; - int64 longValue = 3; - double doubleValue = 4; - bool boolValue = 5; - } -} - //Used to report session state to tb-Service and persist this state in the cache on the tb-Service level. message SubscriptionInfoProto { int64 lastActivityTime = 1; @@ -1632,7 +1592,6 @@ message ToCoreMsg { CalculatedFieldMsgProto calculatedFieldMsg = 53; EntityProfileUpdateMsgProto entityProfileUpdateMsg = 54; ProfileEntityMsgProto profileEntityMsg = 55; - CalculatedFieldStateMsgProto calculatedFieldStateMsg = 56; } /* High priority messages with low latency are handled by ThingsBoard Core Service separately */ @@ -1681,6 +1640,8 @@ message ToRuleEngineMsg { repeated string relationTypes = 4; string failureMessage = 5; TelemetryUpdateMsgProto cfTelemetryUpdateMsg = 6; + EntityProfileUpdateMsgProto entityProfileUpdateMsg = 7; + ProfileEntityMsgProto profileEntityMsg = 8; } message ToRuleEngineNotificationMsg { From 77e99d15df2c2a3f412c48fce87151d2a473be7f Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Thu, 9 Jan 2025 14:51:17 +0200 Subject: [PATCH 5/9] added service files --- .../cf/ctx/CalculatedFieldStateService.java | 30 +++++++++ .../cf/ctx/state/RocksDBStateService.java | 64 +++++++++++++++++++ 2 files changed, 94 insertions(+) create mode 100644 application/src/main/java/org/thingsboard/server/service/cf/ctx/CalculatedFieldStateService.java create mode 100644 application/src/main/java/org/thingsboard/server/service/cf/ctx/state/RocksDBStateService.java diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/CalculatedFieldStateService.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/CalculatedFieldStateService.java new file mode 100644 index 0000000000..8bc5756f4e --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/CalculatedFieldStateService.java @@ -0,0 +1,30 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.cf.ctx; + +import java.util.Map; + +public interface CalculatedFieldStateService { + + Map restoreStates(); + + CalculatedFieldEntityCtx restoreState(CalculatedFieldEntityCtxId ctxId); + + void persistState(CalculatedFieldEntityCtxId ctxId, CalculatedFieldEntityCtx state); + + void removeState(CalculatedFieldEntityCtxId ctxId); + +} diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/RocksDBStateService.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/RocksDBStateService.java new file mode 100644 index 0000000000..db8950804c --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/RocksDBStateService.java @@ -0,0 +1,64 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.cf.ctx.state; + +import lombok.RequiredArgsConstructor; +import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; +import org.springframework.stereotype.Service; +import org.thingsboard.common.util.JacksonUtil; +import org.thingsboard.server.service.cf.RocksDBService; +import org.thingsboard.server.service.cf.ctx.CalculatedFieldEntityCtx; +import org.thingsboard.server.service.cf.ctx.CalculatedFieldEntityCtxId; +import org.thingsboard.server.service.cf.ctx.CalculatedFieldStateService; + +import java.util.Map; +import java.util.Optional; +import java.util.stream.Collectors; + +@Service +@RequiredArgsConstructor +@ConditionalOnExpression("'${service.type:null}'=='monolith'") +public class RocksDBStateService implements CalculatedFieldStateService { + + private final RocksDBService rocksDBService; + + @Override + public Map restoreStates() { + return rocksDBService.getAll().entrySet().stream() + .collect(Collectors.toMap( + entry -> JacksonUtil.fromString(entry.getKey(), CalculatedFieldEntityCtxId.class), + entry -> JacksonUtil.fromString(entry.getValue(), CalculatedFieldEntityCtx.class) + )); + } + + @Override + public CalculatedFieldEntityCtx restoreState(CalculatedFieldEntityCtxId ctxId) { + return Optional.ofNullable(rocksDBService.get(JacksonUtil.writeValueAsString(ctxId))) + .map(storedState -> JacksonUtil.fromString(storedState, CalculatedFieldEntityCtx.class)) + .orElse(null); + } + + @Override + public void persistState(CalculatedFieldEntityCtxId ctxId, CalculatedFieldEntityCtx state) { + rocksDBService.put(JacksonUtil.writeValueAsString(ctxId), JacksonUtil.writeValueAsString(state)); + } + + @Override + public void removeState(CalculatedFieldEntityCtxId ctxId) { + rocksDBService.delete(JacksonUtil.writeValueAsString(ctxId)); + } + +} From 6611f017c7eae1efb05b7bcf0bdf09d17b13ad34 Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Fri, 10 Jan 2025 16:58:26 +0200 Subject: [PATCH 6/9] changed Argument structure --- .../service/cf/CalculatedFieldCache.java | 2 + .../cf/DefaultCalculatedFieldCache.java | 8 ++ ...efaultCalculatedFieldExecutionService.java | 125 ++++++------------ .../cf/ctx/state/CalculatedFieldCtx.java | 15 ++- .../ctx/state/ScriptCalculatedFieldState.java | 2 +- ...CalculatedFieldAttributeUpdateRequest.java | 25 ++++ ...CalculatedFieldTelemetryUpdateRequest.java | 3 + ...alculatedFieldTimeSeriesUpdateRequest.java | 31 +++++ .../TbRuleEngineQueueConsumerManager.java | 5 + .../CalculatedFieldControllerTest.java | 2 +- .../data/cf/configuration/Argument.java | 9 +- .../BaseCalculatedFieldConfiguration.java | 49 +++---- .../cf/configuration/ReferencedEntityKey.java | 30 +++++ .../dao/model/sql/CalculatedFieldEntity.java | 3 + .../server/dao/service/AssetServiceTest.java | 2 +- .../service/CalculatedFieldServiceTest.java | 2 +- .../dao/service/CustomerServiceTest.java | 2 +- .../server/dao/service/DeviceServiceTest.java | 2 +- 18 files changed, 195 insertions(+), 122 deletions(-) create mode 100644 common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/ReferencedEntityKey.java diff --git a/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldCache.java b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldCache.java index bf5dc8d42f..8730aeeedf 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldCache.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldCache.java @@ -38,6 +38,8 @@ public interface CalculatedFieldCache { CalculatedFieldCtx getCalculatedFieldCtx(CalculatedFieldId calculatedFieldId, TbelInvokeService tbelInvokeService); + List getCalculatedFieldCtxsByEntityId(EntityId entityId, TbelInvokeService tbelInvokeService); + Set getEntitiesByProfile(TenantId tenantId, EntityId entityId); void addCalculatedField(TenantId tenantId, CalculatedFieldId calculatedFieldId); diff --git a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldCache.java b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldCache.java index f762ae3530..868001d0d2 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldCache.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldCache.java @@ -61,6 +61,7 @@ public class DefaultCalculatedFieldCache implements CalculatedFieldCache { private final ConcurrentMap> calculatedFieldLinks = new ConcurrentHashMap<>(); private final ConcurrentMap> entityIdCalculatedFieldLinks = new ConcurrentHashMap<>(); private final ConcurrentMap calculatedFieldsCtx = new ConcurrentHashMap<>(); + private final ConcurrentMap> entityIdCalculatedFieldCtxs = new ConcurrentHashMap<>(); private final ConcurrentMap> profileEntities = new ConcurrentHashMap<>(); @Value("${calculatedField.initFetchPackSize:50000}") @@ -126,6 +127,13 @@ public class DefaultCalculatedFieldCache implements CalculatedFieldCache { return ctx; } + @Override + public List getCalculatedFieldCtxsByEntityId(EntityId entityId, TbelInvokeService tbelInvokeService) { + return getCalculatedFieldsByEntityId(entityId).stream() + .map(cf -> getCalculatedFieldCtx(cf.getId(), tbelInvokeService)) + .toList(); + } + @Override public Set getEntitiesByProfile(TenantId tenantId, EntityId entityProfileId) { Set entities = profileEntities.get(entityProfileId); diff --git a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java index e54c1bd5c9..7c17d54649 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java @@ -39,8 +39,6 @@ import org.thingsboard.server.common.data.AttributeScope; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.StringUtils; import org.thingsboard.server.common.data.cf.CalculatedField; -import org.thingsboard.server.common.data.cf.CalculatedFieldLink; -import org.thingsboard.server.common.data.cf.CalculatedFieldLinkConfiguration; import org.thingsboard.server.common.data.cf.CalculatedFieldType; import org.thingsboard.server.common.data.cf.configuration.Argument; import org.thingsboard.server.common.data.cf.configuration.ArgumentType; @@ -273,7 +271,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas case ASSET_PROFILE, DEVICE_PROFILE -> { log.info("Initializing state for all entities in profile: tenantId=[{}], profileId=[{}]", tenantId, entityId); Map commonArguments = calculatedFieldCtx.getArguments().entrySet().stream() - .filter(entry -> !isProfileEntity(entry.getValue().getEntityId())) + .filter(entry -> !isProfileEntity(entry.getValue().getRefEntityId())) .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); fetchArguments(tenantId, entityId, commonArguments, commonArgs -> { calculatedFieldCache.getEntitiesByProfile(tenantId, entityId).forEach(targetEntityId -> { @@ -341,30 +339,30 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas } @Override - public void onTelemetryUpdate(CalculatedFieldTelemetryUpdateRequest calculatedFieldTelemetryUpdateRequest) { + public void onTelemetryUpdate(CalculatedFieldTelemetryUpdateRequest request) { try { - EntityId entityId = calculatedFieldTelemetryUpdateRequest.getEntityId(); + EntityId entityId = request.getEntityId(); if (supportedReferencedEntities.contains(entityId.getEntityType())) { - TenantId tenantId = calculatedFieldTelemetryUpdateRequest.getTenantId(); + TenantId tenantId = request.getTenantId(); TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_RULE_ENGINE, tenantId, entityId); if (tpi.isMyPartition()) { - processCalculatedFields(calculatedFieldTelemetryUpdateRequest, entityId); - processCalculatedFields(calculatedFieldTelemetryUpdateRequest, getProfileId(tenantId, entityId)); + processCalculatedFields(request, entityId); + processCalculatedFields(request, getProfileId(tenantId, entityId)); Map> tpiStatesToUpdate = new HashMap<>(); - processCalculatedFieldLinks(calculatedFieldTelemetryUpdateRequest, tpiStatesToUpdate); + processCalculatedFieldLinks(request, tpiStatesToUpdate); if (!tpiStatesToUpdate.isEmpty()) { tpiStatesToUpdate.forEach((topicPartitionInfo, ctxIds) -> { - TransportProtos.TelemetryUpdateMsgProto telemetryUpdateMsgProto = buildTelemetryUpdateMsgProto(calculatedFieldTelemetryUpdateRequest, ctxIds); + TransportProtos.TelemetryUpdateMsgProto telemetryUpdateMsgProto = buildTelemetryUpdateMsgProto(request, ctxIds); clusterService.pushMsgToRuleEngine(topicPartitionInfo, UUID.randomUUID(), TransportProtos.ToRuleEngineMsg.newBuilder() .setCfTelemetryUpdateMsg(telemetryUpdateMsgProto).build(), null); }); } } else { - TransportProtos.TelemetryUpdateMsgProto telemetryUpdateMsgProto = buildTelemetryUpdateMsgProto(calculatedFieldTelemetryUpdateRequest); + TransportProtos.TelemetryUpdateMsgProto telemetryUpdateMsgProto = buildTelemetryUpdateMsgProto(request); clusterService.pushMsgToRuleEngine(tpi, UUID.randomUUID(), TransportProtos.ToRuleEngineMsg.newBuilder() .setCfTelemetryUpdateMsg(telemetryUpdateMsgProto).build(), null); } @@ -375,13 +373,12 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas } private void processCalculatedFields(CalculatedFieldTelemetryUpdateRequest request, EntityId cfTargetEntityId) { - TenantId tenantId = request.getTenantId(); - EntityId entityId = request.getEntityId(); - if (cfTargetEntityId != null) { - calculatedFieldCache.getCalculatedFieldsByEntityId(cfTargetEntityId).forEach(cf -> { - CalculatedFieldLinkConfiguration linkConfiguration = cf.getConfiguration().getReferencedEntityConfig(cfTargetEntityId); - mapAndProcessUpdatedTelemetry(tenantId, entityId, cf.getId(), request, linkConfiguration); + calculatedFieldCache.getCalculatedFieldCtxsByEntityId(cfTargetEntityId, tbelInvokeService).forEach(ctx -> { + Map updatedTelemetry = request.getMappedTelemetry(ctx); + if (!updatedTelemetry.isEmpty()) { + executeTelemetryUpdate(ctx, request.getEntityId(), request.getPreviousCalculatedFieldIds(), updatedTelemetry); + } }); } } @@ -393,56 +390,32 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas calculatedFieldCache.getCalculatedFieldLinksByEntityId(entityId) .forEach(link -> { CalculatedFieldId calculatedFieldId = link.getCalculatedFieldId(); - EntityId targetEntityId = calculatedFieldCache.getCalculatedField(calculatedFieldId).getEntityId(); + CalculatedFieldCtx ctx = calculatedFieldCache.getCalculatedFieldCtx(calculatedFieldId, tbelInvokeService); + EntityId targetEntityId = ctx.getEntityId(); if (isProfileEntity(targetEntityId)) { calculatedFieldCache.getEntitiesByProfile(tenantId, targetEntityId).forEach(entityByProfile -> { - processCalculatedFieldLink(request, entityByProfile, link, tpiStates); + processCalculatedFieldLink(request, entityByProfile, ctx, tpiStates); }); } else { - processCalculatedFieldLink(request, targetEntityId, link, tpiStates); + processCalculatedFieldLink(request, targetEntityId, ctx, tpiStates); } }); } - private void processCalculatedFieldLink(CalculatedFieldTelemetryUpdateRequest request, EntityId targetEntity, CalculatedFieldLink link, Map> tpiStates) { - TenantId tenantId = request.getTenantId(); - EntityId entityId = request.getEntityId(); - CalculatedFieldId calculatedFieldId = link.getCalculatedFieldId(); - - TopicPartitionInfo targetEntityTpi = partitionService.resolve(ServiceType.TB_RULE_ENGINE, tenantId, targetEntity); + private void processCalculatedFieldLink(CalculatedFieldTelemetryUpdateRequest request, EntityId targetEntity, CalculatedFieldCtx ctx, Map> tpiStates) { + TopicPartitionInfo targetEntityTpi = partitionService.resolve(ServiceType.TB_RULE_ENGINE, request.getTenantId(), targetEntity); if (targetEntityTpi.isMyPartition()) { - mapAndProcessUpdatedTelemetry(tenantId, entityId, calculatedFieldId, request, link.getConfiguration()); + Map updatedTelemetry = request.getMappedTelemetry(ctx); + if (!updatedTelemetry.isEmpty()) { + executeTelemetryUpdate(ctx, request.getEntityId(), request.getPreviousCalculatedFieldIds(), updatedTelemetry); + } } else { List ctxIds = tpiStates.computeIfAbsent(targetEntityTpi, k -> new ArrayList<>()); - ctxIds.add(new CalculatedFieldEntityCtxId(calculatedFieldId, targetEntity)); - } - } - - private void mapAndProcessUpdatedTelemetry(TenantId tenantId, - EntityId entityId, - CalculatedFieldId calculatedFieldId, - CalculatedFieldTelemetryUpdateRequest request, - CalculatedFieldLinkConfiguration linkConfiguration) { - Map telemetryKeys = request.getTelemetryKeysFromLink(linkConfiguration); - Map updatedTelemetry = mapTelemetryKeys(telemetryKeys, request.getKvEntries()); - - if (!updatedTelemetry.isEmpty()) { - List previousCalculatedFieldIds = request.getPreviousCalculatedFieldIds(); - executeTelemetryUpdate(tenantId, entityId, calculatedFieldId, previousCalculatedFieldIds, updatedTelemetry); + ctxIds.add(new CalculatedFieldEntityCtxId(ctx.getCfId(), targetEntity)); } } - private Map mapTelemetryKeys(Map telemetryKeys, List kvEntries) { - return kvEntries.stream() - .filter(entry -> telemetryKeys.containsKey(entry.getKey())) - .collect(Collectors.toMap( - entry -> telemetryKeys.getOrDefault(entry.getKey(), entry.getKey()), - entry -> entry, - (v1, v2) -> v1 - )); - } - @Override public void onTelemetryUpdateMsg(TransportProtos.TelemetryUpdateMsgProto proto) { try { @@ -454,40 +427,26 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas } proto.getLinksList().forEach(ctxIdProto -> { - TenantId tenantId = request.getTenantId(); EntityId entityId = request.getEntityId(); CalculatedFieldId calculatedFieldId = new CalculatedFieldId(new UUID(ctxIdProto.getCalculatedFieldIdMSB(), ctxIdProto.getCalculatedFieldIdLSB())); + CalculatedFieldCtx ctx = calculatedFieldCache.getCalculatedFieldCtx(calculatedFieldId, tbelInvokeService); - CalculatedFieldLinkConfiguration linkConfiguration - = calculatedFieldCache.getCalculatedField(calculatedFieldId).getConfiguration().getReferencedEntityConfig(entityId); - - mapAndProcessUpdatedTelemetry(tenantId, entityId, calculatedFieldId, request, linkConfiguration); + Map updatedTelemetry = request.getMappedTelemetry(ctx); + if (!updatedTelemetry.isEmpty()) { + executeTelemetryUpdate(ctx, entityId, request.getPreviousCalculatedFieldIds(), updatedTelemetry); + } }); } catch (Exception e) { log.trace("Failed to process telemetry update msg: [{}]", proto, e); } } - private void executeTelemetryUpdate(TenantId tenantId, EntityId entityId, CalculatedFieldId calculatedFieldId, List previousCalculatedFieldIds, Map updatedTelemetry) { - log.info("Received telemetry update msg: tenantId=[{}], entityId=[{}], calculatedFieldId=[{}]", tenantId, entityId, calculatedFieldId); - CalculatedField calculatedField = calculatedFieldCache.getCalculatedField(calculatedFieldId); - CalculatedFieldCtx calculatedFieldCtx = calculatedFieldCache.getCalculatedFieldCtx(calculatedFieldId, tbelInvokeService); + private void executeTelemetryUpdate(CalculatedFieldCtx cfCtx, EntityId entityId, List previousCalculatedFieldIds, Map updatedTelemetry) { + log.info("Received telemetry update msg: tenantId=[{}], entityId=[{}], calculatedFieldId=[{}]", cfCtx.getTenantId(), entityId, cfCtx.getCfId()); Map argumentValues = updatedTelemetry.entrySet().stream() .collect(Collectors.toMap(Map.Entry::getKey, entry -> ArgumentEntry.createSingleValueArgument(entry.getValue()))); - EntityId cfEntityId = calculatedField.getEntityId(); - switch (cfEntityId.getEntityType()) { - case ASSET_PROFILE, DEVICE_PROFILE -> { - boolean isCommonEntity = calculatedField.getConfiguration().getReferencedEntities().contains(entityId); - if (isCommonEntity) { - calculatedFieldCache.getEntitiesByProfile(tenantId, cfEntityId).forEach(id -> updateOrInitializeState(calculatedFieldCtx, id, argumentValues, previousCalculatedFieldIds)); - } else { - updateOrInitializeState(calculatedFieldCtx, entityId, argumentValues, previousCalculatedFieldIds); - } - } - default -> - updateOrInitializeState(calculatedFieldCtx, cfEntityId, argumentValues, previousCalculatedFieldIds); - } + updateOrInitializeState(cfCtx, entityId, argumentValues, previousCalculatedFieldIds); } @Override @@ -533,8 +492,6 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas } else { clusterService.pushMsgToRuleEngine(tpi, UUID.randomUUID(), TransportProtos.ToRuleEngineMsg.newBuilder().setProfileEntityMsg(proto).build(), null); } - - } catch (Exception e) { log.trace("Failed to process profile entity msg: [{}]", proto, e); } @@ -616,11 +573,11 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas boolean allKeysPresent = argumentsMap.keySet().containsAll(calculatedFieldCtx.getArguments().keySet()); boolean requiresTsRollingUpdate = calculatedFieldCtx.getArguments().values().stream() - .anyMatch(argument -> ArgumentType.TS_ROLLING.equals(argument.getType()) && state.getArguments().get(argument.getKey()) == null); + .anyMatch(argument -> ArgumentType.TS_ROLLING.equals(argument.getRefEntityKey().getType()) && state.getArguments().get(argument.getRefEntityKey().getKey()) == null); if (!allKeysPresent || requiresTsRollingUpdate) { Map missingArguments = calculatedFieldCtx.getArguments().entrySet().stream() - .filter(entry -> !argumentsMap.containsKey(entry.getKey()) || (ArgumentType.TS_ROLLING.equals(entry.getValue().getType()) && state.getArguments().get(entry.getKey()) == null)) + .filter(entry -> !argumentsMap.containsKey(entry.getKey()) || (ArgumentType.TS_ROLLING.equals(entry.getValue().getRefEntityKey().getType()) && state.getArguments().get(entry.getKey()) == null)) .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); fetchArguments(calculatedFieldCtx.getTenantId(), entityId, missingArguments, argumentsMap::putAll) @@ -696,7 +653,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas } private ListenableFuture fetchArgumentValue(TenantId tenantId, EntityId targetEntityId, Argument argument) { - EntityId argumentEntityId = argument.getEntityId(); + EntityId argumentEntityId = argument.getRefEntityId(); EntityId entityId = isProfileEntity(argumentEntityId) ? targetEntityId : argumentEntityId; @@ -704,17 +661,17 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas } private ListenableFuture fetchKvEntry(TenantId tenantId, EntityId entityId, Argument argument) { - return switch (argument.getType()) { + return switch (argument.getRefEntityKey().getType()) { case TS_ROLLING -> fetchTsRolling(tenantId, entityId, argument); case ATTRIBUTE -> transformSingleValueArgument( Futures.transform( - attributesService.find(tenantId, entityId, argument.getScope(), argument.getKey()), + attributesService.find(tenantId, entityId, argument.getRefEntityKey().getScope(), argument.getRefEntityKey().getKey()), result -> result.or(() -> Optional.of(new BaseAttributeKvEntry(createDefaultKvEntry(argument), System.currentTimeMillis(), 0L))), calculatedFieldCallbackExecutor) ); case TS_LATEST -> transformSingleValueArgument( Futures.transform( - timeseriesService.findLatest(tenantId, entityId, argument.getKey()), + timeseriesService.findLatest(tenantId, entityId, argument.getRefEntityKey().getKey()), result -> result.or(() -> Optional.of(new BasicTsKvEntry(System.currentTimeMillis(), createDefaultKvEntry(argument), 0L))), calculatedFieldCallbackExecutor)); }; @@ -736,7 +693,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas long startTs = currentTime - timeWindow; int limit = argument.getLimit() == 0 ? MAX_LAST_RECORDS_VALUE : argument.getLimit(); - ReadTsKvQuery query = new BaseReadTsKvQuery(argument.getKey(), startTs, currentTime, 0, limit, Aggregation.NONE); + ReadTsKvQuery query = new BaseReadTsKvQuery(argument.getRefEntityKey().getKey(), startTs, currentTime, 0, limit, Aggregation.NONE); ListenableFuture> tsRollingFuture = timeseriesService.findAll(tenantId, entityId, List.of(query)); return Futures.transform(tsRollingFuture, tsRolling -> tsRolling == null ? TsRollingArgumentEntry.EMPTY : ArgumentEntry.createTsRollingArgument(tsRolling), calculatedFieldCallbackExecutor); @@ -826,7 +783,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas } private KvEntry createDefaultKvEntry(Argument argument) { - String key = argument.getKey(); + String key = argument.getRefEntityKey().getKey(); String defaultValue = argument.getDefaultValue(); if (NumberUtils.isParsable(defaultValue)) { return new DoubleDataEntry(key, Double.parseDouble(defaultValue)); diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java index d54a3220ed..e17a1a61f9 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java @@ -22,13 +22,16 @@ import org.thingsboard.server.common.data.cf.CalculatedFieldType; import org.thingsboard.server.common.data.cf.configuration.Argument; import org.thingsboard.server.common.data.cf.configuration.CalculatedFieldConfiguration; import org.thingsboard.server.common.data.cf.configuration.Output; +import org.thingsboard.server.common.data.cf.configuration.ReferencedEntityKey; import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.util.TbPair; import java.util.ArrayList; import java.util.List; import java.util.Map; +import java.util.stream.Collectors; @Data public class CalculatedFieldCtx { @@ -38,7 +41,8 @@ public class CalculatedFieldCtx { private EntityId entityId; private CalculatedFieldType cfType; private final Map arguments; - private final List argKeys; + private final Map, String> referencedEntityKeys; + private final List argNames; private Output output; private String expression; private TbelInvokeService tbelInvokeService; @@ -51,7 +55,12 @@ public class CalculatedFieldCtx { this.cfType = calculatedField.getType(); CalculatedFieldConfiguration configuration = calculatedField.getConfiguration(); this.arguments = configuration.getArguments(); - this.argKeys = new ArrayList<>(arguments.keySet()); + this.referencedEntityKeys = arguments.entrySet().stream() + .collect(Collectors.toMap( + entry -> new TbPair<>(entry.getValue().getRefEntityId(), entry.getValue().getRefEntityKey()), + Map.Entry::getKey + )); + this.argNames = new ArrayList<>(arguments.keySet()); this.output = configuration.getOutput(); this.expression = configuration.getExpression(); this.tbelInvokeService = tbelInvokeService; @@ -69,7 +78,7 @@ public class CalculatedFieldCtx { tenantId, tbelInvokeService, expression, - argKeys.toArray(String[]::new) + argNames.toArray(String[]::new) ); } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldState.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldState.java index de7c514786..0421055fef 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldState.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldState.java @@ -49,7 +49,7 @@ public class ScriptCalculatedFieldState extends BaseCalculatedFieldState { tsRecords.entrySet().removeIf(tsRecord -> tsRecord.getKey() < System.currentTimeMillis() - argument.getTimeWindow()); } }); - Object[] args = ctx.getArgKeys().stream() + Object[] args = ctx.getArgNames().stream() .map(key -> arguments.get(key).getValue()) .toArray(); ListenableFuture> resultFuture = ctx.getCalculatedFieldScriptEngine().executeToMapAsync(args); diff --git a/application/src/main/java/org/thingsboard/server/service/cf/telemetry/CalculatedFieldAttributeUpdateRequest.java b/application/src/main/java/org/thingsboard/server/service/cf/telemetry/CalculatedFieldAttributeUpdateRequest.java index a83cc0fc25..6050370fd2 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/telemetry/CalculatedFieldAttributeUpdateRequest.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/telemetry/CalculatedFieldAttributeUpdateRequest.java @@ -20,11 +20,16 @@ import lombok.Data; import org.thingsboard.rule.engine.api.AttributesSaveRequest; import org.thingsboard.server.common.data.AttributeScope; import org.thingsboard.server.common.data.cf.CalculatedFieldLinkConfiguration; +import org.thingsboard.server.common.data.cf.configuration.ArgumentType; +import org.thingsboard.server.common.data.cf.configuration.ReferencedEntityKey; import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.kv.KvEntry; +import org.thingsboard.server.common.data.util.TbPair; +import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldCtx; +import java.util.HashMap; import java.util.List; import java.util.Map; @@ -55,4 +60,24 @@ public class CalculatedFieldAttributeUpdateRequest implements CalculatedFieldTel }; } + @Override + public Map getMappedTelemetry(CalculatedFieldCtx ctx) { + Map mappedKvEntries = new HashMap<>(); + Map, String> referencedKeys = ctx.getReferencedEntityKeys(); + + kvEntries.forEach(entry -> { + String key = entry.getKey(); + + ReferencedEntityKey referencedEntityKey = new ReferencedEntityKey(key, ArgumentType.ATTRIBUTE, scope); + + String argName = referencedKeys.get(new TbPair<>(entityId, referencedEntityKey)); + + if (argName != null) { + mappedKvEntries.put(argName, entry); + } + }); + + return mappedKvEntries; + } + } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/telemetry/CalculatedFieldTelemetryUpdateRequest.java b/application/src/main/java/org/thingsboard/server/service/cf/telemetry/CalculatedFieldTelemetryUpdateRequest.java index 29ee899ec9..f85117dc41 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/telemetry/CalculatedFieldTelemetryUpdateRequest.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/telemetry/CalculatedFieldTelemetryUpdateRequest.java @@ -20,6 +20,7 @@ import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.kv.KvEntry; +import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldCtx; import java.util.List; import java.util.Map; @@ -36,4 +37,6 @@ public interface CalculatedFieldTelemetryUpdateRequest { Map getTelemetryKeysFromLink(CalculatedFieldLinkConfiguration linkConfiguration); + Map getMappedTelemetry(CalculatedFieldCtx ctx); + } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/telemetry/CalculatedFieldTimeSeriesUpdateRequest.java b/application/src/main/java/org/thingsboard/server/service/cf/telemetry/CalculatedFieldTimeSeriesUpdateRequest.java index 507daf386e..646145a46e 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/telemetry/CalculatedFieldTimeSeriesUpdateRequest.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/telemetry/CalculatedFieldTimeSeriesUpdateRequest.java @@ -19,11 +19,16 @@ import lombok.AllArgsConstructor; import lombok.Data; import org.thingsboard.rule.engine.api.TimeseriesSaveRequest; import org.thingsboard.server.common.data.cf.CalculatedFieldLinkConfiguration; +import org.thingsboard.server.common.data.cf.configuration.ArgumentType; +import org.thingsboard.server.common.data.cf.configuration.ReferencedEntityKey; import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.kv.KvEntry; +import org.thingsboard.server.common.data.util.TbPair; +import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldCtx; +import java.util.HashMap; import java.util.List; import java.util.Map; @@ -48,4 +53,30 @@ public class CalculatedFieldTimeSeriesUpdateRequest implements CalculatedFieldTe return linkConfiguration.getTimeSeries(); } + @Override + public Map getMappedTelemetry(CalculatedFieldCtx ctx) { + Map mappedKvEntries = new HashMap<>(); + Map, String> referencedKeys = ctx.getReferencedEntityKeys(); + + kvEntries.forEach(entry -> { + String key = entry.getKey(); + + ReferencedEntityKey tsLatestKey = new ReferencedEntityKey(key, ArgumentType.TS_LATEST, null); + String argTsLatestName = referencedKeys.get(new TbPair<>(entityId, tsLatestKey)); + + if (argTsLatestName != null) { + mappedKvEntries.put(argTsLatestName, entry); + } else { + ReferencedEntityKey tsRollingKey = new ReferencedEntityKey(key, ArgumentType.TS_ROLLING, null); + String argTsRollingName = referencedKeys.get(new TbPair<>(entityId, tsRollingKey)); + + if (argTsRollingName != null) { + mappedKvEntries.put(argTsRollingName, entry); + } + } + }); + + return mappedKvEntries; + } + } diff --git a/application/src/main/java/org/thingsboard/server/service/queue/ruleengine/TbRuleEngineQueueConsumerManager.java b/application/src/main/java/org/thingsboard/server/service/queue/ruleengine/TbRuleEngineQueueConsumerManager.java index 243a3adbf7..83ab90e3bf 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/ruleengine/TbRuleEngineQueueConsumerManager.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/ruleengine/TbRuleEngineQueueConsumerManager.java @@ -28,6 +28,7 @@ import org.thingsboard.server.common.msg.queue.QueueToRuleEngineMsg; import org.thingsboard.server.common.msg.queue.RuleEngineException; import org.thingsboard.server.common.msg.queue.RuleNodeInfo; import org.thingsboard.server.common.msg.queue.ServiceType; +import org.thingsboard.server.common.msg.queue.TbCallback; import org.thingsboard.server.common.msg.queue.TbMsgCallback; import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; @@ -179,6 +180,10 @@ public class TbRuleEngineQueueConsumerManager extends MainQueueConsumerManager getReferencedEntities() { return arguments.values().stream() - .map(Argument::getEntityId) + .map(Argument::getRefEntityId) .filter(Objects::nonNull) .collect(Collectors.toList()); } @@ -69,24 +70,24 @@ public abstract class BaseCalculatedFieldConfiguration implements CalculatedFiel CalculatedFieldLinkConfiguration linkConfiguration = new CalculatedFieldLinkConfiguration(); arguments.entrySet().stream() - .filter(entry -> entry.getValue().getEntityId().equals(entityId)) + .filter(entry -> entry.getValue().getRefEntityId().equals(entityId)) .forEach(entry -> { - Argument targetArgument = entry.getValue(); - String argumentKey = entry.getKey(); + ReferencedEntityKey refEntityKey = entry.getValue().getRefEntityKey(); + String argumentName = entry.getKey(); - switch (targetArgument.getType()) { + switch (refEntityKey.getType()) { case ATTRIBUTE -> { - switch (targetArgument.getScope()) { + switch (refEntityKey.getScope()) { case CLIENT_SCOPE -> - linkConfiguration.getClientAttributes().put(targetArgument.getKey(), argumentKey); + linkConfiguration.getClientAttributes().put(refEntityKey.getKey(), argumentName); case SERVER_SCOPE -> - linkConfiguration.getServerAttributes().put(targetArgument.getKey(), argumentKey); + linkConfiguration.getServerAttributes().put(refEntityKey.getKey(), argumentName); case SHARED_SCOPE -> - linkConfiguration.getSharedAttributes().put(targetArgument.getKey(), argumentKey); + linkConfiguration.getSharedAttributes().put(refEntityKey.getKey(), argumentName); } } case TS_LATEST, TS_ROLLING -> - linkConfiguration.getTimeSeries().put(targetArgument.getKey(), argumentKey); + linkConfiguration.getTimeSeries().put(refEntityKey.getKey(), argumentName); } }); @@ -118,7 +119,7 @@ public abstract class BaseCalculatedFieldConfiguration implements CalculatedFiel ObjectNode argumentsNode = configNode.putObject("arguments"); arguments.forEach((key, argument) -> { ObjectNode argumentNode = argumentsNode.putObject(key); - EntityId referencedEntityId = argument.getEntityId(); + EntityId referencedEntityId = argument.getRefEntityId(); if (referencedEntityId != null) { argumentNode.put("entityType", referencedEntityId.getEntityType().name()); argumentNode.put("entityId", referencedEntityId.getId().toString()); @@ -126,9 +127,9 @@ public abstract class BaseCalculatedFieldConfiguration implements CalculatedFiel argumentNode.put("entityType", entityType.name()); argumentNode.put("entityId", entityId.toString()); } - argumentNode.put("key", argument.getKey()); - argumentNode.put("type", String.valueOf(argument.getType())); - argumentNode.put("scope", String.valueOf(argument.getScope())); +// argumentNode.put("key", argument.getKey()); +// argumentNode.put("type", String.valueOf(argument.getType())); +// argumentNode.put("scope", String.valueOf(argument.getScope())); argumentNode.put("defaultValue", argument.getDefaultValue()); argumentNode.put("limit", String.valueOf(argument.getLimit())); argumentNode.put("timeWindow", String.valueOf(argument.getTimeWindow())); @@ -165,19 +166,19 @@ public abstract class BaseCalculatedFieldConfiguration implements CalculatedFiel if (argumentNode.hasNonNull("entityType") && argumentNode.hasNonNull("entityId")) { String referencedEntityType = argumentNode.get("entityType").asText(); UUID referencedEntityId = UUID.fromString(argumentNode.get("entityId").asText()); - argument.setEntityId(EntityIdFactory.getByTypeAndUuid(referencedEntityType, referencedEntityId)); + argument.setRefEntityId(EntityIdFactory.getByTypeAndUuid(referencedEntityType, referencedEntityId)); } else { - argument.setEntityId(EntityIdFactory.getByTypeAndUuid(entityType, entityId)); + argument.setRefEntityId(EntityIdFactory.getByTypeAndUuid(entityType, entityId)); } - argument.setKey(argumentNode.get("key").asText()); +// argument.setRefEntityKey(argumentNode.get("key").asText()); JsonNode type = argumentNode.get("type"); - if (type != null && !type.isNull() && !type.asText().equals("null")) { - argument.setType(ArgumentType.valueOf(type.asText())); - } - JsonNode scope = argumentNode.get("scope"); - if (scope != null && !scope.isNull() && !scope.asText().equals("null")) { - argument.setScope(AttributeScope.valueOf(scope.asText())); - } +// if (type != null && !type.isNull() && !type.asText().equals("null")) { +// argument.setType(ArgumentType.valueOf(type.asText())); +// } +// JsonNode scope = argumentNode.get("scope"); +// if (scope != null && !scope.isNull() && !scope.asText().equals("null")) { +// argument.setScope(AttributeScope.valueOf(scope.asText())); +// } if (argumentNode.hasNonNull("defaultValue")) { argument.setDefaultValue(argumentNode.get("defaultValue").asText()); } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/ReferencedEntityKey.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/ReferencedEntityKey.java new file mode 100644 index 0000000000..b49495d959 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/ReferencedEntityKey.java @@ -0,0 +1,30 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.cf.configuration; + +import lombok.AllArgsConstructor; +import lombok.Data; +import org.thingsboard.server.common.data.AttributeScope; + +@Data +@AllArgsConstructor +public class ReferencedEntityKey { + + private String key; + private ArgumentType type; + private AttributeScope scope; + +} diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/sql/CalculatedFieldEntity.java b/dao/src/main/java/org/thingsboard/server/dao/model/sql/CalculatedFieldEntity.java index 6aaaf05836..d1352a26b8 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/sql/CalculatedFieldEntity.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/sql/CalculatedFieldEntity.java @@ -22,6 +22,7 @@ import jakarta.persistence.Entity; import jakarta.persistence.Table; import lombok.Data; import lombok.EqualsAndHashCode; +import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.cf.CalculatedFieldType; @@ -95,6 +96,7 @@ public class CalculatedFieldEntity extends BaseSqlEntity implem this.type = calculatedField.getType().name(); this.name = calculatedField.getName(); this.configurationVersion = calculatedField.getConfigurationVersion(); +// this.configuration = JacksonUtil.valueToTree(calculatedField.getConfiguration()); this.configuration = calculatedField.getConfiguration().calculatedFieldConfigToJson(EntityType.valueOf(entityType), entityId); this.version = calculatedField.getVersion(); if (calculatedField.getExternalId() != null) { @@ -112,6 +114,7 @@ public class CalculatedFieldEntity extends BaseSqlEntity implem calculatedField.setName(name); calculatedField.setConfigurationVersion(configurationVersion); calculatedField.setConfiguration(readCalculatedFieldConfiguration(configuration, EntityType.valueOf(entityType), entityId)); +// calculatedField.setConfiguration(JacksonUtil.treeToValue(configuration, CalculatedFieldConfiguration.class)); calculatedField.setVersion(version); if (externalId != null) { calculatedField.setExternalId(new CalculatedFieldId(externalId)); diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/AssetServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/AssetServiceTest.java index 9a0b9222f0..f38722c9c1 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/AssetServiceTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/AssetServiceTest.java @@ -887,7 +887,7 @@ public class AssetServiceTest extends AbstractServiceTest { Argument argument = new Argument(); argument.setEntityId(savedAsset.getId()); argument.setType(ArgumentType.TS_LATEST); - argument.setKey("temperature"); + argument.setRefEntityKey("temperature"); config.setArguments(Map.of("T", argument)); diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/CalculatedFieldServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/CalculatedFieldServiceTest.java index 5a8f7a2383..a9a70d8bc8 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/CalculatedFieldServiceTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/CalculatedFieldServiceTest.java @@ -156,7 +156,7 @@ public class CalculatedFieldServiceTest extends AbstractServiceTest { Argument argument = new Argument(); argument.setEntityId(referencedEntityId); argument.setType(ArgumentType.TS_LATEST); - argument.setKey("temperature"); + argument.setRefEntityKey("temperature"); config.setArguments(Map.of("T", argument)); diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/CustomerServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/CustomerServiceTest.java index d0ee833261..236b159ddb 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/CustomerServiceTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/CustomerServiceTest.java @@ -382,7 +382,7 @@ public class CustomerServiceTest extends AbstractServiceTest { Argument argument = new Argument(); argument.setEntityId(savedCustomer.getId()); argument.setType(ArgumentType.TS_LATEST); - argument.setKey("temperature"); + argument.setRefEntityKey("temperature"); config.setArguments(Map.of("T", argument)); diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/DeviceServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/DeviceServiceTest.java index 5b060ae145..cca30e6fbc 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/DeviceServiceTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/DeviceServiceTest.java @@ -1225,7 +1225,7 @@ public class DeviceServiceTest extends AbstractServiceTest { Argument argument = new Argument(); argument.setEntityId(device.getId()); argument.setType(ArgumentType.TS_LATEST); - argument.setKey("temperature"); + argument.setRefEntityKey("temperature"); config.setArguments(Map.of("T", argument)); From f0a36d500883a382e540e861b0a36be20e5e9fbf Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Tue, 21 Jan 2025 10:03:50 +0200 Subject: [PATCH 7/9] fixed tests --- .../BaseCalculatedFieldConfiguration.java | 129 +----------------- .../CalculatedFieldConfiguration.java | 6 - .../ScriptCalculatedFieldConfiguration.java | 12 -- .../SimpleCalculatedFieldConfiguration.java | 12 -- .../dao/model/sql/CalculatedFieldEntity.java | 16 +-- ...efaultNativeCalculatedFieldRepository.java | 11 +- .../server/dao/service/AssetServiceTest.java | 7 +- .../service/CalculatedFieldServiceTest.java | 7 +- .../dao/service/CustomerServiceTest.java | 7 +- .../server/dao/service/DeviceServiceTest.java | 7 +- 10 files changed, 20 insertions(+), 194 deletions(-) diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/BaseCalculatedFieldConfiguration.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/BaseCalculatedFieldConfiguration.java index 8cab602683..87cece0419 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/BaseCalculatedFieldConfiguration.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/BaseCalculatedFieldConfiguration.java @@ -15,48 +15,25 @@ */ package org.thingsboard.server.common.data.cf.configuration; -import com.fasterxml.jackson.annotation.JsonIgnore; -import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.databind.node.ObjectNode; import lombok.Data; -import org.thingsboard.server.common.data.AttributeScope; -import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.cf.CalculatedFieldLink; import org.thingsboard.server.common.data.cf.CalculatedFieldLinkConfiguration; import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.EntityId; -import org.thingsboard.server.common.data.id.EntityIdFactory; import org.thingsboard.server.common.data.id.TenantId; -import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Objects; -import java.util.UUID; import java.util.stream.Collectors; @Data public abstract class BaseCalculatedFieldConfiguration implements CalculatedFieldConfiguration { - @JsonIgnore - private final ObjectMapper mapper = new ObjectMapper(); - protected Map arguments; protected String expression; protected Output output; - public BaseCalculatedFieldConfiguration() { - } - - public BaseCalculatedFieldConfiguration(JsonNode config, EntityType entityType, UUID entityId) { -// BaseCalculatedFieldConfiguration calculatedFieldConfig = mapper.convertValue(config, BaseCalculatedFieldConfiguration.class); - BaseCalculatedFieldConfiguration calculatedFieldConfig = toCalculatedFieldConfig(config, entityType, entityId); - this.arguments = calculatedFieldConfig.getArguments(); - this.expression = calculatedFieldConfig.getExpression(); - this.output = calculatedFieldConfig.getOutput(); - } - @Override public List getReferencedEntities() { return arguments.values().stream() @@ -70,7 +47,7 @@ public abstract class BaseCalculatedFieldConfiguration implements CalculatedFiel CalculatedFieldLinkConfiguration linkConfiguration = new CalculatedFieldLinkConfiguration(); arguments.entrySet().stream() - .filter(entry -> entry.getValue().getRefEntityId().equals(entityId)) + .filter(entry -> entry.getValue().getRefEntityId() != null && entry.getValue().getRefEntityId().equals(entityId)) .forEach(entry -> { ReferencedEntityKey refEntityKey = entry.getValue().getRefEntityKey(); String argumentName = entry.getKey(); @@ -112,108 +89,4 @@ public abstract class BaseCalculatedFieldConfiguration implements CalculatedFiel return link; } - @Override - public JsonNode calculatedFieldConfigToJson(EntityType entityType, UUID entityId) { - ObjectNode configNode = mapper.createObjectNode(); - - ObjectNode argumentsNode = configNode.putObject("arguments"); - arguments.forEach((key, argument) -> { - ObjectNode argumentNode = argumentsNode.putObject(key); - EntityId referencedEntityId = argument.getRefEntityId(); - if (referencedEntityId != null) { - argumentNode.put("entityType", referencedEntityId.getEntityType().name()); - argumentNode.put("entityId", referencedEntityId.getId().toString()); - } else { - argumentNode.put("entityType", entityType.name()); - argumentNode.put("entityId", entityId.toString()); - } -// argumentNode.put("key", argument.getKey()); -// argumentNode.put("type", String.valueOf(argument.getType())); -// argumentNode.put("scope", String.valueOf(argument.getScope())); - argumentNode.put("defaultValue", argument.getDefaultValue()); - argumentNode.put("limit", String.valueOf(argument.getLimit())); - argumentNode.put("timeWindow", String.valueOf(argument.getTimeWindow())); - }); - - if (expression != null) { - configNode.put("expression", expression); - } - - if (output != null) { - ObjectNode outputNode = configNode.putObject("output"); - outputNode.put("name", output.getName()); - outputNode.put("type", String.valueOf(output.getType())); - if (output.getScope() != null) { - outputNode.put("scope", String.valueOf(output.getScope())); - } - } - - return configNode; - } - - private BaseCalculatedFieldConfiguration toCalculatedFieldConfig(JsonNode config, EntityType entityType, UUID entityId) { - if (config == null || !config.isObject()) { - return null; - } - - Map arguments = new HashMap<>(); - JsonNode argumentsNode = config.get("arguments"); - if (argumentsNode != null && argumentsNode.isObject()) { - argumentsNode.fields().forEachRemaining(entry -> { - String key = entry.getKey(); - JsonNode argumentNode = entry.getValue(); - Argument argument = new Argument(); - if (argumentNode.hasNonNull("entityType") && argumentNode.hasNonNull("entityId")) { - String referencedEntityType = argumentNode.get("entityType").asText(); - UUID referencedEntityId = UUID.fromString(argumentNode.get("entityId").asText()); - argument.setRefEntityId(EntityIdFactory.getByTypeAndUuid(referencedEntityType, referencedEntityId)); - } else { - argument.setRefEntityId(EntityIdFactory.getByTypeAndUuid(entityType, entityId)); - } -// argument.setRefEntityKey(argumentNode.get("key").asText()); - JsonNode type = argumentNode.get("type"); -// if (type != null && !type.isNull() && !type.asText().equals("null")) { -// argument.setType(ArgumentType.valueOf(type.asText())); -// } -// JsonNode scope = argumentNode.get("scope"); -// if (scope != null && !scope.isNull() && !scope.asText().equals("null")) { -// argument.setScope(AttributeScope.valueOf(scope.asText())); -// } - if (argumentNode.hasNonNull("defaultValue")) { - argument.setDefaultValue(argumentNode.get("defaultValue").asText()); - } - if (argumentNode.hasNonNull("limit")) { - argument.setLimit(argumentNode.get("limit").asInt()); - } - if (argumentNode.hasNonNull("timeWindow")) { - argument.setTimeWindow(argumentNode.get("timeWindow").asInt()); - } - arguments.put(key, argument); - }); - } - this.setArguments(arguments); - - JsonNode expressionNode = config.get("expression"); - if (expressionNode != null && expressionNode.isTextual()) { - this.setExpression(expressionNode.asText()); - } - - JsonNode outputNode = config.get("output"); - if (outputNode != null) { - Output output = new Output(); - output.setName(outputNode.get("name").asText()); - JsonNode type = outputNode.get("type"); - if (type != null && !type.isNull() && !type.asText().equals("null")) { - output.setType(OutputType.valueOf(type.asText())); - } - JsonNode scope = outputNode.get("scope"); - if (scope != null && !scope.isNull() && !scope.asText().equals("null")) { - output.setScope(AttributeScope.valueOf(scope.asText())); - } - this.setOutput(output); - } - - return this; - } - } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/CalculatedFieldConfiguration.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/CalculatedFieldConfiguration.java index ac94ade134..8f56bf491d 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/CalculatedFieldConfiguration.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/CalculatedFieldConfiguration.java @@ -18,8 +18,6 @@ package org.thingsboard.server.common.data.cf.configuration; import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonSubTypes; import com.fasterxml.jackson.annotation.JsonTypeInfo; -import com.fasterxml.jackson.databind.JsonNode; -import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.cf.CalculatedFieldLink; import org.thingsboard.server.common.data.cf.CalculatedFieldLinkConfiguration; import org.thingsboard.server.common.data.cf.CalculatedFieldType; @@ -29,7 +27,6 @@ import org.thingsboard.server.common.data.id.TenantId; import java.util.List; import java.util.Map; -import java.util.UUID; @JsonTypeInfo( use = JsonTypeInfo.Id.NAME, @@ -57,9 +54,6 @@ public interface CalculatedFieldConfiguration { @JsonIgnore CalculatedFieldLinkConfiguration getReferencedEntityConfig(EntityId entityId); - @JsonIgnore - JsonNode calculatedFieldConfigToJson(EntityType entityType, UUID entityId); - List buildCalculatedFieldLinks(TenantId tenantId, EntityId cfEntityId, CalculatedFieldId calculatedFieldId); CalculatedFieldLink buildCalculatedFieldLink(TenantId tenantId, EntityId referencedEntityId, CalculatedFieldId calculatedFieldId); diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/ScriptCalculatedFieldConfiguration.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/ScriptCalculatedFieldConfiguration.java index a24328b4c9..017fc5a485 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/ScriptCalculatedFieldConfiguration.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/ScriptCalculatedFieldConfiguration.java @@ -15,24 +15,12 @@ */ package org.thingsboard.server.common.data.cf.configuration; -import com.fasterxml.jackson.databind.JsonNode; import lombok.Data; -import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.cf.CalculatedFieldType; -import java.util.UUID; - @Data public class ScriptCalculatedFieldConfiguration extends BaseCalculatedFieldConfiguration implements CalculatedFieldConfiguration { - public ScriptCalculatedFieldConfiguration() { - super(); - } - - public ScriptCalculatedFieldConfiguration(JsonNode config, EntityType entityType, UUID entityId) { - super(config, entityType, entityId); - } - @Override public CalculatedFieldType getType() { return CalculatedFieldType.SCRIPT; diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/SimpleCalculatedFieldConfiguration.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/SimpleCalculatedFieldConfiguration.java index af11d2f5d8..6312c3e1db 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/SimpleCalculatedFieldConfiguration.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/SimpleCalculatedFieldConfiguration.java @@ -15,24 +15,12 @@ */ package org.thingsboard.server.common.data.cf.configuration; -import com.fasterxml.jackson.databind.JsonNode; import lombok.Data; -import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.cf.CalculatedFieldType; -import java.util.UUID; - @Data public class SimpleCalculatedFieldConfiguration extends BaseCalculatedFieldConfiguration implements CalculatedFieldConfiguration { - public SimpleCalculatedFieldConfiguration() { - super(); - } - - public SimpleCalculatedFieldConfiguration(JsonNode config, EntityType entityType, UUID entityId) { - super(config, entityType, entityId); - } - @Override public CalculatedFieldType getType() { return CalculatedFieldType.SIMPLE; diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/sql/CalculatedFieldEntity.java b/dao/src/main/java/org/thingsboard/server/dao/model/sql/CalculatedFieldEntity.java index d1352a26b8..a0157cde66 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/sql/CalculatedFieldEntity.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/sql/CalculatedFieldEntity.java @@ -23,12 +23,9 @@ import jakarta.persistence.Table; import lombok.Data; import lombok.EqualsAndHashCode; import org.thingsboard.common.util.JacksonUtil; -import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.cf.CalculatedFieldType; import org.thingsboard.server.common.data.cf.configuration.CalculatedFieldConfiguration; -import org.thingsboard.server.common.data.cf.configuration.ScriptCalculatedFieldConfiguration; -import org.thingsboard.server.common.data.cf.configuration.SimpleCalculatedFieldConfiguration; import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.EntityIdFactory; import org.thingsboard.server.common.data.id.TenantId; @@ -96,8 +93,7 @@ public class CalculatedFieldEntity extends BaseSqlEntity implem this.type = calculatedField.getType().name(); this.name = calculatedField.getName(); this.configurationVersion = calculatedField.getConfigurationVersion(); -// this.configuration = JacksonUtil.valueToTree(calculatedField.getConfiguration()); - this.configuration = calculatedField.getConfiguration().calculatedFieldConfigToJson(EntityType.valueOf(entityType), entityId); + this.configuration = JacksonUtil.valueToTree(calculatedField.getConfiguration()); this.version = calculatedField.getVersion(); if (calculatedField.getExternalId() != null) { this.externalId = calculatedField.getExternalId().getId(); @@ -113,8 +109,7 @@ public class CalculatedFieldEntity extends BaseSqlEntity implem calculatedField.setType(CalculatedFieldType.valueOf(type)); calculatedField.setName(name); calculatedField.setConfigurationVersion(configurationVersion); - calculatedField.setConfiguration(readCalculatedFieldConfiguration(configuration, EntityType.valueOf(entityType), entityId)); -// calculatedField.setConfiguration(JacksonUtil.treeToValue(configuration, CalculatedFieldConfiguration.class)); + calculatedField.setConfiguration(JacksonUtil.treeToValue(configuration, CalculatedFieldConfiguration.class)); calculatedField.setVersion(version); if (externalId != null) { calculatedField.setExternalId(new CalculatedFieldId(externalId)); @@ -122,11 +117,4 @@ public class CalculatedFieldEntity extends BaseSqlEntity implem return calculatedField; } - private CalculatedFieldConfiguration readCalculatedFieldConfiguration(JsonNode config, EntityType entityType, UUID entityId) { - return switch (CalculatedFieldType.valueOf(type)) { - case SIMPLE -> new SimpleCalculatedFieldConfiguration(config, entityType, entityId); - case SCRIPT -> new ScriptCalculatedFieldConfiguration(config, entityType, entityId); - }; - } - } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/cf/DefaultNativeCalculatedFieldRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/cf/DefaultNativeCalculatedFieldRepository.java index a5a2743f26..bb88982e3d 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/cf/DefaultNativeCalculatedFieldRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/cf/DefaultNativeCalculatedFieldRepository.java @@ -29,8 +29,6 @@ import org.thingsboard.server.common.data.cf.CalculatedFieldLink; import org.thingsboard.server.common.data.cf.CalculatedFieldLinkConfiguration; import org.thingsboard.server.common.data.cf.CalculatedFieldType; import org.thingsboard.server.common.data.cf.configuration.CalculatedFieldConfiguration; -import org.thingsboard.server.common.data.cf.configuration.ScriptCalculatedFieldConfiguration; -import org.thingsboard.server.common.data.cf.configuration.SimpleCalculatedFieldConfiguration; import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.CalculatedFieldLinkId; import org.thingsboard.server.common.data.id.EntityIdFactory; @@ -90,7 +88,7 @@ public class DefaultNativeCalculatedFieldRepository implements NativeCalculatedF calculatedField.setType(type); calculatedField.setName(name); calculatedField.setConfigurationVersion(configurationVersion); - calculatedField.setConfiguration(readCalculatedFieldConfiguration(type, configuration, entityType, entityId)); + calculatedField.setConfiguration(JacksonUtil.treeToValue(configuration, CalculatedFieldConfiguration.class)); calculatedField.setVersion(version); calculatedField.setExternalId(externalIdObj != null ? new CalculatedFieldId(UUID.fromString((String) externalIdObj)) : null); @@ -135,11 +133,4 @@ public class DefaultNativeCalculatedFieldRepository implements NativeCalculatedF }); } - private CalculatedFieldConfiguration readCalculatedFieldConfiguration(CalculatedFieldType type, JsonNode config, EntityType entityType, UUID entityId) { - return switch (type) { - case SIMPLE -> new SimpleCalculatedFieldConfiguration(config, entityType, entityId); - case SCRIPT -> new ScriptCalculatedFieldConfiguration(config, entityType, entityId); - }; - } - } diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/AssetServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/AssetServiceTest.java index f38722c9c1..aed1621e1c 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/AssetServiceTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/AssetServiceTest.java @@ -36,6 +36,7 @@ import org.thingsboard.server.common.data.cf.configuration.Argument; import org.thingsboard.server.common.data.cf.configuration.ArgumentType; import org.thingsboard.server.common.data.cf.configuration.Output; import org.thingsboard.server.common.data.cf.configuration.OutputType; +import org.thingsboard.server.common.data.cf.configuration.ReferencedEntityKey; import org.thingsboard.server.common.data.cf.configuration.SimpleCalculatedFieldConfiguration; import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.TenantId; @@ -885,9 +886,9 @@ public class AssetServiceTest extends AbstractServiceTest { SimpleCalculatedFieldConfiguration config = new SimpleCalculatedFieldConfiguration(); Argument argument = new Argument(); - argument.setEntityId(savedAsset.getId()); - argument.setType(ArgumentType.TS_LATEST); - argument.setRefEntityKey("temperature"); + argument.setRefEntityId(savedAsset.getId()); + ReferencedEntityKey refEntityKey = new ReferencedEntityKey("temperature", ArgumentType.TS_LATEST, null); + argument.setRefEntityKey(refEntityKey); config.setArguments(Map.of("T", argument)); diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/CalculatedFieldServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/CalculatedFieldServiceTest.java index a9a70d8bc8..6dd84714f8 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/CalculatedFieldServiceTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/CalculatedFieldServiceTest.java @@ -30,6 +30,7 @@ import org.thingsboard.server.common.data.cf.configuration.ArgumentType; import org.thingsboard.server.common.data.cf.configuration.CalculatedFieldConfiguration; import org.thingsboard.server.common.data.cf.configuration.Output; import org.thingsboard.server.common.data.cf.configuration.OutputType; +import org.thingsboard.server.common.data.cf.configuration.ReferencedEntityKey; import org.thingsboard.server.common.data.cf.configuration.SimpleCalculatedFieldConfiguration; import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.EntityId; @@ -154,9 +155,9 @@ public class CalculatedFieldServiceTest extends AbstractServiceTest { SimpleCalculatedFieldConfiguration config = new SimpleCalculatedFieldConfiguration(); Argument argument = new Argument(); - argument.setEntityId(referencedEntityId); - argument.setType(ArgumentType.TS_LATEST); - argument.setRefEntityKey("temperature"); + argument.setRefEntityId(referencedEntityId); + ReferencedEntityKey refEntityKey = new ReferencedEntityKey("temperature", ArgumentType.TS_LATEST, null); + argument.setRefEntityKey(refEntityKey); config.setArguments(Map.of("T", argument)); diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/CustomerServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/CustomerServiceTest.java index 236b159ddb..6e57279f38 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/CustomerServiceTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/CustomerServiceTest.java @@ -37,6 +37,7 @@ import org.thingsboard.server.common.data.cf.configuration.Argument; import org.thingsboard.server.common.data.cf.configuration.ArgumentType; import org.thingsboard.server.common.data.cf.configuration.Output; import org.thingsboard.server.common.data.cf.configuration.OutputType; +import org.thingsboard.server.common.data.cf.configuration.ReferencedEntityKey; import org.thingsboard.server.common.data.cf.configuration.SimpleCalculatedFieldConfiguration; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageData; @@ -380,9 +381,9 @@ public class CustomerServiceTest extends AbstractServiceTest { SimpleCalculatedFieldConfiguration config = new SimpleCalculatedFieldConfiguration(); Argument argument = new Argument(); - argument.setEntityId(savedCustomer.getId()); - argument.setType(ArgumentType.TS_LATEST); - argument.setRefEntityKey("temperature"); + argument.setRefEntityId(savedCustomer.getId()); + ReferencedEntityKey refEntityKey = new ReferencedEntityKey("temperature", ArgumentType.TS_LATEST, null); + argument.setRefEntityKey(refEntityKey); config.setArguments(Map.of("T", argument)); diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/DeviceServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/DeviceServiceTest.java index cca30e6fbc..959825e113 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/DeviceServiceTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/DeviceServiceTest.java @@ -45,6 +45,7 @@ import org.thingsboard.server.common.data.cf.configuration.Argument; import org.thingsboard.server.common.data.cf.configuration.ArgumentType; import org.thingsboard.server.common.data.cf.configuration.Output; import org.thingsboard.server.common.data.cf.configuration.OutputType; +import org.thingsboard.server.common.data.cf.configuration.ReferencedEntityKey; import org.thingsboard.server.common.data.cf.configuration.SimpleCalculatedFieldConfiguration; import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.DeviceProfileId; @@ -1223,9 +1224,9 @@ public class DeviceServiceTest extends AbstractServiceTest { SimpleCalculatedFieldConfiguration config = new SimpleCalculatedFieldConfiguration(); Argument argument = new Argument(); - argument.setEntityId(device.getId()); - argument.setType(ArgumentType.TS_LATEST); - argument.setRefEntityKey("temperature"); + argument.setRefEntityId(device.getId()); + ReferencedEntityKey refEntityKey = new ReferencedEntityKey("temperature", ArgumentType.TS_LATEST, null); + argument.setRefEntityKey(refEntityKey); config.setArguments(Map.of("T", argument)); From d9ccc8118cf316389e3c284e468b4236118462bb Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Tue, 21 Jan 2025 10:52:06 +0200 Subject: [PATCH 8/9] fixed calculated field controller test --- .../server/controller/CalculatedFieldControllerTest.java | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/application/src/test/java/org/thingsboard/server/controller/CalculatedFieldControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/CalculatedFieldControllerTest.java index 565321afbc..b1c7547251 100644 --- a/application/src/test/java/org/thingsboard/server/controller/CalculatedFieldControllerTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/CalculatedFieldControllerTest.java @@ -28,6 +28,7 @@ import org.thingsboard.server.common.data.cf.configuration.ArgumentType; import org.thingsboard.server.common.data.cf.configuration.CalculatedFieldConfiguration; import org.thingsboard.server.common.data.cf.configuration.Output; import org.thingsboard.server.common.data.cf.configuration.OutputType; +import org.thingsboard.server.common.data.cf.configuration.ReferencedEntityKey; import org.thingsboard.server.common.data.cf.configuration.SimpleCalculatedFieldConfiguration; import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.EntityId; @@ -141,9 +142,9 @@ public class CalculatedFieldControllerTest extends AbstractControllerTest { SimpleCalculatedFieldConfiguration config = new SimpleCalculatedFieldConfiguration(); Argument argument = new Argument(); - argument.setEntityId(referencedEntityId); - argument.setType(ArgumentType.TS_LATEST); - argument.setRefEntityKey("temperature"); + argument.setRefEntityId(referencedEntityId); + ReferencedEntityKey refEntityKey = new ReferencedEntityKey("temperature", ArgumentType.TS_LATEST, null); + argument.setRefEntityKey(refEntityKey); config.setArguments(Map.of("T", argument)); From 9ef68584c9aecb19659496210cf914a7eb5f81b5 Mon Sep 17 00:00:00 2001 From: IrynaMatveieva Date: Wed, 22 Jan 2025 09:40:12 +0200 Subject: [PATCH 9/9] updated getMappedTelemetry method --- ...efaultCalculatedFieldExecutionService.java | 19 ++++++++++--------- .../cf/ctx/state/CalculatedFieldCtx.java | 2 +- ...CalculatedFieldAttributeUpdateRequest.java | 15 ++------------- ...CalculatedFieldTelemetryUpdateRequest.java | 5 +---- ...alculatedFieldTimeSeriesUpdateRequest.java | 13 +++---------- 5 files changed, 17 insertions(+), 37 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java index 7c17d54649..84a21ab315 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java @@ -271,7 +271,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas case ASSET_PROFILE, DEVICE_PROFILE -> { log.info("Initializing state for all entities in profile: tenantId=[{}], profileId=[{}]", tenantId, entityId); Map commonArguments = calculatedFieldCtx.getArguments().entrySet().stream() - .filter(entry -> !isProfileEntity(entry.getValue().getRefEntityId())) + .filter(entry -> entry.getValue().getRefEntityId() != null) .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); fetchArguments(tenantId, entityId, commonArguments, commonArgs -> { calculatedFieldCache.getEntitiesByProfile(tenantId, entityId).forEach(targetEntityId -> { @@ -375,9 +375,10 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas private void processCalculatedFields(CalculatedFieldTelemetryUpdateRequest request, EntityId cfTargetEntityId) { if (cfTargetEntityId != null) { calculatedFieldCache.getCalculatedFieldCtxsByEntityId(cfTargetEntityId, tbelInvokeService).forEach(ctx -> { - Map updatedTelemetry = request.getMappedTelemetry(ctx); + Map updatedTelemetry = request.getMappedTelemetry(ctx, cfTargetEntityId); if (!updatedTelemetry.isEmpty()) { - executeTelemetryUpdate(ctx, request.getEntityId(), request.getPreviousCalculatedFieldIds(), updatedTelemetry); + EntityId targetEntityId = isProfileEntity(cfTargetEntityId) ? request.getEntityId() : cfTargetEntityId; + executeTelemetryUpdate(ctx, targetEntityId, request.getPreviousCalculatedFieldIds(), updatedTelemetry); } }); } @@ -406,9 +407,9 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas private void processCalculatedFieldLink(CalculatedFieldTelemetryUpdateRequest request, EntityId targetEntity, CalculatedFieldCtx ctx, Map> tpiStates) { TopicPartitionInfo targetEntityTpi = partitionService.resolve(ServiceType.TB_RULE_ENGINE, request.getTenantId(), targetEntity); if (targetEntityTpi.isMyPartition()) { - Map updatedTelemetry = request.getMappedTelemetry(ctx); + Map updatedTelemetry = request.getMappedTelemetry(ctx, request.getEntityId()); if (!updatedTelemetry.isEmpty()) { - executeTelemetryUpdate(ctx, request.getEntityId(), request.getPreviousCalculatedFieldIds(), updatedTelemetry); + executeTelemetryUpdate(ctx, targetEntity, request.getPreviousCalculatedFieldIds(), updatedTelemetry); } } else { List ctxIds = tpiStates.computeIfAbsent(targetEntityTpi, k -> new ArrayList<>()); @@ -427,13 +428,13 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas } proto.getLinksList().forEach(ctxIdProto -> { - EntityId entityId = request.getEntityId(); CalculatedFieldId calculatedFieldId = new CalculatedFieldId(new UUID(ctxIdProto.getCalculatedFieldIdMSB(), ctxIdProto.getCalculatedFieldIdLSB())); CalculatedFieldCtx ctx = calculatedFieldCache.getCalculatedFieldCtx(calculatedFieldId, tbelInvokeService); - Map updatedTelemetry = request.getMappedTelemetry(ctx); + Map updatedTelemetry = request.getMappedTelemetry(ctx, request.getEntityId()); if (!updatedTelemetry.isEmpty()) { - executeTelemetryUpdate(ctx, entityId, request.getPreviousCalculatedFieldIds(), updatedTelemetry); + EntityId targetEntityId = EntityIdFactory.getByTypeAndUuid(ctxIdProto.getEntityType(), new UUID(ctxIdProto.getEntityIdMSB(), ctxIdProto.getEntityIdLSB())); + executeTelemetryUpdate(ctx, targetEntityId, request.getPreviousCalculatedFieldIds(), updatedTelemetry); } }); } catch (Exception e) { @@ -654,7 +655,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas private ListenableFuture fetchArgumentValue(TenantId tenantId, EntityId targetEntityId, Argument argument) { EntityId argumentEntityId = argument.getRefEntityId(); - EntityId entityId = isProfileEntity(argumentEntityId) + EntityId entityId = (argumentEntityId == null || isProfileEntity(argumentEntityId)) ? targetEntityId : argumentEntityId; return fetchKvEntry(tenantId, entityId, argument); diff --git a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java index e17a1a61f9..cb4052b7df 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java @@ -57,7 +57,7 @@ public class CalculatedFieldCtx { this.arguments = configuration.getArguments(); this.referencedEntityKeys = arguments.entrySet().stream() .collect(Collectors.toMap( - entry -> new TbPair<>(entry.getValue().getRefEntityId(), entry.getValue().getRefEntityKey()), + entry -> new TbPair<>(entry.getValue().getRefEntityId() == null ? entityId : entry.getValue().getRefEntityId(), entry.getValue().getRefEntityKey()), Map.Entry::getKey )); this.argNames = new ArrayList<>(arguments.keySet()); diff --git a/application/src/main/java/org/thingsboard/server/service/cf/telemetry/CalculatedFieldAttributeUpdateRequest.java b/application/src/main/java/org/thingsboard/server/service/cf/telemetry/CalculatedFieldAttributeUpdateRequest.java index 6050370fd2..d2eb31cd6d 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/telemetry/CalculatedFieldAttributeUpdateRequest.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/telemetry/CalculatedFieldAttributeUpdateRequest.java @@ -19,7 +19,6 @@ import lombok.AllArgsConstructor; import lombok.Data; import org.thingsboard.rule.engine.api.AttributesSaveRequest; import org.thingsboard.server.common.data.AttributeScope; -import org.thingsboard.server.common.data.cf.CalculatedFieldLinkConfiguration; import org.thingsboard.server.common.data.cf.configuration.ArgumentType; import org.thingsboard.server.common.data.cf.configuration.ReferencedEntityKey; import org.thingsboard.server.common.data.id.CalculatedFieldId; @@ -52,16 +51,7 @@ public class CalculatedFieldAttributeUpdateRequest implements CalculatedFieldTel } @Override - public Map getTelemetryKeysFromLink(CalculatedFieldLinkConfiguration linkConfiguration) { - return switch (scope) { - case CLIENT_SCOPE -> linkConfiguration.getClientAttributes(); - case SERVER_SCOPE -> linkConfiguration.getServerAttributes(); - case SHARED_SCOPE -> linkConfiguration.getSharedAttributes(); - }; - } - - @Override - public Map getMappedTelemetry(CalculatedFieldCtx ctx) { + public Map getMappedTelemetry(CalculatedFieldCtx ctx, EntityId referencedEntityId) { Map mappedKvEntries = new HashMap<>(); Map, String> referencedKeys = ctx.getReferencedEntityKeys(); @@ -70,7 +60,7 @@ public class CalculatedFieldAttributeUpdateRequest implements CalculatedFieldTel ReferencedEntityKey referencedEntityKey = new ReferencedEntityKey(key, ArgumentType.ATTRIBUTE, scope); - String argName = referencedKeys.get(new TbPair<>(entityId, referencedEntityKey)); + String argName = referencedKeys.get(new TbPair<>(referencedEntityId, referencedEntityKey)); if (argName != null) { mappedKvEntries.put(argName, entry); @@ -79,5 +69,4 @@ public class CalculatedFieldAttributeUpdateRequest implements CalculatedFieldTel return mappedKvEntries; } - } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/telemetry/CalculatedFieldTelemetryUpdateRequest.java b/application/src/main/java/org/thingsboard/server/service/cf/telemetry/CalculatedFieldTelemetryUpdateRequest.java index f85117dc41..3f7250f4ef 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/telemetry/CalculatedFieldTelemetryUpdateRequest.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/telemetry/CalculatedFieldTelemetryUpdateRequest.java @@ -15,7 +15,6 @@ */ package org.thingsboard.server.service.cf.telemetry; -import org.thingsboard.server.common.data.cf.CalculatedFieldLinkConfiguration; import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; @@ -35,8 +34,6 @@ public interface CalculatedFieldTelemetryUpdateRequest { List getPreviousCalculatedFieldIds(); - Map getTelemetryKeysFromLink(CalculatedFieldLinkConfiguration linkConfiguration); - - Map getMappedTelemetry(CalculatedFieldCtx ctx); + Map getMappedTelemetry(CalculatedFieldCtx ctx, EntityId referencedEntityId); } diff --git a/application/src/main/java/org/thingsboard/server/service/cf/telemetry/CalculatedFieldTimeSeriesUpdateRequest.java b/application/src/main/java/org/thingsboard/server/service/cf/telemetry/CalculatedFieldTimeSeriesUpdateRequest.java index 646145a46e..a5637c8cfd 100644 --- a/application/src/main/java/org/thingsboard/server/service/cf/telemetry/CalculatedFieldTimeSeriesUpdateRequest.java +++ b/application/src/main/java/org/thingsboard/server/service/cf/telemetry/CalculatedFieldTimeSeriesUpdateRequest.java @@ -18,7 +18,6 @@ package org.thingsboard.server.service.cf.telemetry; import lombok.AllArgsConstructor; import lombok.Data; import org.thingsboard.rule.engine.api.TimeseriesSaveRequest; -import org.thingsboard.server.common.data.cf.CalculatedFieldLinkConfiguration; import org.thingsboard.server.common.data.cf.configuration.ArgumentType; import org.thingsboard.server.common.data.cf.configuration.ReferencedEntityKey; import org.thingsboard.server.common.data.id.CalculatedFieldId; @@ -49,12 +48,7 @@ public class CalculatedFieldTimeSeriesUpdateRequest implements CalculatedFieldTe } @Override - public Map getTelemetryKeysFromLink(CalculatedFieldLinkConfiguration linkConfiguration) { - return linkConfiguration.getTimeSeries(); - } - - @Override - public Map getMappedTelemetry(CalculatedFieldCtx ctx) { + public Map getMappedTelemetry(CalculatedFieldCtx ctx, EntityId referencedEntityId) { Map mappedKvEntries = new HashMap<>(); Map, String> referencedKeys = ctx.getReferencedEntityKeys(); @@ -62,13 +56,13 @@ public class CalculatedFieldTimeSeriesUpdateRequest implements CalculatedFieldTe String key = entry.getKey(); ReferencedEntityKey tsLatestKey = new ReferencedEntityKey(key, ArgumentType.TS_LATEST, null); - String argTsLatestName = referencedKeys.get(new TbPair<>(entityId, tsLatestKey)); + String argTsLatestName = referencedKeys.get(new TbPair<>(referencedEntityId, tsLatestKey)); if (argTsLatestName != null) { mappedKvEntries.put(argTsLatestName, entry); } else { ReferencedEntityKey tsRollingKey = new ReferencedEntityKey(key, ArgumentType.TS_ROLLING, null); - String argTsRollingName = referencedKeys.get(new TbPair<>(entityId, tsRollingKey)); + String argTsRollingName = referencedKeys.get(new TbPair<>(referencedEntityId, tsRollingKey)); if (argTsRollingName != null) { mappedKvEntries.put(argTsRollingName, entry); @@ -78,5 +72,4 @@ public class CalculatedFieldTimeSeriesUpdateRequest implements CalculatedFieldTe return mappedKvEntries; } - }