Browse Source

Merge with origin

pull/12487/head
Andrii Shvaika 2 years ago
parent
commit
22339199dc
  1. 10
      application/src/main/java/org/thingsboard/server/controller/BaseController.java
  2. 43
      application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldCache.java
  3. 13
      application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldExecutionService.java
  4. 9
      application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldResult.java
  5. 212
      application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldCache.java
  6. 687
      application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java
  7. 5
      application/src/main/java/org/thingsboard/server/service/cf/ctx/CalculatedFieldEntityCtx.java
  8. 12
      application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ArgumentEntry.java
  9. 2
      application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ArgumentEntryType.java
  10. 43
      application/src/main/java/org/thingsboard/server/service/cf/ctx/state/BaseCalculatedFieldState.java
  11. 10
      application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java
  12. 7
      application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldState.java
  13. 37
      application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldState.java
  14. 37
      application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldState.java
  15. 41
      application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SingleValueArgumentEntry.java
  16. 56
      application/src/main/java/org/thingsboard/server/service/cf/ctx/state/TsRollingArgumentEntry.java
  17. 38
      application/src/main/java/org/thingsboard/server/service/cf/telemetry/CalculatedFieldAttributeUpdateRequest.java
  18. 38
      application/src/main/java/org/thingsboard/server/service/cf/telemetry/CalculatedFieldTelemetryUpdateRequest.java
  19. 42
      application/src/main/java/org/thingsboard/server/service/cf/telemetry/CalculatedFieldTimeSeriesUpdateRequest.java
  20. 18
      application/src/main/java/org/thingsboard/server/service/entitiy/cf/DefaultTbCalculatedFieldService.java
  21. 36
      application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java
  22. 16
      application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java
  23. 4
      application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java
  24. 6
      application/src/test/java/org/thingsboard/server/controller/CalculatedFieldControllerTest.java
  25. 4
      common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedFieldLinkConfiguration.java
  26. 2
      common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/Argument.java
  27. 22
      common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/ArgumentType.java
  28. 45
      common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/BaseCalculatedFieldConfiguration.java
  29. 2
      common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/Output.java
  30. 22
      common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/OutputType.java
  31. 45
      common/message/src/main/java/org/thingsboard/server/common/msg/TbMsg.java
  32. 7
      common/message/src/main/proto/tbmsg.proto
  33. 40
      common/proto/src/main/java/org/thingsboard/server/common/util/ProtoUtils.java
  34. 46
      common/proto/src/main/proto/queue.proto
  35. 6
      dao/src/test/java/org/thingsboard/server/dao/service/AssetServiceTest.java
  36. 6
      dao/src/test/java/org/thingsboard/server/dao/service/CalculatedFieldServiceTest.java
  37. 6
      dao/src/test/java/org/thingsboard/server/dao/service/CustomerServiceTest.java
  38. 6
      dao/src/test/java/org/thingsboard/server/dao/service/DeviceServiceTest.java
  39. 10
      rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/AttributesSaveRequest.java
  40. 10
      rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/TimeseriesSaveRequest.java
  41. 1
      rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/TbMsgAttributesNode.java
  42. 1
      rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/TbMsgTimeseriesNode.java

10
application/src/main/java/org/thingsboard/server/controller/BaseController.java

@ -366,6 +366,9 @@ public abstract class BaseController {
@Autowired @Autowired
protected TbServiceInfoProvider serviceInfoProvider; protected TbServiceInfoProvider serviceInfoProvider;
@Autowired
protected CalculatedFieldService calculatedFieldService;
@Autowired @Autowired
protected NotificationTargetService notificationTargetService; protected NotificationTargetService notificationTargetService;
@ -990,10 +993,15 @@ public abstract class BaseController {
} }
return new HomeDashboardInfo(dashboardId, hideDashboardToolbar); return new HomeDashboardInfo(dashboardId, hideDashboardToolbar);
} }
} catch (Exception ignored) {} } catch (Exception ignored) {
}
return null; return null;
} }
protected CalculatedField checkCalculatedFieldId(CalculatedFieldId calculatedFieldId, Operation operation) throws ThingsboardException {
return checkEntityId(calculatedFieldId, calculatedFieldService::findById, operation);
}
protected MediaType parseMediaType(String contentType) { protected MediaType parseMediaType(String contentType) {
try { try {
return MediaType.parseMediaType(contentType); return MediaType.parseMediaType(contentType);

43
application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldCache.java

@ -0,0 +1,43 @@
/**
* Copyright © 2016-2024 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.service.cf;
import org.thingsboard.script.api.tbel.TbelInvokeService;
import org.thingsboard.server.common.data.cf.CalculatedField;
import org.thingsboard.server.common.data.cf.CalculatedFieldLink;
import org.thingsboard.server.common.data.id.CalculatedFieldId;
import org.thingsboard.server.common.data.id.EntityId;
import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldCtx;
import java.util.List;
import java.util.Set;
public interface CalculatedFieldCache {
CalculatedField getCalculatedField(TenantId tenantId, CalculatedFieldId calculatedFieldId);
List<CalculatedFieldLink> getCalculatedFieldLinks(TenantId tenantId, CalculatedFieldId calculatedFieldId);
List<CalculatedFieldLink> getCalculatedFieldLinksByEntityId(TenantId tenantId, EntityId entityId);
CalculatedFieldCtx getCalculatedFieldCtx(TenantId tenantId, CalculatedFieldId calculatedFieldId, TbelInvokeService tbelInvokeService);
Set<EntityId> getEntitiesByProfile(TenantId tenantId, EntityId entityId);
void evict(CalculatedFieldId calculatedFieldId);
}

13
application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldExecutionService.java

@ -15,22 +15,19 @@
*/ */
package org.thingsboard.server.service.cf; package org.thingsboard.server.service.cf;
import org.thingsboard.server.common.data.id.CalculatedFieldId;
import org.thingsboard.server.common.data.id.EntityId;
import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.data.kv.KvEntry;
import org.thingsboard.server.common.msg.queue.TbCallback; import org.thingsboard.server.common.msg.queue.TbCallback;
import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.gen.transport.TransportProtos;
import org.thingsboard.server.service.cf.telemetry.CalculatedFieldTelemetryUpdateRequest;
import java.util.Map;
public interface CalculatedFieldExecutionService { public interface CalculatedFieldExecutionService {
void onCalculatedFieldMsg(TransportProtos.CalculatedFieldMsgProto proto, TbCallback callback); void onCalculatedFieldMsg(TransportProtos.CalculatedFieldMsgProto proto, TbCallback callback);
void onTelemetryUpdate(TenantId tenantId, EntityId entityId, CalculatedFieldId calculatedFieldId, Map<String, KvEntry> updatedTelemetry); void onTelemetryUpdate(CalculatedFieldTelemetryUpdateRequest calculatedFieldTelemetryUpdateRequest);
void onCalculatedFieldStateMsg(TransportProtos.CalculatedFieldStateMsgProto proto, TbCallback callback);
void onEntityProfileChanged(TransportProtos.EntityProfileUpdateMsgProto proto, TbCallback callback); void onEntityProfileChangedMsg(TransportProtos.EntityProfileUpdateMsgProto proto, TbCallback callback);
void onProfileEntityMsg(TransportProtos.ProfileEntityMsgProto proto, TbCallback callback); void onProfileEntityMsg(TransportProtos.ProfileEntityMsgProto proto, TbCallback callback);

9
application/src/main/java/org/thingsboard/server/service/cf/CalculatedFieldResult.java

@ -17,17 +17,18 @@ package org.thingsboard.server.service.cf;
import lombok.Data; import lombok.Data;
import org.thingsboard.server.common.data.AttributeScope; import org.thingsboard.server.common.data.AttributeScope;
import org.thingsboard.server.common.data.cf.configuration.OutputType;
import java.util.Map; import java.util.Map;
@Data @Data
public final class CalculatedFieldResult { public final class CalculatedFieldResult {
private String type; private final OutputType type;
private AttributeScope scope; private final AttributeScope scope;
private Map<String, Object> resultMap; private final Map<String, Object> resultMap;
public CalculatedFieldResult(String type, AttributeScope scope, Map<String, Object> resultMap) { public CalculatedFieldResult(OutputType type, AttributeScope scope, Map<String, Object> resultMap) {
this.type = type; this.type = type;
this.scope = scope; this.scope = scope;
this.resultMap = resultMap == null ? Map.of() : Map.copyOf(resultMap); this.resultMap = resultMap == null ? Map.of() : Map.copyOf(resultMap);

212
application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldCache.java

@ -0,0 +1,212 @@
/**
* Copyright © 2016-2024 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.service.cf;
import jakarta.annotation.PostConstruct;
import lombok.Getter;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Service;
import org.thingsboard.script.api.tbel.TbelInvokeService;
import org.thingsboard.server.common.data.cf.CalculatedField;
import org.thingsboard.server.common.data.cf.CalculatedFieldLink;
import org.thingsboard.server.common.data.id.AssetProfileId;
import org.thingsboard.server.common.data.id.CalculatedFieldId;
import org.thingsboard.server.common.data.id.DeviceProfileId;
import org.thingsboard.server.common.data.id.EntityId;
import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.data.page.PageDataIterable;
import org.thingsboard.server.dao.asset.AssetService;
import org.thingsboard.server.dao.cf.CalculatedFieldService;
import org.thingsboard.server.dao.device.DeviceService;
import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldCtx;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap;
import java.util.concurrent.locks.Lock;
import java.util.concurrent.locks.ReentrantLock;
@Service
@Slf4j
@RequiredArgsConstructor
public class DefaultCalculatedFieldCache implements CalculatedFieldCache {
private final Lock calculatedFieldFetchLock = new ReentrantLock();
private final CalculatedFieldService calculatedFieldService;
private final AssetService assetService;
private final DeviceService deviceService;
private final ConcurrentMap<CalculatedFieldId, CalculatedField> calculatedFields = new ConcurrentHashMap<>();
private final ConcurrentMap<CalculatedFieldId, List<CalculatedFieldLink>> calculatedFieldLinks = new ConcurrentHashMap<>();
private final ConcurrentMap<EntityId, List<CalculatedFieldLink>> entityIdCalculatedFieldLinks = new ConcurrentHashMap<>();
private final ConcurrentMap<CalculatedFieldId, CalculatedFieldCtx> calculatedFieldsCtx = new ConcurrentHashMap<>();
private final ConcurrentMap<EntityId, Set<EntityId>> profileEntities = new ConcurrentHashMap<>();
@Value("${calculatedField.initFetchPackSize:50000}")
@Getter
private int initFetchPackSize;
@PostConstruct
public void init() {
// to discuss: fetch on start or fetch on demand
PageDataIterable<CalculatedField> cfs = new PageDataIterable<>(calculatedFieldService::findAllCalculatedFields, initFetchPackSize);
cfs.forEach(cf -> calculatedFields.putIfAbsent(cf.getId(), cf));
PageDataIterable<CalculatedFieldLink> cfls = new PageDataIterable<>(calculatedFieldService::findAllCalculatedFieldLinks, initFetchPackSize);
cfls.forEach(link -> calculatedFieldLinks.computeIfAbsent(link.getCalculatedFieldId(), id -> new ArrayList<>()).add(link));
}
@Override
public CalculatedField getCalculatedField(TenantId tenantId, CalculatedFieldId calculatedFieldId) {
CalculatedField calculatedField = calculatedFields.get(calculatedFieldId);
if (calculatedField == null) {
calculatedFieldFetchLock.lock();
try {
calculatedField = calculatedFields.get(calculatedFieldId);
if (calculatedField == null) {
calculatedField = calculatedFieldService.findById(tenantId, calculatedFieldId);
if (calculatedField != null) {
calculatedFields.put(calculatedFieldId, calculatedField);
log.debug("[{}] Fetch calculated field into cache: {}", calculatedFieldId, calculatedField);
}
}
} finally {
calculatedFieldFetchLock.unlock();
}
}
log.trace("[{}] Found calculated field in cache: {}", calculatedFieldId, calculatedField);
return calculatedField;
}
@Override
public List<CalculatedFieldLink> getCalculatedFieldLinks(TenantId tenantId, CalculatedFieldId calculatedFieldId) {
List<CalculatedFieldLink> cfLinks = calculatedFieldLinks.get(calculatedFieldId);
if (cfLinks == null || cfLinks.isEmpty()) {
calculatedFieldFetchLock.lock();
try {
cfLinks = calculatedFieldLinks.get(calculatedFieldId);
if (cfLinks == null || cfLinks.isEmpty()) {
cfLinks = calculatedFieldService.findAllCalculatedFieldLinksById(tenantId, calculatedFieldId);
if (cfLinks != null) {
calculatedFieldLinks.put(calculatedFieldId, cfLinks);
log.debug("[{}] Fetch calculated field links into cache: {}", calculatedFieldId, cfLinks);
}
}
} finally {
calculatedFieldFetchLock.unlock();
}
}
log.trace("[{}] Found calculated field links in cache: {}", calculatedFieldId, cfLinks);
return cfLinks;
}
@Override
public List<CalculatedFieldLink> getCalculatedFieldLinksByEntityId(TenantId tenantId, EntityId entityId) {
List<CalculatedFieldLink> cfLinks = entityIdCalculatedFieldLinks.get(entityId);
if (cfLinks == null || cfLinks.isEmpty()) {
calculatedFieldFetchLock.lock();
try {
cfLinks = entityIdCalculatedFieldLinks.get(entityId);
if (cfLinks == null || cfLinks.isEmpty()) {
cfLinks = calculatedFieldService.findAllCalculatedFieldLinksByEntityId(tenantId, entityId);
if (cfLinks != null) {
entityIdCalculatedFieldLinks.put(entityId, cfLinks);
log.debug("[{}] Fetch calculated field links by entity id into cache: {}", entityId, cfLinks);
}
}
} finally {
calculatedFieldFetchLock.unlock();
}
}
log.trace("[{}] Found calculated field links by entity id in cache: {}", entityId, cfLinks);
return cfLinks;
}
@Override
public CalculatedFieldCtx getCalculatedFieldCtx(TenantId tenantId, CalculatedFieldId calculatedFieldId, TbelInvokeService tbelInvokeService) {
CalculatedFieldCtx ctx = calculatedFieldsCtx.get(calculatedFieldId);
if (ctx == null) {
calculatedFieldFetchLock.lock();
try {
ctx = calculatedFieldsCtx.get(calculatedFieldId);
if (ctx == null) {
CalculatedField calculatedField = getCalculatedField(tenantId, calculatedFieldId);
if (calculatedField != null) {
ctx = new CalculatedFieldCtx(calculatedField, tbelInvokeService);
calculatedFieldsCtx.put(calculatedFieldId, ctx);
log.debug("[{}] Put calculated field ctx into cache: {}", calculatedFieldId, ctx);
}
}
} finally {
calculatedFieldFetchLock.unlock();
}
}
log.trace("[{}] Found calculated field ctx in cache: {}", calculatedFieldId, ctx);
return ctx;
}
@Override
public Set<EntityId> getEntitiesByProfile(TenantId tenantId, EntityId entityProfileId) {
Set<EntityId> entities = profileEntities.get(entityProfileId);
if (entities == null) {
calculatedFieldFetchLock.lock();
try {
entities = profileEntities.get(entityProfileId);
if (entities == null) {
entities = switch (entityProfileId.getEntityType()) {
case ASSET_PROFILE -> profileEntities.computeIfAbsent(entityProfileId, profileId -> {
Set<EntityId> assetIds = new HashSet<>();
(new PageDataIterable<>(pageLink ->
assetService.findAssetIdsByTenantIdAndAssetProfileId(tenantId, (AssetProfileId) profileId, pageLink), initFetchPackSize)).forEach(assetIds::add);
return assetIds;
});
case DEVICE_PROFILE -> profileEntities.computeIfAbsent(entityProfileId, profileId -> {
Set<EntityId> deviceIds = new HashSet<>();
(new PageDataIterable<>(pageLink ->
deviceService.findDeviceIdsByTenantIdAndDeviceProfileId(tenantId, (DeviceProfileId) entityProfileId, pageLink), initFetchPackSize)).forEach(deviceIds::add);
return deviceIds;
});
default ->
throw new IllegalArgumentException("Entity type should be ASSET_PROFILE or DEVICE_PROFILE.");
};
}
} finally {
calculatedFieldFetchLock.unlock();
}
}
log.trace("[{}] Found entities by profile in cache: {}", entityProfileId, entities);
return entities;
}
@Override
public void evict(CalculatedFieldId calculatedFieldId) {
CalculatedField oldCalculatedField = calculatedFields.remove(calculatedFieldId);
log.debug("[{}] evict calculated field from cache: {}", calculatedFieldId, oldCalculatedField);
calculatedFieldLinks.remove(calculatedFieldId);
log.debug("[{}] evict calculated field links from cache: {}", calculatedFieldId, oldCalculatedField);
calculatedFieldsCtx.remove(calculatedFieldId);
log.debug("[{}] evict calculated field ctx from cache: {}", calculatedFieldId, oldCalculatedField);
entityIdCalculatedFieldLinks.forEach((entityId, calculatedFieldLinks) -> calculatedFieldLinks.removeIf(link -> link.getCalculatedFieldId().equals(calculatedFieldId)));
log.debug("[{}] evict calculated field links from cached links by entity id: {}", calculatedFieldId, oldCalculatedField);
}
}

687
application/src/main/java/org/thingsboard/server/service/cf/DefaultCalculatedFieldExecutionService.java

@ -17,6 +17,7 @@ package org.thingsboard.server.service.cf;
import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.node.ObjectNode; import com.fasterxml.jackson.databind.node.ObjectNode;
import com.google.common.collect.Lists;
import com.google.common.util.concurrent.FutureCallback; import com.google.common.util.concurrent.FutureCallback;
import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.Futures;
import com.google.common.util.concurrent.ListenableFuture; import com.google.common.util.concurrent.ListenableFuture;
@ -34,14 +35,18 @@ import org.thingsboard.common.util.JacksonUtil;
import org.thingsboard.common.util.ThingsBoardExecutors; import org.thingsboard.common.util.ThingsBoardExecutors;
import org.thingsboard.script.api.tbel.TbelInvokeService; import org.thingsboard.script.api.tbel.TbelInvokeService;
import org.thingsboard.server.cluster.TbClusterService; import org.thingsboard.server.cluster.TbClusterService;
import org.thingsboard.server.common.data.AttributeScope;
import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.EntityType;
import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.cf.CalculatedField;
import org.thingsboard.server.common.data.cf.CalculatedFieldLink;
import org.thingsboard.server.common.data.cf.CalculatedFieldType; import org.thingsboard.server.common.data.cf.CalculatedFieldType;
import org.thingsboard.server.common.data.cf.configuration.Argument; import org.thingsboard.server.common.data.cf.configuration.Argument;
import org.thingsboard.server.common.data.cf.configuration.ArgumentType;
import org.thingsboard.server.common.data.cf.configuration.CalculatedFieldConfiguration; import org.thingsboard.server.common.data.cf.configuration.CalculatedFieldConfiguration;
import org.thingsboard.server.common.data.id.AssetProfileId; import org.thingsboard.server.common.data.cf.configuration.OutputType;
import org.thingsboard.server.common.data.id.AssetId;
import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.CalculatedFieldId;
import org.thingsboard.server.common.data.id.DeviceProfileId; import org.thingsboard.server.common.data.id.DeviceId;
import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.EntityId;
import org.thingsboard.server.common.data.id.EntityIdFactory; import org.thingsboard.server.common.data.id.EntityIdFactory;
import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.id.TenantId;
@ -59,12 +64,11 @@ import org.thingsboard.server.common.data.msg.TbMsgType;
import org.thingsboard.server.common.data.page.PageDataIterable; import org.thingsboard.server.common.data.page.PageDataIterable;
import org.thingsboard.server.common.msg.TbMsg; import org.thingsboard.server.common.msg.TbMsg;
import org.thingsboard.server.common.msg.TbMsgMetaData; import org.thingsboard.server.common.msg.TbMsgMetaData;
import org.thingsboard.server.common.msg.queue.ServiceType;
import org.thingsboard.server.common.msg.queue.TbCallback; import org.thingsboard.server.common.msg.queue.TbCallback;
import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; import org.thingsboard.server.common.msg.queue.TopicPartitionInfo;
import org.thingsboard.server.dao.asset.AssetService;
import org.thingsboard.server.dao.attributes.AttributesService; import org.thingsboard.server.dao.attributes.AttributesService;
import org.thingsboard.server.dao.cf.CalculatedFieldService; import org.thingsboard.server.dao.cf.CalculatedFieldService;
import org.thingsboard.server.dao.device.DeviceService;
import org.thingsboard.server.dao.timeseries.TimeseriesService; import org.thingsboard.server.dao.timeseries.TimeseriesService;
import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.gen.transport.TransportProtos;
import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.queue.util.TbCoreComponent;
@ -75,23 +79,31 @@ import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldCtx;
import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldState; import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldState;
import org.thingsboard.server.service.cf.ctx.state.ScriptCalculatedFieldState; import org.thingsboard.server.service.cf.ctx.state.ScriptCalculatedFieldState;
import org.thingsboard.server.service.cf.ctx.state.SimpleCalculatedFieldState; import org.thingsboard.server.service.cf.ctx.state.SimpleCalculatedFieldState;
import org.thingsboard.server.service.cf.ctx.state.SingleValueArgumentEntry;
import org.thingsboard.server.service.cf.ctx.state.TsRollingArgumentEntry;
import org.thingsboard.server.service.cf.telemetry.CalculatedFieldTelemetryUpdateRequest;
import org.thingsboard.server.service.partition.AbstractPartitionBasedService; import org.thingsboard.server.service.partition.AbstractPartitionBasedService;
import org.thingsboard.server.service.profile.TbAssetProfileCache;
import org.thingsboard.server.service.profile.TbDeviceProfileCache;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.Collections; import java.util.EnumSet;
import java.util.HashMap; import java.util.HashMap;
import java.util.HashSet;
import java.util.List; import java.util.List;
import java.util.Map; import java.util.Map;
import java.util.Optional; import java.util.Optional;
import java.util.Set; import java.util.Set;
import java.util.UUID; import java.util.UUID;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap; import java.util.concurrent.ConcurrentMap;
import java.util.function.Consumer; import java.util.function.Consumer;
import java.util.stream.Collectors; import java.util.stream.Collectors;
import java.util.stream.Stream;
import static org.thingsboard.server.common.data.DataConstants.SCOPE; import static org.thingsboard.server.common.data.DataConstants.SCOPE;
import static org.thingsboard.server.common.util.ProtoUtils.fromObjectProto;
import static org.thingsboard.server.common.util.ProtoUtils.toObjectProto;
@TbCoreComponent @TbCoreComponent
@Service @Service
@ -100,8 +112,9 @@ import static org.thingsboard.server.common.data.DataConstants.SCOPE;
public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBasedService<CalculatedFieldId> implements CalculatedFieldExecutionService { public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBasedService<CalculatedFieldId> implements CalculatedFieldExecutionService {
private final CalculatedFieldService calculatedFieldService; private final CalculatedFieldService calculatedFieldService;
private final AssetService assetService; private final TbAssetProfileCache assetProfileCache;
private final DeviceService deviceService; private final TbDeviceProfileCache deviceProfileCache;
private final CalculatedFieldCache calculatedFieldCache;
private final AttributesService attributesService; private final AttributesService attributesService;
private final TimeseriesService timeseriesService; private final TimeseriesService timeseriesService;
private final RocksDBService rocksDBService; private final RocksDBService rocksDBService;
@ -111,14 +124,14 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas
private ListeningExecutorService calculatedFieldExecutor; private ListeningExecutorService calculatedFieldExecutor;
private ListeningExecutorService calculatedFieldCallbackExecutor; private ListeningExecutorService calculatedFieldCallbackExecutor;
private final ConcurrentMap<CalculatedFieldId, CalculatedField> calculatedFields = new ConcurrentHashMap<>();
private final ConcurrentMap<CalculatedFieldId, CalculatedFieldCtx> calculatedFieldsCtx = new ConcurrentHashMap<>();
private final ConcurrentMap<CalculatedFieldEntityCtxId, CalculatedFieldEntityCtx> states = new ConcurrentHashMap<>(); private final ConcurrentMap<CalculatedFieldEntityCtxId, CalculatedFieldEntityCtx> states = new ConcurrentHashMap<>();
private final ConcurrentMap<EntityId, Set<EntityId>> profileEntities = new ConcurrentHashMap<>();
private static final int MAX_LAST_RECORDS_VALUE = 1024; private static final int MAX_LAST_RECORDS_VALUE = 1024;
private static final Set<EntityType> supportedReferencedEntities = EnumSet.of(
EntityType.DEVICE, EntityType.ASSET, EntityType.CUSTOMER, EntityType.TENANT
);
@Value("${calculatedField.initFetchPackSize:50000}") @Value("${calculatedField.initFetchPackSize:50000}")
@Getter @Getter
private int initFetchPackSize; private int initFetchPackSize;
@ -134,6 +147,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas
@PreDestroy @PreDestroy
public void stop() { public void stop() {
super.stop();
if (calculatedFieldExecutor != null) { if (calculatedFieldExecutor != null) {
calculatedFieldExecutor.shutdownNow(); calculatedFieldExecutor.shutdownNow();
} }
@ -154,13 +168,69 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas
@Override @Override
protected Map<TopicPartitionInfo, List<ListenableFuture<?>>> onAddedPartitions(Set<TopicPartitionInfo> addedPartitions) { protected Map<TopicPartitionInfo, List<ListenableFuture<?>>> onAddedPartitions(Set<TopicPartitionInfo> addedPartitions) {
// TODO: implementation for cluster mode var result = new HashMap<TopicPartitionInfo, List<ListenableFuture<?>>>();
return Map.of(); PageDataIterable<CalculatedField> cfs = new PageDataIterable<>(calculatedFieldService::findAllCalculatedFields, initFetchPackSize);
Map<TopicPartitionInfo, List<CalculatedField>> tpiCalculatedFieldMap = new HashMap<>();
for (CalculatedField cf : cfs) {
TopicPartitionInfo tpi;
try {
tpi = partitionService.resolve(ServiceType.TB_CORE, cf.getTenantId(), cf.getId());
} catch (Exception e) {
log.warn("Failed to resolve partition for CalculatedField [{}], tenant id [{}]. Reason: {}",
cf.getId(), cf.getTenantId(), e.getMessage());
continue;
}
if (addedPartitions.contains(tpi) && states.keySet().stream().noneMatch(ctxId -> ctxId.cfId().equals(cf.getId().getId()))) {
tpiCalculatedFieldMap.computeIfAbsent(tpi, k -> new ArrayList<>()).add(cf);
}
}
for (var entry : tpiCalculatedFieldMap.entrySet()) {
for (List<CalculatedField> partition : Lists.partition(entry.getValue(), 1000)) {
log.info("[{}] Submit task for CalculatedFields: {}", entry.getKey(), partition.size());
var future = calculatedFieldExecutor.submit(() -> {
try {
for (CalculatedField cf : partition) {
EntityId cfEntityId = cf.getEntityId();
if (isProfileEntity(cfEntityId)) {
calculatedFieldCache.getEntitiesByProfile(cf.getTenantId(), cfEntityId)
.forEach(entityId -> restoreState(cf, entityId));
} else {
restoreState(cf, cfEntityId);
}
}
} catch (Throwable t) {
log.error("Unexpected exception while restoring CalculatedField states", t);
throw t;
}
});
result.computeIfAbsent(entry.getKey(), k -> new ArrayList<>()).add(future);
}
}
return result;
}
private void restoreState(CalculatedField cf, EntityId entityId) {
CalculatedFieldEntityCtxId ctxId = new CalculatedFieldEntityCtxId(cf.getId().getId(), entityId.getId());
String storedState = rocksDBService.get(JacksonUtil.writeValueAsString(ctxId));
if (storedState != null) {
CalculatedFieldEntityCtx restoredCtx = JacksonUtil.fromString(storedState, CalculatedFieldEntityCtx.class);
states.put(ctxId, restoredCtx);
log.info("Restored state for CalculatedField [{}]", cf.getId());
} else {
log.warn("No state found for CalculatedField [{}], entity [{}].", cf.getId(), entityId);
}
} }
@Override @Override
protected void cleanupEntityOnPartitionRemoval(CalculatedFieldId entityId) { protected void cleanupEntityOnPartitionRemoval(CalculatedFieldId entityId) {
// TODO: implementation for cluster mode cleanupEntity(entityId);
}
private void cleanupEntity(CalculatedFieldId calculatedFieldId) {
states.keySet().removeIf(ctxId -> ctxId.cfId().equals(calculatedFieldId.getId()));
} }
@Override @Override
@ -169,12 +239,18 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas
TenantId tenantId = TenantId.fromUUID(new UUID(proto.getTenantIdMSB(), proto.getTenantIdLSB())); TenantId tenantId = TenantId.fromUUID(new UUID(proto.getTenantIdMSB(), proto.getTenantIdLSB()));
CalculatedFieldId calculatedFieldId = new CalculatedFieldId(new UUID(proto.getCalculatedFieldIdMSB(), proto.getCalculatedFieldIdLSB())); CalculatedFieldId calculatedFieldId = new CalculatedFieldId(new UUID(proto.getCalculatedFieldIdMSB(), proto.getCalculatedFieldIdLSB()));
log.info("Received CalculatedFieldMsgProto for processing: tenantId=[{}], calculatedFieldId=[{}]", tenantId, calculatedFieldId); log.info("Received CalculatedFieldMsgProto for processing: tenantId=[{}], calculatedFieldId=[{}]", tenantId, calculatedFieldId);
TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_CORE, tenantId, calculatedFieldId);
if (!tpi.isMyPartition()) {
clusterService.pushMsgToCore(tenantId, calculatedFieldId, TransportProtos.ToCoreMsg.newBuilder().setCalculatedFieldMsg(proto).build(), null);
log.debug("[{}][{}] Calculated field belongs to external partition. Probably rebalancing is in progress. Topic: {}", tenantId, calculatedFieldId, tpi.getFullTopicName());
callback.onFailure(new RuntimeException("Calculated field belongs to external partition " + tpi.getFullTopicName() + "!"));
}
if (proto.getDeleted()) { if (proto.getDeleted()) {
log.warn("Executing onCalculatedFieldDelete, calculatedFieldId=[{}]", calculatedFieldId); log.warn("Executing onCalculatedFieldDelete, calculatedFieldId=[{}]", calculatedFieldId);
onCalculatedFieldDelete(calculatedFieldId, callback); onCalculatedFieldDelete(tenantId, calculatedFieldId, callback);
callback.onSuccess(); callback.onSuccess();
} }
CalculatedField cf = getOrFetchFromDb(tenantId, calculatedFieldId); CalculatedField cf = calculatedFieldCache.getCalculatedField(tenantId, calculatedFieldId);
if (proto.getUpdated()) { if (proto.getUpdated()) {
log.info("Executing onCalculatedFieldUpdate, calculatedFieldId=[{}]", calculatedFieldId); log.info("Executing onCalculatedFieldUpdate, calculatedFieldId=[{}]", calculatedFieldId);
boolean shouldReinit = onCalculatedFieldUpdate(cf, callback); boolean shouldReinit = onCalculatedFieldUpdate(cf, callback);
@ -184,8 +260,7 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas
} }
if (cf != null) { if (cf != null) {
EntityId entityId = cf.getEntityId(); EntityId entityId = cf.getEntityId();
CalculatedFieldCtx calculatedFieldCtx = new CalculatedFieldCtx(cf, tbelInvokeService); CalculatedFieldCtx calculatedFieldCtx = calculatedFieldCache.getCalculatedFieldCtx(tenantId, calculatedFieldId, tbelInvokeService);
calculatedFieldsCtx.put(calculatedFieldId, calculatedFieldCtx);
switch (entityId.getEntityType()) { switch (entityId.getEntityType()) {
case ASSET, DEVICE -> { case ASSET, DEVICE -> {
log.info("Initializing state for entity: tenantId=[{}], entityId=[{}]", tenantId, entityId); log.info("Initializing state for entity: tenantId=[{}], entityId=[{}]", tenantId, entityId);
@ -193,9 +268,12 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas
} }
case ASSET_PROFILE, DEVICE_PROFILE -> { case ASSET_PROFILE, DEVICE_PROFILE -> {
log.info("Initializing state for all entities in profile: tenantId=[{}], profileId=[{}]", tenantId, entityId); log.info("Initializing state for all entities in profile: tenantId=[{}], profileId=[{}]", tenantId, entityId);
fetchCommonArguments(calculatedFieldCtx, callback, commonArguments -> { Map<String, Argument> commonArguments = calculatedFieldCtx.getArguments().entrySet().stream()
getOrFetchFromDBProfileEntities(tenantId, entityId).forEach(assetId -> { .filter(entry -> !isProfileEntity(entry.getValue().getEntityId()))
initializeStateForEntity(calculatedFieldCtx, assetId, commonArguments, callback); .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue));
fetchArguments(tenantId, entityId, commonArguments, commonArgs -> {
calculatedFieldCache.getEntitiesByProfile(tenantId, entityId).forEach(targetEntityId -> {
initializeStateForEntity(calculatedFieldCtx, targetEntityId, commonArgs, callback);
}); });
}); });
} }
@ -215,35 +293,157 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas
} }
} }
private boolean onCalculatedFieldUpdate(CalculatedField updatedCalculatedField, TbCallback callback) {
CalculatedField oldCalculatedField = calculatedFieldCache.getCalculatedField(updatedCalculatedField.getTenantId(), updatedCalculatedField.getId());
boolean shouldReinit = true;
if (hasSignificantChanges(oldCalculatedField, updatedCalculatedField)) {
onCalculatedFieldDelete(updatedCalculatedField.getTenantId(), updatedCalculatedField.getId(), callback);
} else {
callback.onSuccess();
shouldReinit = false;
}
return shouldReinit;
}
private void onCalculatedFieldDelete(TenantId tenantId, CalculatedFieldId calculatedFieldId, TbCallback callback) {
try {
cleanupEntity(calculatedFieldId);
TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_CORE, tenantId, calculatedFieldId);
Set<CalculatedFieldId> calculatedFieldIds = partitionedEntities.get(tpi);
if (calculatedFieldIds != null) {
calculatedFieldIds.remove(calculatedFieldId);
}
calculatedFieldCache.evict(calculatedFieldId);
states.keySet().removeIf(ctxId -> ctxId.cfId().equals(calculatedFieldId.getId()));
List<String> statesToRemove = states.keySet().stream()
.filter(ctxId -> ctxId.cfId().equals(calculatedFieldId.getId()))
.map(JacksonUtil::writeValueAsString)
.toList();
rocksDBService.deleteAll(statesToRemove);
} catch (Exception e) {
log.trace("Failed to delete calculated field: [{}]", calculatedFieldId, e);
callback.onFailure(e);
}
}
private boolean hasSignificantChanges(CalculatedField oldCalculatedField, CalculatedField newCalculatedField) {
if (oldCalculatedField == null) {
return true;
}
boolean entityIdChanged = !oldCalculatedField.getEntityId().equals(newCalculatedField.getEntityId());
boolean typeChanged = !oldCalculatedField.getType().equals(newCalculatedField.getType());
CalculatedFieldConfiguration oldConfig = oldCalculatedField.getConfiguration();
CalculatedFieldConfiguration newConfig = newCalculatedField.getConfiguration();
boolean argumentsChanged = !oldConfig.getArguments().equals(newConfig.getArguments());
boolean outputTypeChanged = !oldConfig.getOutput().getType().equals(newConfig.getOutput().getType());
boolean expressionChanged = !oldConfig.getExpression().equals(newConfig.getExpression());
return entityIdChanged || typeChanged || argumentsChanged || outputTypeChanged || expressionChanged;
}
@Override @Override
public void onTelemetryUpdate(TenantId tenantId, EntityId entityId, CalculatedFieldId calculatedFieldId, Map<String, KvEntry> updatedTelemetry) { public void onTelemetryUpdate(CalculatedFieldTelemetryUpdateRequest calculatedFieldTelemetryUpdateRequest) {
try { try {
log.info("Received telemetry update msg: tenantId=[{}], calculatedFieldId=[{}]", tenantId, calculatedFieldId); TenantId tenantId = calculatedFieldTelemetryUpdateRequest.getTenantId();
CalculatedField calculatedField = getOrFetchFromDb(tenantId, calculatedFieldId); EntityId entityId = calculatedFieldTelemetryUpdateRequest.getEntityId();
CalculatedFieldCtx calculatedFieldCtx = calculatedFieldsCtx.computeIfAbsent(calculatedFieldId, id -> new CalculatedFieldCtx(calculatedField, tbelInvokeService)); AttributeScope scope = calculatedFieldTelemetryUpdateRequest.getScope();
Map<String, ArgumentEntry> argumentValues = updatedTelemetry.entrySet().stream() List<? extends KvEntry> telemetry = calculatedFieldTelemetryUpdateRequest.getKvEntries();
.collect(Collectors.toMap(Map.Entry::getKey, entry -> ArgumentEntry.createSingleValueArgument(entry.getValue()))); List<CalculatedFieldId> calculatedFieldIds = calculatedFieldTelemetryUpdateRequest.getCalculatedFieldIds();
EntityId cfEntityId = calculatedField.getEntityId(); if (supportedReferencedEntities.contains(entityId.getEntityType())) {
switch (cfEntityId.getEntityType()) { EntityId profileId = getProfileId(tenantId, entityId);
case ASSET_PROFILE, DEVICE_PROFILE -> {
boolean isCommonEntity = calculatedField.getConfiguration().getReferencedEntities().contains(entityId); List<CalculatedFieldLink> cfLinks = Stream.concat(
if (isCommonEntity) { calculatedFieldCache.getCalculatedFieldLinksByEntityId(tenantId, entityId).stream(),
getOrFetchFromDBProfileEntities(tenantId, cfEntityId).forEach(id -> updateOrInitializeState(calculatedFieldCtx, id, argumentValues)); profileId != null ? calculatedFieldCache.getCalculatedFieldLinksByEntityId(tenantId, profileId).stream() : Stream.empty()
} else { ).toList();
updateOrInitializeState(calculatedFieldCtx, entityId, argumentValues);
cfLinks.forEach(link -> {
CalculatedFieldId calculatedFieldId = link.getCalculatedFieldId();
Map<String, String> telemetryKeys = getTelemetryKeysFromLink(link, scope);
Map<String, KvEntry> updatedTelemetry = telemetry.stream()
.filter(entry -> telemetryKeys.containsValue(entry.getKey()))
.collect(Collectors.toMap(
entry -> getMappedKey(entry, telemetryKeys),
entry -> entry,
(v1, v2) -> v1
));
if (!updatedTelemetry.isEmpty()) {
executeTelemetryUpdate(tenantId, entityId, calculatedFieldId, calculatedFieldIds, updatedTelemetry);
} }
});
}
} catch (Exception e) {
log.trace("Failed to update telemetry.", e);
}
}
private Map<String, String> getTelemetryKeysFromLink(CalculatedFieldLink link, AttributeScope scope) {
return scope == null ? link.getConfiguration().getTimeSeries() : switch (scope) {
case CLIENT_SCOPE -> link.getConfiguration().getClientAttributes();
case SERVER_SCOPE -> link.getConfiguration().getServerAttributes();
case SHARED_SCOPE -> link.getConfiguration().getSharedAttributes();
};
}
private String getMappedKey(KvEntry entry, Map<String, String> telemetry) {
return telemetry.entrySet().stream()
.filter(kvEntry -> kvEntry.getValue().equals(entry.getKey()))
.map(Map.Entry::getKey)
.findFirst()
.orElse(entry.getKey());
}
private void executeTelemetryUpdate(TenantId tenantId, EntityId entityId, CalculatedFieldId calculatedFieldId, List<CalculatedFieldId> calculatedFieldIds, Map<String, KvEntry> updatedTelemetry) {
log.info("Received telemetry update msg: tenantId=[{}], entityId=[{}], calculatedFieldId=[{}]", tenantId, entityId, calculatedFieldId);
CalculatedField calculatedField = calculatedFieldCache.getCalculatedField(tenantId, calculatedFieldId);
CalculatedFieldCtx calculatedFieldCtx = calculatedFieldCache.getCalculatedFieldCtx(tenantId, calculatedFieldId, tbelInvokeService);
Map<String, ArgumentEntry> argumentValues = updatedTelemetry.entrySet().stream()
.collect(Collectors.toMap(Map.Entry::getKey, entry -> ArgumentEntry.createSingleValueArgument(entry.getValue())));
EntityId cfEntityId = calculatedField.getEntityId();
switch (cfEntityId.getEntityType()) {
case ASSET_PROFILE, DEVICE_PROFILE -> {
boolean isCommonEntity = calculatedField.getConfiguration().getReferencedEntities().contains(entityId);
if (isCommonEntity) {
calculatedFieldCache.getEntitiesByProfile(tenantId, cfEntityId).forEach(id -> updateOrInitializeState(calculatedFieldCtx, id, argumentValues, calculatedFieldIds));
} else {
updateOrInitializeState(calculatedFieldCtx, entityId, argumentValues, calculatedFieldIds);
} }
default -> updateOrInitializeState(calculatedFieldCtx, cfEntityId, argumentValues);
} }
log.info("Successfully updated telemetry for calculatedFieldId: [{}]", calculatedFieldId); default -> updateOrInitializeState(calculatedFieldCtx, cfEntityId, argumentValues, calculatedFieldIds);
}
log.info("Successfully updated telemetry for calculatedFieldId: [{}]", calculatedFieldId);
}
@Override
public void onCalculatedFieldStateMsg(TransportProtos.CalculatedFieldStateMsgProto proto, TbCallback callback) {
try {
TenantId tenantId = TenantId.fromUUID(new UUID(proto.getTenantIdMSB(), proto.getTenantIdLSB()));
CalculatedFieldId calculatedFieldId = new CalculatedFieldId(new UUID(proto.getCalculatedFieldIdMSB(), proto.getCalculatedFieldIdLSB()));
EntityId entityId = EntityIdFactory.getByTypeAndUuid(proto.getEntityType(), new UUID(proto.getEntityIdMSB(), proto.getEntityIdLSB()));
if (proto.getClear()) {
clearState(tenantId, calculatedFieldId, entityId);
return;
}
List<CalculatedFieldId> calculatedFieldIds = proto.getCalculatedFieldsList().stream()
.map(cfIdProto -> new CalculatedFieldId(new UUID(cfIdProto.getCalculatedFieldIdMSB(), cfIdProto.getCalculatedFieldIdLSB())))
.toList();
Map<String, ArgumentEntry> argumentsMap = proto.getArgumentsMap().entrySet().stream()
.collect(Collectors.toMap(Map.Entry::getKey, entry -> fromArgumentEntryProto(entry.getValue())));
CalculatedFieldCtx calculatedFieldCtx = calculatedFieldCache.getCalculatedFieldCtx(tenantId, calculatedFieldId, tbelInvokeService);
updateOrInitializeState(calculatedFieldCtx, entityId, argumentsMap, calculatedFieldIds);
} catch (Exception e) { } catch (Exception e) {
log.trace("Failed to update telemetry for calculatedFieldId: [{}]", calculatedFieldId, e); log.trace("Failed to process calculated field update state msg: [{}]", proto, e);
} }
} }
@Override @Override
public void onEntityProfileChanged(TransportProtos.EntityProfileUpdateMsgProto proto, TbCallback callback) { public void onEntityProfileChangedMsg(TransportProtos.EntityProfileUpdateMsgProto proto, TbCallback callback) {
try { try {
TenantId tenantId = TenantId.fromUUID(new UUID(proto.getTenantIdMSB(), proto.getTenantIdLSB())); TenantId tenantId = TenantId.fromUUID(new UUID(proto.getTenantIdMSB(), proto.getTenantIdLSB()));
EntityId entityId = EntityIdFactory.getByTypeAndUuid(proto.getEntityType(), new UUID(proto.getEntityIdMSB(), proto.getEntityIdLSB())); EntityId entityId = EntityIdFactory.getByTypeAndUuid(proto.getEntityType(), new UUID(proto.getEntityIdMSB(), proto.getEntityIdLSB()));
@ -251,15 +451,11 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas
EntityId newProfileId = EntityIdFactory.getByTypeAndUuid(proto.getEntityProfileType(), new UUID(proto.getNewProfileIdMSB(), proto.getNewProfileIdLSB())); EntityId newProfileId = EntityIdFactory.getByTypeAndUuid(proto.getEntityProfileType(), new UUID(proto.getNewProfileIdMSB(), proto.getNewProfileIdLSB()));
log.info("Received EntityProfileUpdateMsgProto for processing: tenantId=[{}], entityId=[{}]", tenantId, entityId); log.info("Received EntityProfileUpdateMsgProto for processing: tenantId=[{}], entityId=[{}]", tenantId, entityId);
profileEntities.get(oldProfileId).remove(entityId); calculatedFieldCache.getEntitiesByProfile(tenantId, oldProfileId).remove(entityId);
profileEntities.computeIfAbsent(newProfileId, id -> new HashSet<>()).add(entityId); calculatedFieldCache.getEntitiesByProfile(tenantId, newProfileId).add(entityId);
calculatedFieldService.findCalculatedFieldIdsByEntityId(tenantId, oldProfileId) calculatedFieldService.findCalculatedFieldIdsByEntityId(tenantId, oldProfileId)
.forEach(cfId -> { .forEach(cfId -> clearState(tenantId, cfId, entityId));
CalculatedFieldEntityCtxId ctxId = new CalculatedFieldEntityCtxId(cfId.getId(), entityId.getId());
states.remove(ctxId);
rocksDBService.delete(JacksonUtil.writeValueAsString(ctxId));
});
initializeStateForEntityByProfile(tenantId, entityId, newProfileId, callback); initializeStateForEntityByProfile(tenantId, entityId, newProfileId, callback);
} catch (Exception e) { } catch (Exception e) {
@ -276,16 +472,15 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas
log.info("Received ProfileEntityMsgProto for processing: tenantId=[{}], entityId=[{}]", tenantId, entityId); log.info("Received ProfileEntityMsgProto for processing: tenantId=[{}], entityId=[{}]", tenantId, entityId);
if (proto.getDeleted()) { if (proto.getDeleted()) {
log.info("Executing profile entity deleted msg, tenantId=[{}], entityId=[{}]", tenantId, entityId); log.info("Executing profile entity deleted msg, tenantId=[{}], entityId=[{}]", tenantId, entityId);
profileEntities.get(profileId).remove(entityId); calculatedFieldCache.getEntitiesByProfile(tenantId, profileId).remove(entityId);
List<String> statesToRemove = states.keySet().stream() List<CalculatedFieldId> calculatedFieldIds = Stream.concat(
.filter(ctxEntityId -> ctxEntityId.entityId().equals(entityId.getId())) calculatedFieldCache.getCalculatedFieldLinksByEntityId(tenantId, entityId).stream().map(CalculatedFieldLink::getCalculatedFieldId),
.map(JacksonUtil::writeValueAsString) calculatedFieldCache.getCalculatedFieldLinksByEntityId(tenantId, profileId).stream().map(CalculatedFieldLink::getCalculatedFieldId)
.toList(); ).toList();
states.keySet().removeIf(ctxEntityId -> ctxEntityId.entityId().equals(entityId.getId())); calculatedFieldIds.forEach(cfId -> clearState(tenantId, cfId, entityId));
rocksDBService.deleteAll(statesToRemove);
} else { } else {
log.info("Executing profile entity added msg, tenantId=[{}], entityId=[{}]", tenantId, entityId); log.info("Executing profile entity added msg, tenantId=[{}], entityId=[{}]", tenantId, entityId);
profileEntities.computeIfAbsent(profileId, id -> new HashSet<>()).add(entityId); calculatedFieldCache.getEntitiesByProfile(tenantId, profileId).add(entityId);
initializeStateForEntityByProfile(tenantId, entityId, profileId, callback); initializeStateForEntityByProfile(tenantId, entityId, profileId, callback);
} }
} catch (Exception e) { } catch (Exception e) {
@ -293,88 +488,36 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas
} }
} }
private boolean onCalculatedFieldUpdate(CalculatedField updatedCalculatedField, TbCallback callback) { private void clearState(TenantId tenantId, CalculatedFieldId calculatedFieldId, EntityId entityId) {
CalculatedField oldCalculatedField = getOrFetchFromDb(updatedCalculatedField.getTenantId(), updatedCalculatedField.getId()); TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_CORE, tenantId, calculatedFieldId);
boolean shouldReinit = true; if (tpi.isMyPartition()) {
if (hasSignificantChanges(oldCalculatedField, updatedCalculatedField)) { log.warn("Executing clearState, calculatedFieldId=[{}], entityId=[{}]", calculatedFieldId, entityId);
onCalculatedFieldDelete(updatedCalculatedField.getId(), callback); CalculatedFieldEntityCtxId ctxId = new CalculatedFieldEntityCtxId(calculatedFieldId.getId(), entityId.getId());
states.remove(ctxId);
rocksDBService.delete(JacksonUtil.writeValueAsString(ctxId));
} else { } else {
calculatedFields.put(updatedCalculatedField.getId(), updatedCalculatedField); sendClearCalculatedFieldStateMsg(tenantId, calculatedFieldId, entityId);
calculatedFieldsCtx.put(updatedCalculatedField.getId(), new CalculatedFieldCtx(updatedCalculatedField, tbelInvokeService));
callback.onSuccess();
shouldReinit = false;
}
return shouldReinit;
}
private void onCalculatedFieldDelete(CalculatedFieldId calculatedFieldId, TbCallback callback) {
try {
calculatedFields.remove(calculatedFieldId);
calculatedFieldsCtx.remove(calculatedFieldId);
states.keySet().removeIf(ctxId -> ctxId.cfId().equals(calculatedFieldId.getId()));
List<String> statesToRemove = states.keySet().stream()
.filter(ctxId -> ctxId.cfId().equals(calculatedFieldId.getId()))
.map(JacksonUtil::writeValueAsString)
.toList();
rocksDBService.deleteAll(statesToRemove);
} catch (Exception e) {
log.trace("Failed to delete calculated field: [{}]", calculatedFieldId, e);
callback.onFailure(e);
} }
} }
private CalculatedField getOrFetchFromDb(TenantId tenantId, CalculatedFieldId calculatedFieldId) {
return calculatedFields.computeIfAbsent(calculatedFieldId, cfId -> calculatedFieldService.findById(tenantId, calculatedFieldId));
}
private Set<EntityId> getOrFetchFromDBProfileEntities(TenantId tenantId, EntityId entityProfileId) {
return switch (entityProfileId.getEntityType()) {
case ASSET_PROFILE -> profileEntities.computeIfAbsent(entityProfileId, profileId -> {
Set<EntityId> assetIds = new HashSet<>();
(new PageDataIterable<>(pageLink ->
assetService.findAssetIdsByTenantIdAndAssetProfileId(tenantId, (AssetProfileId) profileId, pageLink), initFetchPackSize)).forEach(assetIds::add);
return assetIds;
});
case DEVICE_PROFILE -> profileEntities.computeIfAbsent(entityProfileId, profileId -> {
Set<EntityId> deviceIds = new HashSet<>();
(new PageDataIterable<>(pageLink ->
deviceService.findDeviceIdsByTenantIdAndDeviceProfileId(tenantId, (DeviceProfileId) entityProfileId, pageLink), initFetchPackSize)).forEach(deviceIds::add);
return deviceIds;
});
default -> throw new IllegalArgumentException("Entity type should be ASSET_PROFILE or DEVICE_PROFILE.");
};
}
private boolean hasSignificantChanges(CalculatedField oldCalculatedField, CalculatedField newCalculatedField) {
if (oldCalculatedField == null) {
return true;
}
boolean entityIdChanged = !oldCalculatedField.getEntityId().equals(newCalculatedField.getEntityId());
boolean typeChanged = !oldCalculatedField.getType().equals(newCalculatedField.getType());
CalculatedFieldConfiguration oldConfig = oldCalculatedField.getConfiguration();
CalculatedFieldConfiguration newConfig = newCalculatedField.getConfiguration();
boolean argumentsChanged = !oldConfig.getArguments().equals(newConfig.getArguments());
boolean outputTypeChanged = !oldConfig.getOutput().getType().equals(newConfig.getOutput().getType());
boolean expressionChanged = !oldConfig.getExpression().equals(newConfig.getExpression());
return entityIdChanged || typeChanged || argumentsChanged || outputTypeChanged || expressionChanged;
}
private void initializeStateForEntityByProfile(TenantId tenantId, EntityId entityId, EntityId profileId, TbCallback callback) { private void initializeStateForEntityByProfile(TenantId tenantId, EntityId entityId, EntityId profileId, TbCallback callback) {
calculatedFieldService.findCalculatedFieldIdsByEntityId(tenantId, profileId) calculatedFieldService.findCalculatedFieldIdsByEntityId(tenantId, profileId)
.stream() .stream()
.map(cfId -> calculatedFieldsCtx.computeIfAbsent(cfId, id -> new CalculatedFieldCtx(calculatedFieldService.findById(tenantId, id), tbelInvokeService))) .map(cfId -> calculatedFieldCache.getCalculatedFieldCtx(tenantId, cfId, tbelInvokeService))
.forEach(cfCtx -> initializeStateForEntity(cfCtx, entityId, callback)); .forEach(cfCtx -> initializeStateForEntity(cfCtx, entityId, callback));
} }
private void fetchCommonArguments(CalculatedFieldCtx calculatedFieldCtx, TbCallback callback, Consumer<Map<String, ArgumentEntry>> onComplete) { private void initializeStateForEntity(CalculatedFieldCtx calculatedFieldCtx, EntityId entityId, TbCallback callback) {
Map<String, ArgumentEntry> argumentValues = new HashMap<>(); initializeStateForEntity(calculatedFieldCtx, entityId, new HashMap<>(), callback);
}
private void initializeStateForEntity(CalculatedFieldCtx calculatedFieldCtx, EntityId entityId, Map<String, ArgumentEntry> commonArguments, TbCallback callback) {
Map<String, ArgumentEntry> argumentValues = new HashMap<>(commonArguments);
List<ListenableFuture<ArgumentEntry>> futures = new ArrayList<>(); List<ListenableFuture<ArgumentEntry>> futures = new ArrayList<>();
calculatedFieldCtx.getArguments().forEach((key, argument) -> { calculatedFieldCtx.getArguments().forEach((key, argument) -> {
if (!EntityType.DEVICE_PROFILE.equals(argument.getEntityId().getEntityType()) && if (!commonArguments.containsKey(key)) {
!EntityType.ASSET_PROFILE.equals(argument.getEntityId().getEntityType())) { futures.add(Futures.transform(fetchArgumentValue(calculatedFieldCtx.getTenantId(), entityId, argument),
futures.add(Futures.transform(fetchKvEntry(calculatedFieldCtx.getTenantId(), argument.getEntityId(), argument),
result -> { result -> {
argumentValues.put(key, result); argumentValues.put(key, result);
return result; return result;
@ -385,54 +528,135 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas
Futures.addCallback(Futures.allAsList(futures), new FutureCallback<>() { Futures.addCallback(Futures.allAsList(futures), new FutureCallback<>() {
@Override @Override
public void onSuccess(List<ArgumentEntry> results) { public void onSuccess(List<ArgumentEntry> results) {
onComplete.accept(argumentValues); updateOrInitializeState(calculatedFieldCtx, entityId, argumentValues, new ArrayList<>());
callback.onSuccess();
} }
@Override @Override
public void onFailure(Throwable t) { public void onFailure(Throwable t) {
log.error("Failed to fetch common arguments", t); log.error("Failed to initialize state for entity: [{}]", entityId, t);
callback.onFailure(t); callback.onFailure(t);
} }
}, calculatedFieldCallbackExecutor); }, calculatedFieldCallbackExecutor);
} }
private void initializeStateForEntity(CalculatedFieldCtx calculatedFieldCtx, EntityId entityId, TbCallback callback) { private void updateOrInitializeState(CalculatedFieldCtx calculatedFieldCtx, EntityId entityId, Map<String, ArgumentEntry> argumentValues, List<CalculatedFieldId> calculatedFieldIds) {
initializeStateForEntity(calculatedFieldCtx, entityId, new HashMap<>(), callback); TenantId tenantId = calculatedFieldCtx.getTenantId();
} CalculatedFieldId cfId = calculatedFieldCtx.getCfId();
Map<String, ArgumentEntry> argumentsMap = new HashMap<>(argumentValues);
private void initializeStateForEntity(CalculatedFieldCtx calculatedFieldCtx, EntityId entityId, Map<String, ArgumentEntry> commonArguments, TbCallback callback) { TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_CORE, tenantId, cfId);
Map<String, ArgumentEntry> argumentValues = new HashMap<>(commonArguments); if (tpi.isMyPartition()) {
List<ListenableFuture<ArgumentEntry>> futures = new ArrayList<>();
calculatedFieldCtx.getArguments().forEach((key, argument) -> { CalculatedFieldEntityCtxId entityCtxId = new CalculatedFieldEntityCtxId(cfId.getId(), entityId.getId());
if (!commonArguments.containsKey(key)) {
futures.add(Futures.transform(fetchArgumentValue(calculatedFieldCtx, entityId, argument),
result -> {
argumentValues.put(key, result);
return result;
}, calculatedFieldCallbackExecutor));
}
});
Futures.addCallback(Futures.allAsList(futures), new FutureCallback<>() { states.compute(entityCtxId, (ctxId, ctx) -> {
CalculatedFieldEntityCtx calculatedFieldEntityCtx = ctx != null ? ctx : fetchCalculatedFieldEntityState(ctxId, calculatedFieldCtx.getCfType());
CompletableFuture<Void> updateFuture = new CompletableFuture<>();
Consumer<CalculatedFieldState> performUpdateState = (state) -> {
if (state.updateState(argumentsMap)) {
calculatedFieldEntityCtx.setState(state);
rocksDBService.put(JacksonUtil.writeValueAsString(entityCtxId), JacksonUtil.writeValueAsString(calculatedFieldEntityCtx));
Map<String, ArgumentEntry> arguments = state.getArguments();
boolean allArgsPresent = arguments.keySet().containsAll(calculatedFieldCtx.getArguments().keySet()) &&
!arguments.containsValue(SingleValueArgumentEntry.EMPTY) && !arguments.containsValue(TsRollingArgumentEntry.EMPTY);
if (allArgsPresent) {
performCalculation(calculatedFieldCtx, state, entityId, calculatedFieldIds);
}
}
updateFuture.complete(null);
};
CalculatedFieldState state = calculatedFieldEntityCtx.getState();
boolean allKeysPresent = argumentsMap.keySet().containsAll(calculatedFieldCtx.getArguments().keySet());
boolean requiresTsRollingUpdate = calculatedFieldCtx.getArguments().values().stream()
.anyMatch(argument -> ArgumentType.TS_ROLLING.equals(argument.getType()) && state.getArguments().get(argument.getKey()) == null);
if (!allKeysPresent || requiresTsRollingUpdate) {
Map<String, Argument> missingArguments = calculatedFieldCtx.getArguments().entrySet().stream()
.filter(entry -> !argumentsMap.containsKey(entry.getKey()) || (ArgumentType.TS_ROLLING.equals(entry.getValue().getType()) && state.getArguments().get(entry.getKey()) == null))
.collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue));
fetchArguments(calculatedFieldCtx.getTenantId(), entityId, missingArguments, argumentsMap::putAll)
.addListener(() -> performUpdateState.accept(state),
calculatedFieldCallbackExecutor);
} else {
performUpdateState.accept(state);
}
try {
updateFuture.join();
} catch (Exception e) {
log.trace("Failed to update state for ctxId [{}].", ctxId, e);
throw new RuntimeException("Failed to update or initialize state.", e);
}
return calculatedFieldEntityCtx;
});
} else {
sendUpdateCalculatedFieldStateMsg(tenantId, cfId, entityId, calculatedFieldIds, argumentsMap);
}
}
private void performCalculation(CalculatedFieldCtx calculatedFieldCtx, CalculatedFieldState state, EntityId entityId, List<CalculatedFieldId> calculatedFieldIds) {
ListenableFuture<CalculatedFieldResult> resultFuture = state.performCalculation(calculatedFieldCtx);
Futures.addCallback(resultFuture, new FutureCallback<>() {
@Override @Override
public void onSuccess(List<ArgumentEntry> results) { public void onSuccess(CalculatedFieldResult result) {
updateOrInitializeState(calculatedFieldCtx, entityId, argumentValues); if (result != null) {
callback.onSuccess(); pushMsgToRuleEngine(calculatedFieldCtx.getTenantId(), calculatedFieldCtx.getCfId(), entityId, result, calculatedFieldIds);
}
} }
@Override @Override
public void onFailure(Throwable t) { public void onFailure(Throwable t) {
log.error("Failed to initialize state for entity: [{}]", entityId, t); log.warn("[{}] Failed to perform calculation. entityId: [{}]", calculatedFieldCtx.getCfId(), entityId, t);
callback.onFailure(t);
} }
}, MoreExecutors.directExecutor());
}
private void pushMsgToRuleEngine(TenantId tenantId, CalculatedFieldId calculatedFieldId, EntityId originatorId, CalculatedFieldResult calculatedFieldResult, List<CalculatedFieldId> calculatedFieldIds) {
try {
OutputType type = calculatedFieldResult.getType();
TbMsgType msgType = OutputType.ATTRIBUTES.equals(type) ? TbMsgType.POST_ATTRIBUTES_REQUEST : TbMsgType.POST_TELEMETRY_REQUEST;
TbMsgMetaData md = OutputType.ATTRIBUTES.equals(type) ? new TbMsgMetaData(Map.of(SCOPE, calculatedFieldResult.getScope().name())) : TbMsgMetaData.EMPTY;
ObjectNode payload = createJsonPayload(calculatedFieldResult);
if (calculatedFieldIds == null) {
calculatedFieldIds = new ArrayList<>();
}
if (calculatedFieldIds.contains(calculatedFieldId)) {
throw new IllegalArgumentException("Calculated field [" + calculatedFieldId.getId() + "] refers to itself, causing an infinite loop.");
}
calculatedFieldIds.add(calculatedFieldId);
TbMsg msg = TbMsg.newMsg().type(msgType).originator(originatorId).calculatedFieldIds(calculatedFieldIds).metaData(md).data(JacksonUtil.writeValueAsString(payload)).build();
clusterService.pushMsgToRuleEngine(tenantId, originatorId, msg, null);
} catch (Exception e) {
log.warn("[{}] Failed to push message to rule engine. CalculatedFieldResult: {}", originatorId, calculatedFieldResult, e);
}
}
private ListenableFuture<Void> fetchArguments(TenantId tenantId, EntityId entityId, Map<String, Argument> necessaryArguments, Consumer<Map<String, ArgumentEntry>> onComplete) {
Map<String, ArgumentEntry> argumentValues = new HashMap<>();
List<ListenableFuture<ArgumentEntry>> futures = new ArrayList<>();
necessaryArguments.forEach((key, argument) -> {
futures.add(Futures.transform(fetchArgumentValue(tenantId, entityId, argument),
result -> {
argumentValues.put(key, result);
return result;
}, calculatedFieldCallbackExecutor));
});
return Futures.transform(Futures.allAsList(futures), results -> {
onComplete.accept(argumentValues);
return null;
}, calculatedFieldCallbackExecutor); }, calculatedFieldCallbackExecutor);
} }
private ListenableFuture<ArgumentEntry> fetchArgumentValue(CalculatedFieldCtx calculatedFieldCtx, EntityId targetEntityId, Argument argument) { private ListenableFuture<ArgumentEntry> fetchArgumentValue(TenantId tenantId, EntityId targetEntityId, Argument argument) {
TenantId tenantId = calculatedFieldCtx.getTenantId();
EntityId argumentEntityId = argument.getEntityId(); EntityId argumentEntityId = argument.getEntityId();
EntityId entityId = EntityType.DEVICE_PROFILE.equals(argumentEntityId.getEntityType()) || EntityType.ASSET_PROFILE.equals(argumentEntityId.getEntityType()) EntityId entityId = isProfileEntity(argumentEntityId)
? targetEntityId ? targetEntityId
: argumentEntityId; : argumentEntityId;
return fetchKvEntry(tenantId, entityId, argument); return fetchKvEntry(tenantId, entityId, argument);
@ -440,22 +664,31 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas
private ListenableFuture<ArgumentEntry> fetchKvEntry(TenantId tenantId, EntityId entityId, Argument argument) { private ListenableFuture<ArgumentEntry> fetchKvEntry(TenantId tenantId, EntityId entityId, Argument argument) {
return switch (argument.getType()) { return switch (argument.getType()) {
case "TS_ROLLING" -> fetchTsRolling(tenantId, entityId, argument); case TS_ROLLING -> fetchTsRolling(tenantId, entityId, argument);
case "ATTRIBUTE" -> transformSingleValueArgument( case ATTRIBUTE -> transformSingleValueArgument(
Futures.transform( Futures.transform(
attributesService.find(tenantId, entityId, argument.getScope(), argument.getKey()), attributesService.find(tenantId, entityId, argument.getScope(), argument.getKey()),
result -> result.or(() -> Optional.of(new BaseAttributeKvEntry(System.currentTimeMillis(), createDefaultKvEntry(argument)))), result -> result.or(() -> Optional.of(new BaseAttributeKvEntry(createDefaultKvEntry(argument), System.currentTimeMillis(), 0L))),
calculatedFieldCallbackExecutor) calculatedFieldCallbackExecutor)
); );
case "TS_LATEST" -> transformSingleValueArgument( case TS_LATEST -> transformSingleValueArgument(
Futures.transform( Futures.transform(
timeseriesService.findLatest(tenantId, entityId, argument.getKey()), timeseriesService.findLatest(tenantId, entityId, argument.getKey()),
result -> result.or(() -> Optional.of(new BasicTsKvEntry(System.currentTimeMillis(), createDefaultKvEntry(argument)))), result -> result.or(() -> Optional.of(new BasicTsKvEntry(System.currentTimeMillis(), createDefaultKvEntry(argument), 0L))),
calculatedFieldCallbackExecutor)); calculatedFieldCallbackExecutor));
default -> throw new IllegalArgumentException("Invalid argument type '" + argument.getType() + "'.");
}; };
} }
private ListenableFuture<ArgumentEntry> transformSingleValueArgument(ListenableFuture<Optional<? extends KvEntry>> kvEntryFuture) {
return Futures.transform(kvEntryFuture, kvEntry -> {
if (kvEntry.isPresent() && kvEntry.get().getValue() != null) {
return ArgumentEntry.createSingleValueArgument(kvEntry.get());
} else {
return SingleValueArgumentEntry.EMPTY;
}
}, calculatedFieldCallbackExecutor);
}
private ListenableFuture<ArgumentEntry> fetchTsRolling(TenantId tenantId, EntityId entityId, Argument argument) { private ListenableFuture<ArgumentEntry> fetchTsRolling(TenantId tenantId, EntityId entityId, Argument argument) {
long currentTime = System.currentTimeMillis(); long currentTime = System.currentTimeMillis();
long timeWindow = argument.getTimeWindow() == 0 ? System.currentTimeMillis() : argument.getTimeWindow(); long timeWindow = argument.getTimeWindow() == 0 ? System.currentTimeMillis() : argument.getTimeWindow();
@ -465,11 +698,83 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas
ReadTsKvQuery query = new BaseReadTsKvQuery(argument.getKey(), startTs, currentTime, 0, limit, Aggregation.NONE); ReadTsKvQuery query = new BaseReadTsKvQuery(argument.getKey(), startTs, currentTime, 0, limit, Aggregation.NONE);
ListenableFuture<List<TsKvEntry>> tsRollingFuture = timeseriesService.findAll(tenantId, entityId, List.of(query)); ListenableFuture<List<TsKvEntry>> tsRollingFuture = timeseriesService.findAll(tenantId, entityId, List.of(query));
return Futures.transform(tsRollingFuture, tsRolling -> tsRolling == null ? ArgumentEntry.createTsRollingArgument(Collections.emptyList()) : ArgumentEntry.createTsRollingArgument(tsRolling), calculatedFieldCallbackExecutor); return Futures.transform(tsRollingFuture, tsRolling -> tsRolling == null ? TsRollingArgumentEntry.EMPTY : ArgumentEntry.createTsRollingArgument(tsRolling), calculatedFieldCallbackExecutor);
} }
private ListenableFuture<ArgumentEntry> transformSingleValueArgument(ListenableFuture<Optional<? extends KvEntry>> kvEntryFuture) { private void sendUpdateCalculatedFieldStateMsg(TenantId tenantId, CalculatedFieldId calculatedFieldId, EntityId entityId, List<CalculatedFieldId> calculatedFieldIds, Map<String, ArgumentEntry> argumentValues) {
return Futures.transform(kvEntryFuture, kvEntry -> ArgumentEntry.createSingleValueArgument(kvEntry.orElse(null)), calculatedFieldCallbackExecutor); TransportProtos.CalculatedFieldStateMsgProto.Builder msgBuilder = createBaseCalculatedFieldStateMsg(tenantId, calculatedFieldId, entityId);
if (argumentValues != null) {
argumentValues.forEach((key, argumentEntry) -> msgBuilder.putArguments(key, toArgumentEntryProto(argumentEntry)));
}
if (calculatedFieldIds != null) {
calculatedFieldIds.forEach(cfId -> msgBuilder.addCalculatedFields(
TransportProtos.CalculatedFieldIdProto.newBuilder()
.setCalculatedFieldIdMSB(cfId.getId().getMostSignificantBits())
.setCalculatedFieldIdLSB(cfId.getId().getLeastSignificantBits())
.build()
));
}
clusterService.pushMsgToCore(tenantId, calculatedFieldId, TransportProtos.ToCoreMsg.newBuilder().setCalculatedFieldStateMsg(msgBuilder).build(), null);
}
private void sendClearCalculatedFieldStateMsg(TenantId tenantId, CalculatedFieldId calculatedFieldId, EntityId entityId) {
TransportProtos.CalculatedFieldStateMsgProto msg = createBaseCalculatedFieldStateMsg(tenantId, calculatedFieldId, entityId)
.setClear(true)
.build();
clusterService.pushMsgToCore(tenantId, calculatedFieldId, TransportProtos.ToCoreMsg.newBuilder().setCalculatedFieldStateMsg(msg).build(), null);
}
private TransportProtos.CalculatedFieldStateMsgProto.Builder createBaseCalculatedFieldStateMsg(
TenantId tenantId,
CalculatedFieldId calculatedFieldId,
EntityId entityId
) {
return TransportProtos.CalculatedFieldStateMsgProto.newBuilder()
.setTenantIdMSB(tenantId.getId().getMostSignificantBits())
.setTenantIdLSB(tenantId.getId().getLeastSignificantBits())
.setCalculatedFieldIdMSB(calculatedFieldId.getId().getMostSignificantBits())
.setCalculatedFieldIdLSB(calculatedFieldId.getId().getLeastSignificantBits())
.setEntityType(entityId.getEntityType().name())
.setEntityIdMSB(entityId.getId().getMostSignificantBits())
.setEntityIdLSB(entityId.getId().getLeastSignificantBits());
}
private TransportProtos.ArgumentEntryProto toArgumentEntryProto(ArgumentEntry argumentEntry) {
TransportProtos.ArgumentEntryProto.Builder argumentProtoBuilder = TransportProtos.ArgumentEntryProto.newBuilder();
if (argumentEntry instanceof TsRollingArgumentEntry tsRollingArgumentEntry) {
TransportProtos.TsRollingProto.Builder tsRollingProtoBuilder = TransportProtos.TsRollingProto.newBuilder();
tsRollingArgumentEntry.getTsRecords().forEach((ts, value) ->
tsRollingProtoBuilder.putTsRecords(ts, toObjectProto(value))
);
argumentProtoBuilder.setTsRecords(tsRollingProtoBuilder.build());
} else if (argumentEntry instanceof SingleValueArgumentEntry singleValueArgumentEntry) {
argumentProtoBuilder.setSingleValue(
TransportProtos.SingleValueProto.newBuilder()
.setTs(singleValueArgumentEntry.getTs())
.setValue(toObjectProto(singleValueArgumentEntry.getValue()))
.build()
);
}
return argumentProtoBuilder.build();
}
private ArgumentEntry fromArgumentEntryProto(TransportProtos.ArgumentEntryProto entryProto) {
if (entryProto.hasTsRecords()) {
TsRollingArgumentEntry tsRollingArgumentEntry = new TsRollingArgumentEntry();
entryProto.getTsRecords().getTsRecordsMap().forEach((ts, objectProto) ->
tsRollingArgumentEntry.getTsRecords().put(ts, fromObjectProto(objectProto))
);
return tsRollingArgumentEntry;
} else if (entryProto.hasSingleValue()) {
TransportProtos.SingleValueProto singleValueProto = entryProto.getSingleValue();
return new SingleValueArgumentEntry(singleValueProto.getTs(), fromObjectProto(singleValueProto.getValue()), singleValueProto.getVersion());
} else {
throw new IllegalArgumentException("Unsupported ArgumentEntryProto type");
}
} }
private KvEntry createDefaultKvEntry(Argument argument) { private KvEntry createDefaultKvEntry(Argument argument) {
@ -484,56 +789,12 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas
return new StringDataEntry(key, defaultValue); return new StringDataEntry(key, defaultValue);
} }
private void updateOrInitializeState(CalculatedFieldCtx calculatedFieldCtx, EntityId entityId, Map<String, ArgumentEntry> argumentValues) { private CalculatedFieldEntityCtx fetchCalculatedFieldEntityState(CalculatedFieldEntityCtxId entityCtxId, CalculatedFieldType cfType) {
CalculatedFieldEntityCtxId entityCtxId = new CalculatedFieldEntityCtxId(calculatedFieldCtx.getCfId().getId(), entityId.getId());
CalculatedFieldEntityCtx calculatedFieldEntityCtx = states.computeIfAbsent(entityCtxId, this::fetchCalculatedFieldEntityState);
CalculatedFieldState state = calculatedFieldEntityCtx.getState();
if (state == null) {
state = createStateByType(calculatedFieldCtx.getCfType());
}
state.initState(argumentValues);
calculatedFieldEntityCtx.setState(state);
states.put(entityCtxId, calculatedFieldEntityCtx);
rocksDBService.put(JacksonUtil.writeValueAsString(entityCtxId), JacksonUtil.writeValueAsString(calculatedFieldEntityCtx));
ListenableFuture<CalculatedFieldResult> resultFuture = state.performCalculation(calculatedFieldCtx);
Futures.addCallback(resultFuture, new FutureCallback<>() {
@Override
public void onSuccess(CalculatedFieldResult result) {
if (result != null) {
pushMsgToRuleEngine(calculatedFieldCtx.getTenantId(), entityId, result);
}
}
@Override
public void onFailure(Throwable t) {
log.warn("[{}] Failed to perform calculation. entityId: [{}]", calculatedFieldCtx.getCfId(), entityId, t);
}
}, MoreExecutors.directExecutor());
}
private CalculatedFieldEntityCtx fetchCalculatedFieldEntityState(CalculatedFieldEntityCtxId entityCtxId) {
String stateStr = rocksDBService.get(JacksonUtil.writeValueAsString(entityCtxId)); String stateStr = rocksDBService.get(JacksonUtil.writeValueAsString(entityCtxId));
if (stateStr == null) { if (stateStr == null) {
return new CalculatedFieldEntityCtx(entityCtxId, null); return new CalculatedFieldEntityCtx(entityCtxId, createStateByType(cfType));
}
return JacksonUtil.fromString(rocksDBService.get(JacksonUtil.writeValueAsString(entityCtxId)), CalculatedFieldEntityCtx.class);
}
private void pushMsgToRuleEngine(TenantId tenantId, EntityId originatorId, CalculatedFieldResult calculatedFieldResult) {
try {
String type = calculatedFieldResult.getType();
TbMsgType msgType = "ATTRIBUTES".equals(type) ? TbMsgType.POST_ATTRIBUTES_REQUEST : TbMsgType.POST_TELEMETRY_REQUEST;
TbMsgMetaData md = "ATTRIBUTES".equals(type) ? new TbMsgMetaData(Map.of(SCOPE, calculatedFieldResult.getScope().name())) : TbMsgMetaData.EMPTY;
ObjectNode payload = createJsonPayload(calculatedFieldResult);
TbMsg msg = TbMsg.newMsg(msgType, originatorId, md, JacksonUtil.writeValueAsString(payload));
clusterService.pushMsgToRuleEngine(tenantId, originatorId, msg, null);
} catch (Exception e) {
log.warn("[{}] Failed to push message to rule engine. CalculatedFieldResult: {}", originatorId, calculatedFieldResult, e);
} }
return JacksonUtil.fromString(stateStr, CalculatedFieldEntityCtx.class);
} }
private ObjectNode createJsonPayload(CalculatedFieldResult calculatedFieldResult) { private ObjectNode createJsonPayload(CalculatedFieldResult calculatedFieldResult) {
@ -550,4 +811,16 @@ public class DefaultCalculatedFieldExecutionService extends AbstractPartitionBas
}; };
} }
private boolean isProfileEntity(EntityId entityId) {
return EntityType.DEVICE_PROFILE.equals(entityId.getEntityType()) || EntityType.ASSET_PROFILE.equals(entityId.getEntityType());
}
private EntityId getProfileId(TenantId tenantId, EntityId entityId) {
return switch (entityId.getEntityType()) {
case ASSET -> assetProfileCache.get(tenantId, (AssetId) entityId).getId();
case DEVICE -> deviceProfileCache.get(tenantId, (DeviceId) entityId).getId();
default -> null;
};
}
} }

5
application/src/main/java/org/thingsboard/server/service/cf/ctx/CalculatedFieldEntityCtx.java

@ -16,17 +16,16 @@
package org.thingsboard.server.service.cf.ctx; package org.thingsboard.server.service.cf.ctx;
import lombok.Data; import lombok.Data;
import lombok.NoArgsConstructor;
import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldState; import org.thingsboard.server.service.cf.ctx.state.CalculatedFieldState;
@Data @Data
@NoArgsConstructor
public class CalculatedFieldEntityCtx { public class CalculatedFieldEntityCtx {
private CalculatedFieldEntityCtxId id; private CalculatedFieldEntityCtxId id;
private CalculatedFieldState state; private CalculatedFieldState state;
public CalculatedFieldEntityCtx() {
}
public CalculatedFieldEntityCtx(CalculatedFieldEntityCtxId id, CalculatedFieldState state) { public CalculatedFieldEntityCtx(CalculatedFieldEntityCtxId id, CalculatedFieldState state) {
this.id = id; this.id = id;
this.state = state; this.state = state;

12
application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ArgumentEntry.java

@ -22,8 +22,6 @@ import org.thingsboard.server.common.data.kv.KvEntry;
import org.thingsboard.server.common.data.kv.TsKvEntry; import org.thingsboard.server.common.data.kv.TsKvEntry;
import java.util.List; import java.util.List;
import java.util.TreeMap;
import java.util.stream.Collectors;
@JsonTypeInfo( @JsonTypeInfo(
use = JsonTypeInfo.Id.NAME, use = JsonTypeInfo.Id.NAME,
@ -37,17 +35,21 @@ import java.util.stream.Collectors;
public interface ArgumentEntry { public interface ArgumentEntry {
@JsonIgnore @JsonIgnore
ArgumentType getType(); ArgumentEntryType getType();
Object getValue(); Object getValue();
boolean hasUpdatedValue(ArgumentEntry entry);
static ArgumentEntry createSingleValueArgument(KvEntry kvEntry) { static ArgumentEntry createSingleValueArgument(KvEntry kvEntry) {
return new SingleValueArgumentEntry(kvEntry); return new SingleValueArgumentEntry(kvEntry);
} }
static ArgumentEntry createTsRollingArgument(List<TsKvEntry> kvEntries) { static ArgumentEntry createTsRollingArgument(List<TsKvEntry> kvEntries) {
return new TsRollingArgumentEntry(kvEntries.stream(). return new TsRollingArgumentEntry(kvEntries);
collect(Collectors.toMap(TsKvEntry::getTs, TsKvEntry::getValue, (oldValue, newValue) -> newValue, TreeMap::new)));
} }
@JsonIgnore
ArgumentEntry copy();
} }

2
application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ArgumentType.java → application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ArgumentEntryType.java

@ -15,6 +15,6 @@
*/ */
package org.thingsboard.server.service.cf.ctx.state; package org.thingsboard.server.service.cf.ctx.state;
public enum ArgumentType { public enum ArgumentEntryType {
SINGLE_VALUE, TS_ROLLING SINGLE_VALUE, TS_ROLLING
} }

43
application/src/main/java/org/thingsboard/server/service/cf/ctx/state/BaseCalculatedFieldState.java

@ -23,6 +23,7 @@ public abstract class BaseCalculatedFieldState implements CalculatedFieldState {
protected Map<String, ArgumentEntry> arguments; protected Map<String, ArgumentEntry> arguments;
public BaseCalculatedFieldState() { public BaseCalculatedFieldState() {
arguments = new HashMap<>();
} }
@Override @Override
@ -31,26 +32,38 @@ public abstract class BaseCalculatedFieldState implements CalculatedFieldState {
} }
@Override @Override
public void initState(Map<String, ArgumentEntry> argumentValues) { public boolean updateState(Map<String, ArgumentEntry> argumentValues) {
if (arguments == null) { if (arguments == null) {
arguments = new HashMap<>(); arguments = new HashMap<>();
} }
argumentValues.forEach((key, argumentEntry) -> {
ArgumentEntry existingArgumentEntry = arguments.get(key); boolean stateUpdated = false;
if (existingArgumentEntry != null) {
if (existingArgumentEntry instanceof SingleValueArgumentEntry) { for (Map.Entry<String, ArgumentEntry> entry : argumentValues.entrySet()) {
arguments.put(key, argumentEntry); String key = entry.getKey();
} else if (existingArgumentEntry instanceof TsRollingArgumentEntry existingTsRollingArgumentEntry) { ArgumentEntry newEntry = entry.getValue();
if (argumentEntry instanceof TsRollingArgumentEntry tsRollingArgumentEntry) { ArgumentEntry existingEntry = arguments.get(key);
existingTsRollingArgumentEntry.getTsRecords().putAll(tsRollingArgumentEntry.getTsRecords());
} else if (argumentEntry instanceof SingleValueArgumentEntry singleValueArgumentEntry) { if (existingEntry == null || existingEntry.hasUpdatedValue(newEntry)) {
existingTsRollingArgumentEntry.getTsRecords().put(singleValueArgumentEntry.getTs(), singleValueArgumentEntry.getValue()); if (existingEntry instanceof TsRollingArgumentEntry existingTsRollingEntry && newEntry instanceof TsRollingArgumentEntry newTsRollingEntry) {
} existingTsRollingEntry.addAllTsRecords(newTsRollingEntry.getTsRecords());
} else if (existingEntry instanceof TsRollingArgumentEntry existingTsRollingEntry && newEntry instanceof SingleValueArgumentEntry singleValueEntry) {
existingTsRollingEntry.addTsRecord(singleValueEntry.getTs(), singleValueEntry.getValue());
} else if (existingEntry instanceof SingleValueArgumentEntry existingSingleValueEntry && newEntry instanceof SingleValueArgumentEntry singleValueEntry) {
// Long existingVersion = existingSingleValueEntry.getVersion();
// Long newVersion = singleValueEntry.getVersion();
// if (newVersion != null && (existingVersion == null || newVersion > existingVersion)) {
// arguments.put(key, newEntry.copy());
// }
arguments.put(key, newEntry.copy());
} else {
arguments.put(key, newEntry.copy());
} }
} else { stateUpdated = true;
arguments.put(key, argumentEntry);
} }
}); }
return stateUpdated;
} }
} }

10
application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldCtx.java

@ -26,6 +26,8 @@ import org.thingsboard.server.common.data.id.CalculatedFieldId;
import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.EntityId;
import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.id.TenantId;
import java.util.ArrayList;
import java.util.List;
import java.util.Map; import java.util.Map;
@Data @Data
@ -35,7 +37,8 @@ public class CalculatedFieldCtx {
private TenantId tenantId; private TenantId tenantId;
private EntityId entityId; private EntityId entityId;
private CalculatedFieldType cfType; private CalculatedFieldType cfType;
private Map<String, Argument> arguments; private final Map<String, Argument> arguments;
private final List<String> argKeys;
private Output output; private Output output;
private String expression; private String expression;
private TbelInvokeService tbelInvokeService; private TbelInvokeService tbelInvokeService;
@ -48,10 +51,11 @@ public class CalculatedFieldCtx {
this.cfType = calculatedField.getType(); this.cfType = calculatedField.getType();
CalculatedFieldConfiguration configuration = calculatedField.getConfiguration(); CalculatedFieldConfiguration configuration = calculatedField.getConfiguration();
this.arguments = configuration.getArguments(); this.arguments = configuration.getArguments();
this.argKeys = new ArrayList<>(arguments.keySet());
this.output = configuration.getOutput(); this.output = configuration.getOutput();
this.expression = configuration.getExpression(); this.expression = configuration.getExpression();
this.tbelInvokeService = tbelInvokeService; this.tbelInvokeService = tbelInvokeService;
if (!CalculatedFieldType.SIMPLE.equals(calculatedField.getType())) { if (CalculatedFieldType.SCRIPT.equals(calculatedField.getType())) {
this.calculatedFieldScriptEngine = initEngine(tenantId, expression, tbelInvokeService); this.calculatedFieldScriptEngine = initEngine(tenantId, expression, tbelInvokeService);
} }
} }
@ -65,7 +69,7 @@ public class CalculatedFieldCtx {
tenantId, tenantId,
tbelInvokeService, tbelInvokeService,
expression, expression,
arguments.keySet().toArray(new String[0]) argKeys.toArray(String[]::new)
); );
} }

7
application/src/main/java/org/thingsboard/server/service/cf/ctx/state/CalculatedFieldState.java

@ -20,7 +20,6 @@ import com.fasterxml.jackson.annotation.JsonSubTypes;
import com.fasterxml.jackson.annotation.JsonTypeInfo; import com.fasterxml.jackson.annotation.JsonTypeInfo;
import com.google.common.util.concurrent.ListenableFuture; import com.google.common.util.concurrent.ListenableFuture;
import org.thingsboard.server.common.data.cf.CalculatedFieldType; import org.thingsboard.server.common.data.cf.CalculatedFieldType;
import org.thingsboard.server.common.data.cf.configuration.Argument;
import org.thingsboard.server.service.cf.CalculatedFieldResult; import org.thingsboard.server.service.cf.CalculatedFieldResult;
import java.util.Map; import java.util.Map;
@ -41,11 +40,7 @@ public interface CalculatedFieldState {
Map<String, ArgumentEntry> getArguments(); Map<String, ArgumentEntry> getArguments();
default boolean isValid(Map<String, Argument> arguments) { boolean updateState(Map<String, ArgumentEntry> argumentValues);
return getArguments().keySet().containsAll(arguments.keySet());
}
void initState(Map<String, ArgumentEntry> argumentValues);
ListenableFuture<CalculatedFieldResult> performCalculation(CalculatedFieldCtx ctx); ListenableFuture<CalculatedFieldResult> performCalculation(CalculatedFieldCtx ctx);

37
application/src/main/java/org/thingsboard/server/service/cf/ctx/state/ScriptCalculatedFieldState.java

@ -39,26 +39,25 @@ public class ScriptCalculatedFieldState extends BaseCalculatedFieldState {
@Override @Override
public ListenableFuture<CalculatedFieldResult> performCalculation(CalculatedFieldCtx ctx) { public ListenableFuture<CalculatedFieldResult> performCalculation(CalculatedFieldCtx ctx) {
if (isValid(ctx.getArguments())) { arguments.forEach((key, argumentEntry) -> {
arguments.forEach((key, argumentEntry) -> { if (argumentEntry instanceof TsRollingArgumentEntry) {
if (argumentEntry instanceof TsRollingArgumentEntry) { Argument argument = ctx.getArguments().get(key);
Argument argument = ctx.getArguments().get(key); TreeMap<Long, Object> tsRecords = ((TsRollingArgumentEntry) argumentEntry).getTsRecords();
TreeMap<Long, Object> tsRecords = ((TsRollingArgumentEntry) argumentEntry).getTsRecords(); if (tsRecords.size() > argument.getLimit()) {
if (tsRecords.size() > argument.getLimit()) { tsRecords.pollFirstEntry();
tsRecords.pollFirstEntry();
}
tsRecords.entrySet().removeIf(tsRecord -> tsRecord.getKey() < System.currentTimeMillis() - argument.getTimeWindow());
} }
}); tsRecords.entrySet().removeIf(tsRecord -> tsRecord.getKey() < System.currentTimeMillis() - argument.getTimeWindow());
Object[] args = arguments.values().stream().map(ArgumentEntry::getValue).toArray(); }
ListenableFuture<Map<String, Object>> resultFuture = ctx.getCalculatedFieldScriptEngine().executeToMapAsync(args); });
Output output = ctx.getOutput(); Object[] args = ctx.getArgKeys().stream()
return Futures.transform(resultFuture, .map(key -> arguments.get(key).getValue())
result -> new CalculatedFieldResult(output.getType(), output.getScope(), result), .toArray();
MoreExecutors.directExecutor() ListenableFuture<Map<String, Object>> resultFuture = ctx.getCalculatedFieldScriptEngine().executeToMapAsync(args);
); Output output = ctx.getOutput();
} return Futures.transform(resultFuture,
return Futures.immediateFuture(null); result -> new CalculatedFieldResult(output.getType(), output.getScope(), result),
MoreExecutors.directExecutor()
);
} }
} }

37
application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SimpleCalculatedFieldState.java

@ -37,27 +37,24 @@ public class SimpleCalculatedFieldState extends BaseCalculatedFieldState {
@Override @Override
public ListenableFuture<CalculatedFieldResult> performCalculation(CalculatedFieldCtx ctx) { public ListenableFuture<CalculatedFieldResult> performCalculation(CalculatedFieldCtx ctx) {
if (isValid(ctx.getArguments())) { String expression = ctx.getExpression();
String expression = ctx.getExpression(); ThreadLocal<Expression> customExpression = new ThreadLocal<>();
ThreadLocal<Expression> customExpression = new ThreadLocal<>(); var expr = customExpression.get();
var expr = customExpression.get(); if (expr == null) {
if (expr == null) { expr = new ExpressionBuilder(expression)
expr = new ExpressionBuilder(expression) .implicitMultiplication(true)
.implicitMultiplication(true) .variables(this.arguments.keySet())
.variables(this.arguments.keySet()) .build();
.build(); customExpression.set(expr);
customExpression.set(expr);
}
Map<String, Double> variables = new HashMap<>();
this.arguments.forEach((k, v) -> variables.put(k, Double.parseDouble(v.getValue().toString())));
expr.setVariables(variables);
double expressionResult = expr.evaluate();
Output output = ctx.getOutput();
return Futures.immediateFuture(new CalculatedFieldResult(output.getType(), output.getScope(), Map.of(output.getName(), expressionResult)));
} }
return Futures.immediateFuture(null); Map<String, Double> variables = new HashMap<>();
this.arguments.forEach((k, v) -> variables.put(k, Double.parseDouble(v.getValue().toString())));
expr.setVariables(variables);
double expressionResult = expr.evaluate();
Output output = ctx.getOutput();
return Futures.immediateFuture(new CalculatedFieldResult(output.getType(), output.getScope(), Map.of(output.getName(), expressionResult)));
} }
} }

41
application/src/main/java/org/thingsboard/server/service/cf/ctx/state/SingleValueArgumentEntry.java

@ -15,32 +15,47 @@
*/ */
package org.thingsboard.server.service.cf.ctx.state; package org.thingsboard.server.service.cf.ctx.state;
import lombok.AllArgsConstructor;
import lombok.Data; import lombok.Data;
import lombok.NoArgsConstructor;
import org.thingsboard.server.common.data.kv.AttributeKvEntry; import org.thingsboard.server.common.data.kv.AttributeKvEntry;
import org.thingsboard.server.common.data.kv.KvEntry; import org.thingsboard.server.common.data.kv.KvEntry;
import org.thingsboard.server.common.data.kv.TsKvEntry; import org.thingsboard.server.common.data.kv.TsKvEntry;
@Data @Data
@NoArgsConstructor
@AllArgsConstructor
public class SingleValueArgumentEntry implements ArgumentEntry { public class SingleValueArgumentEntry implements ArgumentEntry {
public static final ArgumentEntry EMPTY = new SingleValueArgumentEntry(0);
private long ts; private long ts;
private Object value; private Object value;
public SingleValueArgumentEntry() { private Long version;
}
public SingleValueArgumentEntry(KvEntry entry) { public SingleValueArgumentEntry(KvEntry entry) {
if (entry instanceof TsKvEntry) { if (entry instanceof TsKvEntry tsKvEntry) {
this.ts = ((TsKvEntry) entry).getTs(); this.ts = tsKvEntry.getTs();
} else if (entry instanceof AttributeKvEntry) { this.version = tsKvEntry.getVersion();
this.ts = ((AttributeKvEntry) entry).getLastUpdateTs(); } else if (entry instanceof AttributeKvEntry attributeKvEntry) {
this.ts = attributeKvEntry.getLastUpdateTs();
this.version = attributeKvEntry.getVersion();
} }
this.value = entry.getValue(); this.value = entry.getValue();
} }
/**
* Internal constructor to create immutable SingleValueArgumentEntry.EMPTY
* */
private SingleValueArgumentEntry(int ignored) {
this.ts = System.currentTimeMillis();
this.value = null;
}
@Override @Override
public ArgumentType getType() { public ArgumentEntryType getType() {
return ArgumentType.SINGLE_VALUE; return ArgumentEntryType.SINGLE_VALUE;
} }
@Override @Override
@ -48,4 +63,14 @@ public class SingleValueArgumentEntry implements ArgumentEntry {
return value; return value;
} }
@Override
public boolean hasUpdatedValue(ArgumentEntry entry) {
return this.ts != ((SingleValueArgumentEntry) entry).getTs();
}
@Override
public ArgumentEntry copy() {
return new SingleValueArgumentEntry(this.ts, this.value, this.version);
}
} }

56
application/src/main/java/org/thingsboard/server/service/cf/ctx/state/TsRollingArgumentEntry.java

@ -19,19 +19,40 @@ import com.fasterxml.jackson.annotation.JsonIgnore;
import lombok.AllArgsConstructor; import lombok.AllArgsConstructor;
import lombok.Data; import lombok.Data;
import lombok.NoArgsConstructor; import lombok.NoArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.math.NumberUtils;
import org.thingsboard.server.common.data.kv.TsKvEntry;
import java.util.List;
import java.util.Map;
import java.util.TreeMap; import java.util.TreeMap;
@Data @Data
@NoArgsConstructor @NoArgsConstructor
@AllArgsConstructor @AllArgsConstructor
@Slf4j
public class TsRollingArgumentEntry implements ArgumentEntry { public class TsRollingArgumentEntry implements ArgumentEntry {
private TreeMap<Long, Object> tsRecords; public static final ArgumentEntry EMPTY = new TsRollingArgumentEntry(0);
private static final int MAX_ROLLING_ARGUMENT_ENTRY_SIZE = 1000;
private TreeMap<Long, Object> tsRecords = new TreeMap<>();
public TsRollingArgumentEntry(List<TsKvEntry> kvEntries) {
kvEntries.forEach(tsKvEntry -> addTsRecord(tsKvEntry.getTs(), tsKvEntry.getValue()));
}
/**
* Internal constructor to create immutable TsRollingArgumentEntry.EMPTY
*/
private TsRollingArgumentEntry(int ignored) {
this.tsRecords = new TreeMap<>();
}
@Override @Override
public ArgumentType getType() { public ArgumentEntryType getType() {
return ArgumentType.TS_ROLLING; return ArgumentEntryType.TS_ROLLING;
} }
@JsonIgnore @JsonIgnore
@ -40,4 +61,33 @@ public class TsRollingArgumentEntry implements ArgumentEntry {
return tsRecords; return tsRecords;
} }
@Override
public boolean hasUpdatedValue(ArgumentEntry entry) {
return entry instanceof SingleValueArgumentEntry ?
!tsRecords.containsKey(((SingleValueArgumentEntry) entry).getTs()) :
!tsRecords.keySet().containsAll(((TsRollingArgumentEntry) entry).getTsRecords().keySet());
}
@Override
public ArgumentEntry copy() {
return new TsRollingArgumentEntry(new TreeMap<>(tsRecords));
}
public void addTsRecord(Long key, Object value) {
if (NumberUtils.isParsable(value.toString())) {
tsRecords.put(key, value);
if (tsRecords.size() > MAX_ROLLING_ARGUMENT_ENTRY_SIZE) {
tsRecords.pollFirstEntry();
}
} else {
log.warn("Argument type 'TS_ROLLING' only supports numeric values.");
}
}
public void addAllTsRecords(Map<Long, Object> newRecords) {
for (Map.Entry<Long, Object> entry : newRecords.entrySet()) {
addTsRecord(entry.getKey(), entry.getValue());
}
}
} }

38
application/src/main/java/org/thingsboard/server/service/cf/telemetry/CalculatedFieldAttributeUpdateRequest.java

@ -0,0 +1,38 @@
/**
* Copyright © 2016-2024 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.service.cf.telemetry;
import lombok.AllArgsConstructor;
import lombok.Data;
import org.thingsboard.server.common.data.AttributeScope;
import org.thingsboard.server.common.data.id.CalculatedFieldId;
import org.thingsboard.server.common.data.id.EntityId;
import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.data.kv.AttributeKvEntry;
import java.util.List;
@Data
@AllArgsConstructor
public class CalculatedFieldAttributeUpdateRequest implements CalculatedFieldTelemetryUpdateRequest {
private TenantId tenantId;
private EntityId entityId;
private AttributeScope scope;
private List<AttributeKvEntry> kvEntries;
private List<CalculatedFieldId> calculatedFieldIds;
}

38
application/src/main/java/org/thingsboard/server/service/cf/telemetry/CalculatedFieldTelemetryUpdateRequest.java

@ -0,0 +1,38 @@
/**
* Copyright © 2016-2024 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.service.cf.telemetry;
import org.thingsboard.server.common.data.AttributeScope;
import org.thingsboard.server.common.data.id.CalculatedFieldId;
import org.thingsboard.server.common.data.id.EntityId;
import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.data.kv.KvEntry;
import java.util.List;
public interface CalculatedFieldTelemetryUpdateRequest {
TenantId getTenantId();
EntityId getEntityId();
AttributeScope getScope();
List<? extends KvEntry> getKvEntries();
List<CalculatedFieldId> getCalculatedFieldIds();
}

42
application/src/main/java/org/thingsboard/server/service/cf/telemetry/CalculatedFieldTimeSeriesUpdateRequest.java

@ -0,0 +1,42 @@
/**
* Copyright © 2016-2024 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.service.cf.telemetry;
import lombok.AllArgsConstructor;
import lombok.Data;
import org.thingsboard.server.common.data.AttributeScope;
import org.thingsboard.server.common.data.id.CalculatedFieldId;
import org.thingsboard.server.common.data.id.EntityId;
import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.data.kv.TsKvEntry;
import java.util.List;
@Data
@AllArgsConstructor
public class CalculatedFieldTimeSeriesUpdateRequest implements CalculatedFieldTelemetryUpdateRequest {
private TenantId tenantId;
private EntityId entityId;
private List<TsKvEntry> kvEntries;
private List<CalculatedFieldId> calculatedFieldIds;
@Override
public AttributeScope getScope() {
return null;
}
}

18
application/src/main/java/org/thingsboard/server/service/entitiy/cf/DefaultTbCalculatedFieldService.java

@ -46,6 +46,9 @@ import static org.thingsboard.server.dao.service.Validator.validateEntityId;
@RequiredArgsConstructor @RequiredArgsConstructor
public class DefaultTbCalculatedFieldService extends AbstractTbEntityService implements TbCalculatedFieldService { public class DefaultTbCalculatedFieldService extends AbstractTbEntityService implements TbCalculatedFieldService {
private static final int MAX_ARGUMENT_SIZE = 10;
private static final int MAX_CALCULATED_FIELD_NUMBER = 10;
private final CalculatedFieldService calculatedFieldService; private final CalculatedFieldService calculatedFieldService;
@Override @Override
@ -53,7 +56,9 @@ public class DefaultTbCalculatedFieldService extends AbstractTbEntityService imp
ActionType actionType = calculatedField.getId() == null ? ActionType.ADDED : ActionType.UPDATED; ActionType actionType = calculatedField.getId() == null ? ActionType.ADDED : ActionType.UPDATED;
TenantId tenantId = calculatedField.getTenantId(); TenantId tenantId = calculatedField.getTenantId();
try { try {
checkCalculatedFieldNumber(tenantId, calculatedField.getEntityId());
checkEntityExistence(tenantId, calculatedField.getEntityId()); checkEntityExistence(tenantId, calculatedField.getEntityId());
checkArgumentSize(calculatedField.getConfiguration());
checkReferencedEntities(calculatedField.getConfiguration(), user); checkReferencedEntities(calculatedField.getConfiguration(), user);
CalculatedField savedCalculatedField = checkNotNull(calculatedFieldService.save(calculatedField)); CalculatedField savedCalculatedField = checkNotNull(calculatedFieldService.save(calculatedField));
logEntityActionService.logEntityAction(tenantId, savedCalculatedField.getId(), savedCalculatedField, actionType, user); logEntityActionService.logEntityAction(tenantId, savedCalculatedField.getId(), savedCalculatedField, actionType, user);
@ -105,6 +110,19 @@ public class DefaultTbCalculatedFieldService extends AbstractTbEntityService imp
} }
private void checkArgumentSize(CalculatedFieldConfiguration calculatedFieldConfig) {
if (calculatedFieldConfig.getArguments().size() > MAX_ARGUMENT_SIZE) {
throw new IllegalArgumentException("Too many arguments: " + calculatedFieldConfig.getArguments().size() + ". Max number of argument is " + MAX_ARGUMENT_SIZE);
}
}
private void checkCalculatedFieldNumber(TenantId tenantId, EntityId entityId) {
int numberOfCalculatedFieldsByEntityId = calculatedFieldService.findCalculatedFieldIdsByEntityId(tenantId, entityId).size();
if (numberOfCalculatedFieldsByEntityId >= MAX_CALCULATED_FIELD_NUMBER) {
throw new IllegalArgumentException("Max number of calculated fields for entity is " + MAX_CALCULATED_FIELD_NUMBER);
}
}
private <E extends HasId<I> & HasTenantId, I extends EntityId> E findEntity(TenantId tenantId, EntityId entityId) { private <E extends HasId<I> & HasTenantId, I extends EntityId> E findEntity(TenantId tenantId, EntityId entityId) {
return switch (entityId.getEntityType()) { return switch (entityId.getEntityType()) {
case TENANT, CUSTOMER, ASSET, DEVICE -> (E) entityService.fetchEntity(tenantId, entityId).orElse(null); case TENANT, CUSTOMER, ASSET, DEVICE -> (E) entityService.fetchEntity(tenantId, entityId).orElse(null);

36
application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java

@ -68,7 +68,6 @@ import org.thingsboard.server.common.msg.rpc.FromDeviceRpcResponse;
import org.thingsboard.server.common.msg.rule.engine.DeviceEdgeUpdateMsg; import org.thingsboard.server.common.msg.rule.engine.DeviceEdgeUpdateMsg;
import org.thingsboard.server.common.msg.rule.engine.DeviceNameOrTypeUpdateMsg; import org.thingsboard.server.common.msg.rule.engine.DeviceNameOrTypeUpdateMsg;
import org.thingsboard.server.common.util.ProtoUtils; import org.thingsboard.server.common.util.ProtoUtils;
import org.thingsboard.server.dao.cf.CalculatedFieldService;
import org.thingsboard.server.dao.edge.EdgeService; import org.thingsboard.server.dao.edge.EdgeService;
import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.gen.transport.TransportProtos;
import org.thingsboard.server.gen.transport.TransportProtos.ComponentLifecycleMsgProto; import org.thingsboard.server.gen.transport.TransportProtos.ComponentLifecycleMsgProto;
@ -150,7 +149,6 @@ public class DefaultTbClusterService implements TbClusterService {
private final GatewayNotificationsService gatewayNotificationsService; private final GatewayNotificationsService gatewayNotificationsService;
private final EdgeService edgeService; private final EdgeService edgeService;
private final TbTransactionalCache<EdgeId, String> edgeIdServiceIdCache; private final TbTransactionalCache<EdgeId, String> edgeIdServiceIdCache;
private final CalculatedFieldService calculatedFieldService;
@Override @Override
public void pushMsgToCore(TenantId tenantId, EntityId entityId, ToCoreMsg msg, TbQueueCallback callback) { public void pushMsgToCore(TenantId tenantId, EntityId entityId, ToCoreMsg msg, TbQueueCallback callback) {
@ -393,7 +391,7 @@ public class DefaultTbClusterService implements TbClusterService {
public void onDeviceDeleted(TenantId tenantId, Device device, TbQueueCallback callback) { public void onDeviceDeleted(TenantId tenantId, Device device, TbQueueCallback callback) {
DeviceId deviceId = device.getId(); DeviceId deviceId = device.getId();
gatewayNotificationsService.onDeviceDeleted(device); gatewayNotificationsService.onDeviceDeleted(device);
handleEntityDelete(tenantId, deviceId, device.getDeviceProfileId()); sendProfileEntityEvent(tenantId, deviceId, device.getDeviceProfileId(), false, true);
broadcastEntityDeleteToTransport(tenantId, deviceId, device.getName(), callback); broadcastEntityDeleteToTransport(tenantId, deviceId, device.getName(), callback);
sendDeviceStateServiceEvent(tenantId, deviceId, false, false, true); sendDeviceStateServiceEvent(tenantId, deviceId, false, false, true);
broadcastEntityStateChangeEvent(tenantId, deviceId, ComponentLifecycleEvent.DELETED); broadcastEntityStateChangeEvent(tenantId, deviceId, ComponentLifecycleEvent.DELETED);
@ -402,17 +400,10 @@ public class DefaultTbClusterService implements TbClusterService {
@Override @Override
public void onAssetDeleted(TenantId tenantId, Asset asset, TbQueueCallback callback) { public void onAssetDeleted(TenantId tenantId, Asset asset, TbQueueCallback callback) {
AssetId assetId = asset.getId(); AssetId assetId = asset.getId();
handleEntityDelete(tenantId, assetId, asset.getAssetProfileId()); sendProfileEntityEvent(tenantId, assetId, asset.getAssetProfileId(), false, true);
broadcastEntityStateChangeEvent(tenantId, assetId, ComponentLifecycleEvent.DELETED); broadcastEntityStateChangeEvent(tenantId, assetId, ComponentLifecycleEvent.DELETED);
} }
private void handleEntityDelete(TenantId tenantId, EntityId entityId, EntityId profileId) {
boolean cfExistsByProfile = calculatedFieldService.existsCalculatedFieldByEntityId(tenantId, profileId);
if (cfExistsByProfile) {
sendProfileEntityEvent(tenantId, entityId, profileId, false, true);
}
}
@Override @Override
public void onDeviceAssignedToTenant(TenantId oldTenantId, Device device) { public void onDeviceAssignedToTenant(TenantId oldTenantId, Device device) {
onDeviceDeleted(oldTenantId, device, null); onDeviceDeleted(oldTenantId, device, null);
@ -633,13 +624,13 @@ public class DefaultTbClusterService implements TbClusterService {
} }
boolean deviceTypeChanged = !device.getType().equals(old.getType()); boolean deviceTypeChanged = !device.getType().equals(old.getType());
if (deviceTypeChanged) { if (deviceTypeChanged) {
handleProfileUpdate(device.getTenantId(), device.getId(), old.getDeviceProfileId(), device.getDeviceProfileId()); sendEntityProfileUpdatedEvent(device.getTenantId(), device.getId(), old.getDeviceProfileId(), device.getDeviceProfileId());
} }
if (deviceNameChanged || deviceTypeChanged) { if (deviceNameChanged || deviceTypeChanged) {
pushMsgToCore(new DeviceNameOrTypeUpdateMsg(device.getTenantId(), device.getId(), device.getName(), device.getType()), null); pushMsgToCore(new DeviceNameOrTypeUpdateMsg(device.getTenantId(), device.getId(), device.getName(), device.getType()), null);
} }
} else { } else {
handleEntityCreate(device.getTenantId(), device.getId(), device.getDeviceProfileId()); sendProfileEntityEvent(device.getTenantId(), device.getId(), device.getDeviceProfileId(), true, false);
} }
broadcastEntityStateChangeEvent(device.getTenantId(), device.getId(), created ? ComponentLifecycleEvent.CREATED : ComponentLifecycleEvent.UPDATED); broadcastEntityStateChangeEvent(device.getTenantId(), device.getId(), created ? ComponentLifecycleEvent.CREATED : ComponentLifecycleEvent.UPDATED);
sendDeviceStateServiceEvent(device.getTenantId(), device.getId(), created, !created, false); sendDeviceStateServiceEvent(device.getTenantId(), device.getId(), created, !created, false);
@ -653,29 +644,14 @@ public class DefaultTbClusterService implements TbClusterService {
if (old != null) { if (old != null) {
boolean assetTypeChanged = !asset.getType().equals(old.getType()); boolean assetTypeChanged = !asset.getType().equals(old.getType());
if (assetTypeChanged) { if (assetTypeChanged) {
handleProfileUpdate(asset.getTenantId(), asset.getId(), old.getAssetProfileId(), asset.getAssetProfileId()); sendEntityProfileUpdatedEvent(asset.getTenantId(), asset.getId(), old.getAssetProfileId(), asset.getAssetProfileId());
} }
} else { } else {
handleEntityCreate(asset.getTenantId(), asset.getId(), asset.getAssetProfileId()); sendProfileEntityEvent(asset.getTenantId(), asset.getId(), asset.getAssetProfileId(), true, false);
} }
broadcastEntityStateChangeEvent(asset.getTenantId(), asset.getId(), created ? ComponentLifecycleEvent.CREATED : ComponentLifecycleEvent.UPDATED); broadcastEntityStateChangeEvent(asset.getTenantId(), asset.getId(), created ? ComponentLifecycleEvent.CREATED : ComponentLifecycleEvent.UPDATED);
} }
private void handleProfileUpdate(TenantId tenantId, EntityId entityId, EntityId oldProfileId, EntityId newProfileId) {
boolean cfExistsByOldProfile = calculatedFieldService.existsCalculatedFieldByEntityId(tenantId, oldProfileId);
boolean cfExistsByNewProfile = calculatedFieldService.existsCalculatedFieldByEntityId(tenantId, newProfileId);
if (cfExistsByOldProfile || cfExistsByNewProfile) {
sendEntityProfileUpdatedEvent(tenantId, entityId, oldProfileId, newProfileId);
}
}
private void handleEntityCreate(TenantId tenantId, EntityId entityId, EntityId profileId) {
boolean cfExistsByProfile = calculatedFieldService.existsCalculatedFieldByEntityId(tenantId, profileId);
if (cfExistsByProfile) {
sendProfileEntityEvent(tenantId, entityId, profileId, true, false);
}
}
@Override @Override
public void sendNotificationMsgToEdge(TenantId tenantId, EdgeId edgeId, EntityId entityId, String body, EdgeEventType type, EdgeEventActionType action, EdgeId originatorEdgeId) { public void sendNotificationMsgToEdge(TenantId tenantId, EdgeId edgeId, EntityId entityId, String body, EdgeEventType type, EdgeEventActionType action, EdgeId originatorEdgeId) {
if (!edgesEnabled) { if (!edgesEnabled) {

16
application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java

@ -322,6 +322,8 @@ public class DefaultTbCoreConsumerService extends AbstractConsumerService<ToCore
forwardToCalculatedFieldService(toCoreMsg.getEntityProfileUpdateMsg(), callback); forwardToCalculatedFieldService(toCoreMsg.getEntityProfileUpdateMsg(), callback);
} else if (toCoreMsg.hasProfileEntityMsg()) { } else if (toCoreMsg.hasProfileEntityMsg()) {
forwardToCalculatedFieldService(toCoreMsg.getProfileEntityMsg(), callback); forwardToCalculatedFieldService(toCoreMsg.getProfileEntityMsg(), callback);
} else if (toCoreMsg.hasCalculatedFieldStateMsg()) {
forwardToCalculatedFieldService(toCoreMsg.getCalculatedFieldStateMsg(), callback);
} }
} catch (Throwable e) { } catch (Throwable e) {
log.warn("[{}] Failed to process message: {}", id, msg, e); log.warn("[{}] Failed to process message: {}", id, msg, e);
@ -687,7 +689,7 @@ public class DefaultTbCoreConsumerService extends AbstractConsumerService<ToCore
private void forwardToCalculatedFieldService(TransportProtos.EntityProfileUpdateMsgProto profileUpdateMsg, TbCallback callback) { private void forwardToCalculatedFieldService(TransportProtos.EntityProfileUpdateMsgProto profileUpdateMsg, TbCallback callback) {
var tenantId = toTenantId(profileUpdateMsg.getTenantIdMSB(), profileUpdateMsg.getTenantIdLSB()); var tenantId = toTenantId(profileUpdateMsg.getTenantIdMSB(), profileUpdateMsg.getTenantIdLSB());
var entityId = EntityIdFactory.getByTypeAndUuid(profileUpdateMsg.getEntityProfileType(), new UUID(profileUpdateMsg.getEntityIdMSB(), profileUpdateMsg.getEntityIdLSB())); var entityId = EntityIdFactory.getByTypeAndUuid(profileUpdateMsg.getEntityProfileType(), new UUID(profileUpdateMsg.getEntityIdMSB(), profileUpdateMsg.getEntityIdLSB()));
ListenableFuture<?> future = calculatedFieldsExecutor.submit(() -> calculatedFieldExecutionService.onEntityProfileChanged(profileUpdateMsg, callback)); ListenableFuture<?> future = calculatedFieldsExecutor.submit(() -> calculatedFieldExecutionService.onEntityProfileChangedMsg(profileUpdateMsg, callback));
DonAsynchron.withCallback(future, DonAsynchron.withCallback(future,
__ -> callback.onSuccess(), __ -> callback.onSuccess(),
t -> { t -> {
@ -708,6 +710,18 @@ public class DefaultTbCoreConsumerService extends AbstractConsumerService<ToCore
}); });
} }
private void forwardToCalculatedFieldService(TransportProtos.CalculatedFieldStateMsgProto calculatedFieldStateMsgProto, TbCallback callback) {
var tenantId = toTenantId(calculatedFieldStateMsgProto.getTenantIdMSB(), calculatedFieldStateMsgProto.getTenantIdLSB());
var calculatedFieldId = new CalculatedFieldId(new UUID(calculatedFieldStateMsgProto.getCalculatedFieldIdMSB(), calculatedFieldStateMsgProto.getCalculatedFieldIdLSB()));
ListenableFuture<?> future = calculatedFieldsExecutor.submit(() -> calculatedFieldExecutionService.onCalculatedFieldStateMsg(calculatedFieldStateMsgProto, callback));
DonAsynchron.withCallback(future,
__ -> callback.onSuccess(),
t -> {
log.warn("[{}] Failed to process calculated field state message for entityId [{}]", tenantId.getId(), calculatedFieldId.getId(), t);
callback.onFailure(t);
});
}
private void forwardToNotificationSchedulerService(TransportProtos.NotificationSchedulerServiceMsg msg, TbCallback callback) { private void forwardToNotificationSchedulerService(TransportProtos.NotificationSchedulerServiceMsg msg, TbCallback callback) {
TenantId tenantId = toTenantId(msg.getTenantIdMSB(), msg.getTenantIdLSB()); TenantId tenantId = toTenantId(msg.getTenantIdMSB(), msg.getTenantIdLSB());
NotificationRequestId notificationRequestId = new NotificationRequestId(new UUID(msg.getRequestIdMSB(), msg.getRequestIdLSB())); NotificationRequestId notificationRequestId = new NotificationRequestId(new UUID(msg.getRequestIdMSB(), msg.getRequestIdLSB()));

4
application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java

@ -50,6 +50,8 @@ import org.thingsboard.server.dao.timeseries.TimeseriesService;
import org.thingsboard.server.dao.util.KvUtils; import org.thingsboard.server.dao.util.KvUtils;
import org.thingsboard.server.service.apiusage.TbApiUsageStateService; import org.thingsboard.server.service.apiusage.TbApiUsageStateService;
import org.thingsboard.server.service.cf.CalculatedFieldExecutionService; import org.thingsboard.server.service.cf.CalculatedFieldExecutionService;
import org.thingsboard.server.service.cf.telemetry.CalculatedFieldAttributeUpdateRequest;
import org.thingsboard.server.service.cf.telemetry.CalculatedFieldTimeSeriesUpdateRequest;
import org.thingsboard.server.service.entitiy.entityview.TbEntityViewService; import org.thingsboard.server.service.entitiy.entityview.TbEntityViewService;
import org.thingsboard.server.service.subscription.TbSubscriptionUtils; import org.thingsboard.server.service.subscription.TbSubscriptionUtils;
@ -152,6 +154,7 @@ public class DefaultTelemetrySubscriptionService extends AbstractSubscriptionSer
if (request.isSaveLatest() && !request.isOnlyLatest()) { if (request.isSaveLatest() && !request.isOnlyLatest()) {
addEntityViewCallback(tenantId, entityId, request.getEntries()); addEntityViewCallback(tenantId, entityId, request.getEntries());
} }
calculatedFieldExecutionService.onTelemetryUpdate(new CalculatedFieldTimeSeriesUpdateRequest(tenantId, entityId, request.getEntries(), request.getCalculatedFieldIds()));
return saveFuture; return saveFuture;
} }
@ -167,6 +170,7 @@ public class DefaultTelemetrySubscriptionService extends AbstractSubscriptionSer
ListenableFuture<List<Long>> saveFuture = attrService.save(request.getTenantId(), request.getEntityId(), request.getScope(), request.getEntries()); ListenableFuture<List<Long>> saveFuture = attrService.save(request.getTenantId(), request.getEntityId(), request.getScope(), request.getEntries());
addMainCallback(saveFuture, request.getCallback()); addMainCallback(saveFuture, request.getCallback());
addWsCallback(saveFuture, success -> onAttributesUpdate(request.getTenantId(), request.getEntityId(), request.getScope().name(), request.getEntries(), request.isNotifyDevice())); addWsCallback(saveFuture, success -> onAttributesUpdate(request.getTenantId(), request.getEntityId(), request.getScope().name(), request.getEntries(), request.isNotifyDevice()));
calculatedFieldExecutionService.onTelemetryUpdate(new CalculatedFieldAttributeUpdateRequest(request.getTenantId(), request.getEntityId(), request.getScope(), request.getEntries(), request.getCalculatedFieldIds()));
} }
@Override @Override

6
application/src/test/java/org/thingsboard/server/controller/CalculatedFieldControllerTest.java

@ -24,8 +24,10 @@ import org.thingsboard.server.common.data.User;
import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.cf.CalculatedField;
import org.thingsboard.server.common.data.cf.CalculatedFieldType; import org.thingsboard.server.common.data.cf.CalculatedFieldType;
import org.thingsboard.server.common.data.cf.configuration.Argument; import org.thingsboard.server.common.data.cf.configuration.Argument;
import org.thingsboard.server.common.data.cf.configuration.ArgumentType;
import org.thingsboard.server.common.data.cf.configuration.CalculatedFieldConfiguration; import org.thingsboard.server.common.data.cf.configuration.CalculatedFieldConfiguration;
import org.thingsboard.server.common.data.cf.configuration.Output; import org.thingsboard.server.common.data.cf.configuration.Output;
import org.thingsboard.server.common.data.cf.configuration.OutputType;
import org.thingsboard.server.common.data.cf.configuration.SimpleCalculatedFieldConfiguration; import org.thingsboard.server.common.data.cf.configuration.SimpleCalculatedFieldConfiguration;
import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.DeviceId;
import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.EntityId;
@ -140,7 +142,7 @@ public class CalculatedFieldControllerTest extends AbstractControllerTest {
Argument argument = new Argument(); Argument argument = new Argument();
argument.setEntityId(referencedEntityId); argument.setEntityId(referencedEntityId);
argument.setType("TS_LATEST"); argument.setType(ArgumentType.TS_LATEST);
argument.setKey("temperature"); argument.setKey("temperature");
config.setArguments(Map.of("T", argument)); config.setArguments(Map.of("T", argument));
@ -149,7 +151,7 @@ public class CalculatedFieldControllerTest extends AbstractControllerTest {
Output output = new Output(); Output output = new Output();
output.setName("output"); output.setName("output");
output.setType("TS_LATEST"); output.setType(OutputType.TIME_SERIES);
config.setOutput(output); config.setOutput(output);

4
common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedFieldLinkConfiguration.java

@ -23,7 +23,9 @@ import java.util.Map;
@Data @Data
public class CalculatedFieldLinkConfiguration { public class CalculatedFieldLinkConfiguration {
private Map<String, String> attributes = new HashMap<>(); private Map<String, String> clientAttributes = new HashMap<>();
private Map<String, String> serverAttributes = new HashMap<>();
private Map<String, String> sharedAttributes = new HashMap<>();
private Map<String, String> timeSeries = new HashMap<>(); private Map<String, String> timeSeries = new HashMap<>();
} }

2
common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/Argument.java

@ -24,7 +24,7 @@ public class Argument {
private EntityId entityId; private EntityId entityId;
private String key; private String key;
private String type; private ArgumentType type;
private AttributeScope scope; private AttributeScope scope;
private String defaultValue; private String defaultValue;

22
common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/ArgumentType.java

@ -0,0 +1,22 @@
/**
* Copyright © 2016-2024 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.common.data.cf.configuration;
public enum ArgumentType {
TS_LATEST, ATTRIBUTE, TS_ROLLING
}

45
common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/BaseCalculatedFieldConfiguration.java

@ -65,19 +65,24 @@ public abstract class BaseCalculatedFieldConfiguration implements CalculatedFiel
public CalculatedFieldLinkConfiguration getReferencedEntityConfig(EntityId entityId) { public CalculatedFieldLinkConfiguration getReferencedEntityConfig(EntityId entityId) {
CalculatedFieldLinkConfiguration linkConfiguration = new CalculatedFieldLinkConfiguration(); CalculatedFieldLinkConfiguration linkConfiguration = new CalculatedFieldLinkConfiguration();
for (Map.Entry<String, Argument> entry : arguments.entrySet()) { arguments.entrySet().stream()
Argument argument = entry.getValue(); .filter(entry -> entry.getValue().getEntityId().equals(entityId))
if (argument.getEntityId().equals(entityId)) { .forEach(entry -> {
switch (argument.getType()) { Argument argument = entry.getValue();
case "ATTRIBUTE": String argumentKey = entry.getKey();
linkConfiguration.getAttributes().put(entry.getKey(), argument.getKey());
break; switch (argument.getType()) {
case "TS_LATEST", "TS_ROLLING": case ATTRIBUTE -> {
linkConfiguration.getTimeSeries().put(entry.getKey(), argument.getKey()); switch (argument.getScope()) {
break; case CLIENT_SCOPE -> linkConfiguration.getClientAttributes().put(entry.getKey(), argument.getKey());
} case SERVER_SCOPE -> linkConfiguration.getServerAttributes().put(entry.getKey(), argument.getKey());
} case SHARED_SCOPE -> linkConfiguration.getSharedAttributes().put(entry.getKey(), argument.getKey());
} }
}
case TS_LATEST, TS_ROLLING ->
linkConfiguration.getTimeSeries().put(argumentKey, argument.getKey());
}
});
return linkConfiguration; return linkConfiguration;
} }
@ -98,7 +103,7 @@ public abstract class BaseCalculatedFieldConfiguration implements CalculatedFiel
argumentNode.put("entityId", entityId.toString()); argumentNode.put("entityId", entityId.toString());
} }
argumentNode.put("key", argument.getKey()); argumentNode.put("key", argument.getKey());
argumentNode.put("type", argument.getType()); argumentNode.put("type", String.valueOf(argument.getType()));
argumentNode.put("scope", String.valueOf(argument.getScope())); argumentNode.put("scope", String.valueOf(argument.getScope()));
argumentNode.put("defaultValue", argument.getDefaultValue()); argumentNode.put("defaultValue", argument.getDefaultValue());
argumentNode.put("limit", String.valueOf(argument.getLimit())); argumentNode.put("limit", String.valueOf(argument.getLimit()));
@ -112,7 +117,7 @@ public abstract class BaseCalculatedFieldConfiguration implements CalculatedFiel
if (output != null) { if (output != null) {
ObjectNode outputNode = configNode.putObject("output"); ObjectNode outputNode = configNode.putObject("output");
outputNode.put("name", output.getName()); outputNode.put("name", output.getName());
outputNode.put("type", output.getType()); outputNode.put("type", String.valueOf(output.getType()));
if (output.getScope() != null) { if (output.getScope() != null) {
outputNode.put("scope", String.valueOf(output.getScope())); outputNode.put("scope", String.valueOf(output.getScope()));
} }
@ -141,7 +146,10 @@ public abstract class BaseCalculatedFieldConfiguration implements CalculatedFiel
argument.setEntityId(EntityIdFactory.getByTypeAndUuid(entityType, entityId)); argument.setEntityId(EntityIdFactory.getByTypeAndUuid(entityType, entityId));
} }
argument.setKey(argumentNode.get("key").asText()); argument.setKey(argumentNode.get("key").asText());
argument.setType(argumentNode.get("type").asText()); JsonNode type = argumentNode.get("type");
if (type != null && !type.isNull() && !type.asText().equals("null")) {
argument.setType(ArgumentType.valueOf(type.asText()));
}
JsonNode scope = argumentNode.get("scope"); JsonNode scope = argumentNode.get("scope");
if (scope != null && !scope.isNull() && !scope.asText().equals("null")) { if (scope != null && !scope.isNull() && !scope.asText().equals("null")) {
argument.setScope(AttributeScope.valueOf(scope.asText())); argument.setScope(AttributeScope.valueOf(scope.asText()));
@ -169,7 +177,10 @@ public abstract class BaseCalculatedFieldConfiguration implements CalculatedFiel
if (outputNode != null) { if (outputNode != null) {
Output output = new Output(); Output output = new Output();
output.setName(outputNode.get("name").asText()); output.setName(outputNode.get("name").asText());
output.setType(outputNode.get("type").asText()); JsonNode type = outputNode.get("type");
if (type != null && !type.isNull() && !type.asText().equals("null")) {
output.setType(OutputType.valueOf(type.asText()));
}
JsonNode scope = outputNode.get("scope"); JsonNode scope = outputNode.get("scope");
if (scope != null && !scope.isNull() && !scope.asText().equals("null")) { if (scope != null && !scope.isNull() && !scope.asText().equals("null")) {
output.setScope(AttributeScope.valueOf(scope.asText())); output.setScope(AttributeScope.valueOf(scope.asText()));

2
common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/Output.java

@ -22,7 +22,7 @@ import org.thingsboard.server.common.data.AttributeScope;
public class Output { public class Output {
private String name; private String name;
private String type; private OutputType type;
private AttributeScope scope; private AttributeScope scope;
} }

22
common/data/src/main/java/org/thingsboard/server/common/data/cf/configuration/OutputType.java

@ -0,0 +1,22 @@
/**
* Copyright © 2016-2024 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.common.data.cf.configuration;
public enum OutputType {
TIME_SERIES, ATTRIBUTES
}

45
common/message/src/main/java/org/thingsboard/server/common/msg/TbMsg.java

@ -24,6 +24,7 @@ import lombok.Getter;
import lombok.extern.slf4j.Slf4j; import lombok.extern.slf4j.Slf4j;
import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.EntityType;
import org.thingsboard.server.common.data.StringUtils; import org.thingsboard.server.common.data.StringUtils;
import org.thingsboard.server.common.data.id.CalculatedFieldId;
import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.CustomerId;
import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.EntityId;
import org.thingsboard.server.common.data.id.EntityIdFactory; import org.thingsboard.server.common.data.id.EntityIdFactory;
@ -34,6 +35,8 @@ import org.thingsboard.server.common.msg.gen.MsgProtos;
import org.thingsboard.server.common.msg.queue.TbMsgCallback; import org.thingsboard.server.common.msg.queue.TbMsgCallback;
import java.io.Serializable; import java.io.Serializable;
import java.util.ArrayList;
import java.util.List;
import java.util.Objects; import java.util.Objects;
import java.util.UUID; import java.util.UUID;
@ -64,6 +67,8 @@ public final class TbMsg implements Serializable {
private final UUID correlationId; private final UUID correlationId;
private final Integer partition; private final Integer partition;
private final List<CalculatedFieldId> calculatedFieldIds;
@Getter(value = AccessLevel.NONE) @Getter(value = AccessLevel.NONE)
@JsonIgnore @JsonIgnore
//This field is not serialized because we use queues and there is no need to do it //This field is not serialized because we use queues and there is no need to do it
@ -112,7 +117,7 @@ public final class TbMsg implements Serializable {
} }
private TbMsg(String queueName, UUID id, long ts, TbMsgType internalType, String type, EntityId originator, CustomerId customerId, TbMsgMetaData metaData, TbMsgDataType dataType, String data, private TbMsg(String queueName, UUID id, long ts, TbMsgType internalType, String type, EntityId originator, CustomerId customerId, TbMsgMetaData metaData, TbMsgDataType dataType, String data,
RuleChainId ruleChainId, RuleNodeId ruleNodeId, UUID correlationId, Integer partition, TbMsgProcessingCtx ctx, TbMsgCallback callback) { RuleChainId ruleChainId, RuleNodeId ruleNodeId, UUID correlationId, Integer partition, List<CalculatedFieldId> calculatedFieldIds, TbMsgProcessingCtx ctx, TbMsgCallback callback) {
this.id = id != null ? id : UUID.randomUUID(); this.id = id != null ? id : UUID.randomUUID();
this.queueName = queueName; this.queueName = queueName;
if (ts > 0) { if (ts > 0) {
@ -139,6 +144,7 @@ public final class TbMsg implements Serializable {
this.ruleNodeId = ruleNodeId; this.ruleNodeId = ruleNodeId;
this.correlationId = correlationId; this.correlationId = correlationId;
this.partition = partition; this.partition = partition;
this.calculatedFieldIds = calculatedFieldIds;
this.ctx = ctx != null ? ctx : new TbMsgProcessingCtx(); this.ctx = ctx != null ? ctx : new TbMsgProcessingCtx();
this.callback = Objects.requireNonNullElse(callback, TbMsgCallback.EMPTY); this.callback = Objects.requireNonNullElse(callback, TbMsgCallback.EMPTY);
} }
@ -186,6 +192,16 @@ public final class TbMsg implements Serializable {
builder.setPartition(msg.getPartition()); builder.setPartition(msg.getPartition());
} }
if (msg.getCalculatedFieldIds() != null) {
for (CalculatedFieldId calculatedFieldId : msg.getCalculatedFieldIds()) {
MsgProtos.CalculatedFieldIdProto calculatedFieldIdProto = MsgProtos.CalculatedFieldIdProto.newBuilder()
.setCalculatedFieldIdMSB(calculatedFieldId.getId().getMostSignificantBits())
.setCalculatedFieldIdLSB(calculatedFieldId.getId().getLeastSignificantBits())
.build();
builder.addCalculatedFields(calculatedFieldIdProto);
}
}
builder.setCtx(msg.ctx.toProto()); builder.setCtx(msg.ctx.toProto());
return builder.build().toByteArray(); return builder.build().toByteArray();
} }
@ -200,6 +216,7 @@ public final class TbMsg implements Serializable {
RuleNodeId ruleNodeId = null; RuleNodeId ruleNodeId = null;
UUID correlationId = null; UUID correlationId = null;
Integer partition = null; Integer partition = null;
List<CalculatedFieldId> calculatedFieldIds = new ArrayList<>();
if (proto.getCustomerIdMSB() != 0L && proto.getCustomerIdLSB() != 0L) { if (proto.getCustomerIdMSB() != 0L && proto.getCustomerIdLSB() != 0L) {
customerId = new CustomerId(new UUID(proto.getCustomerIdMSB(), proto.getCustomerIdLSB())); customerId = new CustomerId(new UUID(proto.getCustomerIdMSB(), proto.getCustomerIdLSB()));
} }
@ -214,6 +231,14 @@ public final class TbMsg implements Serializable {
partition = proto.getPartition(); partition = proto.getPartition();
} }
for (MsgProtos.CalculatedFieldIdProto cfIdProto : proto.getCalculatedFieldsList()) {
CalculatedFieldId calculatedFieldId = new CalculatedFieldId(new UUID(
cfIdProto.getCalculatedFieldIdMSB(),
cfIdProto.getCalculatedFieldIdLSB()
));
calculatedFieldIds.add(calculatedFieldId);
}
TbMsgProcessingCtx ctx; TbMsgProcessingCtx ctx;
if (proto.hasCtx()) { if (proto.hasCtx()) {
ctx = TbMsgProcessingCtx.fromProto(proto.getCtx()); ctx = TbMsgProcessingCtx.fromProto(proto.getCtx());
@ -224,7 +249,7 @@ public final class TbMsg implements Serializable {
TbMsgDataType dataType = TbMsgDataType.values()[proto.getDataType()]; TbMsgDataType dataType = TbMsgDataType.values()[proto.getDataType()];
return new TbMsg(queueName, UUID.fromString(proto.getId()), proto.getTs(), null, proto.getType(), entityId, customerId, return new TbMsg(queueName, UUID.fromString(proto.getId()), proto.getTs(), null, proto.getType(), entityId, customerId,
metaData, dataType, proto.getData(), ruleChainId, ruleNodeId, correlationId, partition, ctx, callback); metaData, dataType, proto.getData(), ruleChainId, ruleNodeId, correlationId, partition, calculatedFieldIds, ctx, callback);
} catch (InvalidProtocolBufferException e) { } catch (InvalidProtocolBufferException e) {
throw new IllegalStateException("Could not parse protobuf for TbMsg", e); throw new IllegalStateException("Could not parse protobuf for TbMsg", e);
} }
@ -343,10 +368,12 @@ public final class TbMsg implements Serializable {
protected RuleNodeId ruleNodeId; protected RuleNodeId ruleNodeId;
protected UUID correlationId; protected UUID correlationId;
protected Integer partition; protected Integer partition;
protected List<CalculatedFieldId> calculatedFieldIds;
protected TbMsgProcessingCtx ctx; protected TbMsgProcessingCtx ctx;
protected TbMsgCallback callback; protected TbMsgCallback callback;
TbMsgBuilder() {} TbMsgBuilder() {
}
TbMsgBuilder(TbMsg tbMsg) { TbMsgBuilder(TbMsg tbMsg) {
this.queueName = tbMsg.queueName; this.queueName = tbMsg.queueName;
@ -363,6 +390,7 @@ public final class TbMsg implements Serializable {
this.ruleNodeId = tbMsg.ruleNodeId; this.ruleNodeId = tbMsg.ruleNodeId;
this.correlationId = tbMsg.correlationId; this.correlationId = tbMsg.correlationId;
this.partition = tbMsg.partition; this.partition = tbMsg.partition;
this.calculatedFieldIds = tbMsg.calculatedFieldIds;
this.ctx = tbMsg.ctx; this.ctx = tbMsg.ctx;
this.callback = tbMsg.callback; this.callback = tbMsg.callback;
} }
@ -454,6 +482,11 @@ public final class TbMsg implements Serializable {
return this; return this;
} }
public TbMsgBuilder calculatedFieldIds(List<CalculatedFieldId> calculatedFieldIds) {
this.calculatedFieldIds = calculatedFieldIds;
return this;
}
public TbMsgBuilder ctx(TbMsgProcessingCtx ctx) { public TbMsgBuilder ctx(TbMsgProcessingCtx ctx) {
this.ctx = ctx; this.ctx = ctx;
return this; return this;
@ -465,7 +498,7 @@ public final class TbMsg implements Serializable {
} }
public TbMsg build() { public TbMsg build() {
return new TbMsg(queueName, id, ts, internalType, type, originator, customerId, metaData, dataType, data, ruleChainId, ruleNodeId, correlationId, partition, ctx, callback); return new TbMsg(queueName, id, ts, internalType, type, originator, customerId, metaData, dataType, data, ruleChainId, ruleNodeId, correlationId, partition, calculatedFieldIds, ctx, callback);
} }
public String toString() { public String toString() {
@ -473,8 +506,8 @@ public final class TbMsg implements Serializable {
", type=" + this.type + ", internalType=" + this.internalType + ", originator=" + this.originator + ", type=" + this.type + ", internalType=" + this.internalType + ", originator=" + this.originator +
", customerId=" + this.customerId + ", metaData=" + this.metaData + ", dataType=" + this.dataType + ", customerId=" + this.customerId + ", metaData=" + this.metaData + ", dataType=" + this.dataType +
", data=" + this.data + ", ruleChainId=" + this.ruleChainId + ", ruleNodeId=" + this.ruleNodeId + ", data=" + this.data + ", ruleChainId=" + this.ruleChainId + ", ruleNodeId=" + this.ruleNodeId +
", correlationId=" + this.correlationId + ", partition=" + this.partition + ", ctx=" + this.ctx + ", correlationId=" + this.correlationId + ", partition=" + this.partition + ", calculatedFields=" + this.calculatedFieldIds +
", callback=" + this.callback + ")"; ", ctx=" + this.ctx + ", callback=" + this.callback + ")";
} }
} }

7
common/message/src/main/proto/tbmsg.proto

@ -70,4 +70,11 @@ message TbMsgProto {
int64 correlationIdMSB = 20; int64 correlationIdMSB = 20;
int64 correlationIdLSB = 21; int64 correlationIdLSB = 21;
int32 partition = 22; int32 partition = 22;
repeated CalculatedFieldIdProto calculatedFields = 23;
}
message CalculatedFieldIdProto {
int64 calculatedFieldIdMSB = 1;
int64 calculatedFieldIdLSB = 2;
} }

40
common/proto/src/main/java/org/thingsboard/server/common/util/ProtoUtils.java

@ -1183,6 +1183,46 @@ public class ProtoUtils {
return builder.build(); return builder.build();
} }
public static TransportProtos.ObjectProto toObjectProto(Object value) {
if (value == null) {
throw new IllegalArgumentException("Cannot convert null to ObjectProto");
}
TransportProtos.ObjectProto.Builder builder = TransportProtos.ObjectProto.newBuilder();
if (value instanceof String) {
builder.setStringValue((String) value);
} else if (value instanceof Integer) {
builder.setIntValue((Integer) value);
} else if (value instanceof Long) {
builder.setLongValue((Long) value);
} else if (value instanceof Double) {
builder.setDoubleValue((Double) value);
} else if (value instanceof Boolean) {
builder.setBoolValue((Boolean) value);
} else {
throw new IllegalArgumentException("Unsupported value type: " + value.getClass().getName());
}
return builder.build();
}
public static Object fromObjectProto(TransportProtos.ObjectProto proto) {
try {
return switch (proto.getValueCase()) {
case STRINGVALUE -> proto.getStringValue();
case INTVALUE -> proto.getIntValue();
case LONGVALUE -> proto.getLongValue();
case DOUBLEVALUE -> proto.getDoubleValue();
case BOOLVALUE -> proto.getBoolValue();
case VALUE_NOT_SET -> throw new IllegalArgumentException("Value not set in ObjectProto");
};
} catch (Exception e) {
log.error("Failed to deserialize ObjectProto: [{}]", proto, e);
return null;
}
}
private static boolean isNotNull(Object obj) { private static boolean isNotNull(Object obj) {
return obj != null; return obj != null;
} }

46
common/proto/src/main/proto/queue.proto

@ -809,6 +809,51 @@ message ProfileEntityMsgProto {
bool deleted = 10; bool deleted = 10;
} }
message CalculatedFieldStateMsgProto {
int64 tenantIdMSB = 1;
int64 tenantIdLSB = 2;
int64 calculatedFieldIdMSB = 3;
int64 calculatedFieldIdLSB = 4;
string entityType = 5;
int64 entityIdMSB = 6;
int64 entityIdLSB = 7;
bool clear = 8;
repeated CalculatedFieldIdProto calculatedFields = 9;
map<string, ArgumentEntryProto> arguments = 10;
}
message CalculatedFieldIdProto {
int64 calculatedFieldIdMSB = 1;
int64 calculatedFieldIdLSB = 2;
}
message ArgumentEntryProto {
oneof entry_type {
TsRollingProto tsRecords = 1;
SingleValueProto singleValue = 2;
}
}
message TsRollingProto {
map<int64, ObjectProto> tsRecords = 1;
}
message SingleValueProto {
int64 ts = 1;
ObjectProto value = 2;
int64 version = 3;
}
message ObjectProto {
oneof value {
string stringValue = 1;
int32 intValue = 2;
int64 longValue = 3;
double doubleValue = 4;
bool boolValue = 5;
}
}
//Used to report session state to tb-Service and persist this state in the cache on the tb-Service level. //Used to report session state to tb-Service and persist this state in the cache on the tb-Service level.
message SubscriptionInfoProto { message SubscriptionInfoProto {
int64 lastActivityTime = 1; int64 lastActivityTime = 1;
@ -1555,6 +1600,7 @@ message ToCoreMsg {
CalculatedFieldMsgProto calculatedFieldMsg = 53; CalculatedFieldMsgProto calculatedFieldMsg = 53;
EntityProfileUpdateMsgProto entityProfileUpdateMsg = 54; EntityProfileUpdateMsgProto entityProfileUpdateMsg = 54;
ProfileEntityMsgProto profileEntityMsg = 55; ProfileEntityMsgProto profileEntityMsg = 55;
CalculatedFieldStateMsgProto calculatedFieldStateMsg = 56;
} }
/* High priority messages with low latency are handled by ThingsBoard Core Service separately */ /* High priority messages with low latency are handled by ThingsBoard Core Service separately */

6
dao/src/test/java/org/thingsboard/server/dao/service/AssetServiceTest.java

@ -33,7 +33,9 @@ import org.thingsboard.server.common.data.asset.AssetProfile;
import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.cf.CalculatedField;
import org.thingsboard.server.common.data.cf.CalculatedFieldType; import org.thingsboard.server.common.data.cf.CalculatedFieldType;
import org.thingsboard.server.common.data.cf.configuration.Argument; import org.thingsboard.server.common.data.cf.configuration.Argument;
import org.thingsboard.server.common.data.cf.configuration.ArgumentType;
import org.thingsboard.server.common.data.cf.configuration.Output; import org.thingsboard.server.common.data.cf.configuration.Output;
import org.thingsboard.server.common.data.cf.configuration.OutputType;
import org.thingsboard.server.common.data.cf.configuration.SimpleCalculatedFieldConfiguration; import org.thingsboard.server.common.data.cf.configuration.SimpleCalculatedFieldConfiguration;
import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.CustomerId;
import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.id.TenantId;
@ -884,7 +886,7 @@ public class AssetServiceTest extends AbstractServiceTest {
Argument argument = new Argument(); Argument argument = new Argument();
argument.setEntityId(savedAsset.getId()); argument.setEntityId(savedAsset.getId());
argument.setType("TS_LATEST"); argument.setType(ArgumentType.TS_LATEST);
argument.setKey("temperature"); argument.setKey("temperature");
config.setArguments(Map.of("T", argument)); config.setArguments(Map.of("T", argument));
@ -893,7 +895,7 @@ public class AssetServiceTest extends AbstractServiceTest {
Output output = new Output(); Output output = new Output();
output.setName("output"); output.setName("output");
output.setType("TS_LATEST"); output.setType(OutputType.TIME_SERIES);
config.setOutput(output); config.setOutput(output);

6
dao/src/test/java/org/thingsboard/server/dao/service/CalculatedFieldServiceTest.java

@ -26,8 +26,10 @@ import org.thingsboard.server.common.data.Device;
import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.cf.CalculatedField;
import org.thingsboard.server.common.data.cf.CalculatedFieldType; import org.thingsboard.server.common.data.cf.CalculatedFieldType;
import org.thingsboard.server.common.data.cf.configuration.Argument; import org.thingsboard.server.common.data.cf.configuration.Argument;
import org.thingsboard.server.common.data.cf.configuration.ArgumentType;
import org.thingsboard.server.common.data.cf.configuration.CalculatedFieldConfiguration; import org.thingsboard.server.common.data.cf.configuration.CalculatedFieldConfiguration;
import org.thingsboard.server.common.data.cf.configuration.Output; import org.thingsboard.server.common.data.cf.configuration.Output;
import org.thingsboard.server.common.data.cf.configuration.OutputType;
import org.thingsboard.server.common.data.cf.configuration.SimpleCalculatedFieldConfiguration; import org.thingsboard.server.common.data.cf.configuration.SimpleCalculatedFieldConfiguration;
import org.thingsboard.server.common.data.id.CalculatedFieldId; import org.thingsboard.server.common.data.id.CalculatedFieldId;
import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.EntityId;
@ -153,7 +155,7 @@ public class CalculatedFieldServiceTest extends AbstractServiceTest {
Argument argument = new Argument(); Argument argument = new Argument();
argument.setEntityId(referencedEntityId); argument.setEntityId(referencedEntityId);
argument.setType("TS_LATEST"); argument.setType(ArgumentType.TS_LATEST);
argument.setKey("temperature"); argument.setKey("temperature");
config.setArguments(Map.of("T", argument)); config.setArguments(Map.of("T", argument));
@ -162,7 +164,7 @@ public class CalculatedFieldServiceTest extends AbstractServiceTest {
Output output = new Output(); Output output = new Output();
output.setName("output"); output.setName("output");
output.setType("TS_LATEST"); output.setType(OutputType.TIME_SERIES);
config.setOutput(output); config.setOutput(output);

6
dao/src/test/java/org/thingsboard/server/dao/service/CustomerServiceTest.java

@ -34,7 +34,9 @@ import org.thingsboard.server.common.data.asset.Asset;
import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.cf.CalculatedField;
import org.thingsboard.server.common.data.cf.CalculatedFieldType; import org.thingsboard.server.common.data.cf.CalculatedFieldType;
import org.thingsboard.server.common.data.cf.configuration.Argument; import org.thingsboard.server.common.data.cf.configuration.Argument;
import org.thingsboard.server.common.data.cf.configuration.ArgumentType;
import org.thingsboard.server.common.data.cf.configuration.Output; import org.thingsboard.server.common.data.cf.configuration.Output;
import org.thingsboard.server.common.data.cf.configuration.OutputType;
import org.thingsboard.server.common.data.cf.configuration.SimpleCalculatedFieldConfiguration; import org.thingsboard.server.common.data.cf.configuration.SimpleCalculatedFieldConfiguration;
import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageData;
@ -379,7 +381,7 @@ public class CustomerServiceTest extends AbstractServiceTest {
Argument argument = new Argument(); Argument argument = new Argument();
argument.setEntityId(savedCustomer.getId()); argument.setEntityId(savedCustomer.getId());
argument.setType("TS_LATEST"); argument.setType(ArgumentType.TS_LATEST);
argument.setKey("temperature"); argument.setKey("temperature");
config.setArguments(Map.of("T", argument)); config.setArguments(Map.of("T", argument));
@ -388,7 +390,7 @@ public class CustomerServiceTest extends AbstractServiceTest {
Output output = new Output(); Output output = new Output();
output.setName("output"); output.setName("output");
output.setType("TS_LATEST"); output.setType(OutputType.TIME_SERIES);
config.setOutput(output); config.setOutput(output);

6
dao/src/test/java/org/thingsboard/server/dao/service/DeviceServiceTest.java

@ -42,7 +42,9 @@ import org.thingsboard.server.common.data.TenantProfile;
import org.thingsboard.server.common.data.cf.CalculatedField; import org.thingsboard.server.common.data.cf.CalculatedField;
import org.thingsboard.server.common.data.cf.CalculatedFieldType; import org.thingsboard.server.common.data.cf.CalculatedFieldType;
import org.thingsboard.server.common.data.cf.configuration.Argument; import org.thingsboard.server.common.data.cf.configuration.Argument;
import org.thingsboard.server.common.data.cf.configuration.ArgumentType;
import org.thingsboard.server.common.data.cf.configuration.Output; import org.thingsboard.server.common.data.cf.configuration.Output;
import org.thingsboard.server.common.data.cf.configuration.OutputType;
import org.thingsboard.server.common.data.cf.configuration.SimpleCalculatedFieldConfiguration; import org.thingsboard.server.common.data.cf.configuration.SimpleCalculatedFieldConfiguration;
import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.CustomerId;
import org.thingsboard.server.common.data.id.DeviceProfileId; import org.thingsboard.server.common.data.id.DeviceProfileId;
@ -1222,7 +1224,7 @@ public class DeviceServiceTest extends AbstractServiceTest {
Argument argument = new Argument(); Argument argument = new Argument();
argument.setEntityId(device.getId()); argument.setEntityId(device.getId());
argument.setType("TS_LATEST"); argument.setType(ArgumentType.TS_LATEST);
argument.setKey("temperature"); argument.setKey("temperature");
config.setArguments(Map.of("T", argument)); config.setArguments(Map.of("T", argument));
@ -1231,7 +1233,7 @@ public class DeviceServiceTest extends AbstractServiceTest {
Output output = new Output(); Output output = new Output();
output.setName("output"); output.setName("output");
output.setType("TS_LATEST"); output.setType(OutputType.TIME_SERIES);
config.setOutput(output); config.setOutput(output);

10
rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/AttributesSaveRequest.java

@ -22,6 +22,7 @@ import lombok.AllArgsConstructor;
import lombok.Getter; import lombok.Getter;
import lombok.ToString; import lombok.ToString;
import org.thingsboard.server.common.data.AttributeScope; import org.thingsboard.server.common.data.AttributeScope;
import org.thingsboard.server.common.data.id.CalculatedFieldId;
import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.EntityId;
import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.data.kv.AttributeKvEntry; import org.thingsboard.server.common.data.kv.AttributeKvEntry;
@ -40,6 +41,7 @@ public class AttributesSaveRequest {
private final AttributeScope scope; private final AttributeScope scope;
private final List<AttributeKvEntry> entries; private final List<AttributeKvEntry> entries;
private final boolean notifyDevice; private final boolean notifyDevice;
private final List<CalculatedFieldId> calculatedFieldIds;
private final FutureCallback<Void> callback; private final FutureCallback<Void> callback;
public static Builder builder() { public static Builder builder() {
@ -53,6 +55,7 @@ public class AttributesSaveRequest {
private AttributeScope scope; private AttributeScope scope;
private List<AttributeKvEntry> entries; private List<AttributeKvEntry> entries;
private boolean notifyDevice = true; private boolean notifyDevice = true;
private List<CalculatedFieldId> calculatedFieldIds;
private FutureCallback<Void> callback; private FutureCallback<Void> callback;
Builder() {} Builder() {}
@ -100,6 +103,11 @@ public class AttributesSaveRequest {
return this; return this;
} }
public Builder calculatedFieldIds(List<CalculatedFieldId> calculatedFieldIds) {
this.calculatedFieldIds = calculatedFieldIds;
return this;
}
public Builder callback(FutureCallback<Void> callback) { public Builder callback(FutureCallback<Void> callback) {
this.callback = callback; this.callback = callback;
return this; return this;
@ -120,7 +128,7 @@ public class AttributesSaveRequest {
} }
public AttributesSaveRequest build() { public AttributesSaveRequest build() {
return new AttributesSaveRequest(tenantId, entityId, scope, entries, notifyDevice, callback); return new AttributesSaveRequest(tenantId, entityId, scope, entries, notifyDevice, calculatedFieldIds, callback);
} }
} }

10
rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/TimeseriesSaveRequest.java

@ -20,6 +20,7 @@ import com.google.common.util.concurrent.SettableFuture;
import lombok.AccessLevel; import lombok.AccessLevel;
import lombok.AllArgsConstructor; import lombok.AllArgsConstructor;
import lombok.Getter; import lombok.Getter;
import org.thingsboard.server.common.data.id.CalculatedFieldId;
import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.CustomerId;
import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.EntityId;
import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.id.TenantId;
@ -40,6 +41,7 @@ public class TimeseriesSaveRequest {
private final long ttl; private final long ttl;
private final boolean saveLatest; private final boolean saveLatest;
private final boolean onlyLatest; private final boolean onlyLatest;
private final List<CalculatedFieldId> calculatedFieldIds;
private final FutureCallback<Void> callback; private final FutureCallback<Void> callback;
public static Builder builder() { public static Builder builder() {
@ -56,6 +58,7 @@ public class TimeseriesSaveRequest {
private FutureCallback<Void> callback; private FutureCallback<Void> callback;
private boolean saveLatest = true; private boolean saveLatest = true;
private boolean onlyLatest; private boolean onlyLatest;
private List<CalculatedFieldId> calculatedFieldIds;
Builder() {} Builder() {}
@ -103,6 +106,11 @@ public class TimeseriesSaveRequest {
return this; return this;
} }
public Builder calculatedFieldIds(List<CalculatedFieldId> calculatedFieldIds) {
this.calculatedFieldIds = calculatedFieldIds;
return this;
}
public Builder callback(FutureCallback<Void> callback) { public Builder callback(FutureCallback<Void> callback) {
this.callback = callback; this.callback = callback;
return this; return this;
@ -123,7 +131,7 @@ public class TimeseriesSaveRequest {
} }
public TimeseriesSaveRequest build() { public TimeseriesSaveRequest build() {
return new TimeseriesSaveRequest(tenantId, customerId, entityId, entries, ttl, saveLatest, onlyLatest, callback); return new TimeseriesSaveRequest(tenantId, customerId, entityId, entries, ttl, saveLatest, onlyLatest, calculatedFieldIds, callback);
} }
} }

1
rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/TbMsgAttributesNode.java

@ -125,6 +125,7 @@ public class TbMsgAttributesNode implements TbNode {
.scope(scope) .scope(scope)
.entries(attributes) .entries(attributes)
.notifyDevice(config.isNotifyDevice() || checkNotifyDeviceMdValue(msg.getMetaData().getValue(NOTIFY_DEVICE_METADATA_KEY))) .notifyDevice(config.isNotifyDevice() || checkNotifyDeviceMdValue(msg.getMetaData().getValue(NOTIFY_DEVICE_METADATA_KEY)))
.calculatedFieldIds(msg.getCalculatedFieldIds())
.callback(callback) .callback(callback)
.build()); .build());
} }

1
rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/telemetry/TbMsgTimeseriesNode.java

@ -112,6 +112,7 @@ public class TbMsgTimeseriesNode implements TbNode {
.entries(tsKvEntryList) .entries(tsKvEntryList)
.ttl(ttl) .ttl(ttl)
.saveLatest(!config.isSkipLatestPersistence()) .saveLatest(!config.isSkipLatestPersistence())
.calculatedFieldIds(msg.getCalculatedFieldIds())
.callback(new TelemetryNodeCallback(ctx, msg)) .callback(new TelemetryNodeCallback(ctx, msg))
.build()); .build());
} }

Loading…
Cancel
Save